1use crate::extension::RegisteredBackend;
3use crate::{
4 backends::{self, BackendCamera, BackendType},
5 format::*,
6 BackendAvailability, BackendId, BackendOptions, BackendPolicy, CameraConfig, CameraDeviceInfo,
7 CameraError, CameraFacing, CameraResult, CaptureFormat, CaptureMode, CapturePlan,
8 CaptureProfile, CaptureRequest, DeviceCapabilities, DeviceId, DeviceInfo, DeviceSelector,
9 Frame, FrameHub, FrameMetrics, FrameReceiver, IdentityStability, NegotiatedCapture,
10 StreamStats, SubscriptionOptions, UsbIdentity, VideoFormat,
11};
12use std::collections::HashMap;
13use std::sync::{
14 atomic::{AtomicBool, Ordering},
15 Arc,
16};
17use std::time::Duration;
18use tokio::sync::{broadcast, watch, Mutex};
19
20#[derive(Clone, Copy)]
21struct StartupDeadline(tokio::time::Instant);
22
23impl StartupDeadline {
24 fn new(timeout: Duration) -> Self {
25 Self(tokio::time::Instant::now() + timeout)
26 }
27
28 async fn run<T>(
29 self,
30 stage: crate::OperationStage,
31 future: impl std::future::Future<Output = CameraResult<T>>,
32 ) -> CameraResult<T> {
33 tokio::time::timeout_at(self.0, future)
34 .await
35 .map_err(|_| CameraError::timeout(stage))?
36 }
37
38 fn remaining(self) -> Duration {
39 self.0
40 .saturating_duration_since(tokio::time::Instant::now())
41 }
42}
43
44#[derive(Clone, Debug)]
47pub struct CameraSystem {
48 policy: BackendPolicy,
49 synthetic: bool,
50 registered: Arc<HashMap<BackendId, RegisteredBackend>>,
51 backend_options: Arc<Vec<BackendOptions>>,
52}
53
54#[derive(Default)]
55pub struct SystemBuilder {
57 policy: BackendPolicy,
58 registered: HashMap<BackendId, RegisteredBackend>,
59 backend_options: Vec<BackendOptions>,
60}
61
62impl std::fmt::Debug for SystemBuilder {
63 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
64 f.debug_struct("SystemBuilder")
65 .field("policy", &self.policy)
66 .field("registered", &self.registered.keys().collect::<Vec<_>>())
67 .field("backend_options", &self.backend_options)
68 .finish()
69 }
70}
71
72impl SystemBuilder {
73 pub fn backend_policy(mut self, policy: BackendPolicy) -> Self {
75 self.policy = policy;
76 self
77 }
78
79 #[cfg(feature = "custom-backend")]
80 pub fn register_backend(mut self, provider: impl crate::BackendProvider) -> Self {
82 let id = provider.id();
83 self.registered.insert(
84 id.clone(),
85 RegisteredBackend {
86 id,
87 provider: Arc::new(provider),
88 },
89 );
90 self
91 }
92
93 pub fn backend_options(mut self, options: impl Into<BackendOptions>) -> Self {
95 self.backend_options.push(options.into());
96 self
97 }
98
99 pub fn build(self) -> CameraResult<CameraSystem> {
101 for options in &self.backend_options {
102 options.validate()?;
103 }
104 if let BackendPolicy::Require(selected) = &self.policy {
105 for options in &self.backend_options {
106 let option_backend = options.backend();
107 if &option_backend != selected {
108 return Err(CameraError::backend_option_mismatch(
109 option_backend,
110 selected.clone(),
111 ));
112 }
113 }
114 }
115 if let BackendPolicy::Prefer(backends) = &self.policy {
116 if backends.is_empty() {
117 return Err(CameraError::invalid_config(
118 "Backend preference list cannot be empty".into(),
119 ));
120 }
121 }
122 Ok(CameraSystem {
123 policy: self.policy,
124 synthetic: false,
125 registered: Arc::new(self.registered),
126 backend_options: Arc::new(self.backend_options),
127 })
128 }
129}
130
131impl Default for CameraSystem {
132 fn default() -> Self {
133 Self::builder().build().expect("default system is valid")
134 }
135}
136impl CameraSystem {
137 pub fn builder() -> SystemBuilder {
139 SystemBuilder::default()
140 }
141
142 pub fn new() -> Self {
144 Self::default()
145 }
146
147 pub fn synthetic() -> Self {
149 Self {
150 policy: BackendPolicy::Require(BackendId::SYNTHETIC),
151 synthetic: true,
152 registered: Arc::new(HashMap::new()),
153 backend_options: Arc::new(Vec::new()),
154 }
155 }
156
157 pub fn backend_policy(&self) -> &BackendPolicy {
159 &self.policy
160 }
161
162 pub fn available_backends(&self) -> Vec<BackendId> {
164 let mut available = backends::available_backends()
165 .into_iter()
166 .map(BackendId::from)
167 .collect::<Vec<_>>();
168 available.extend(self.registered.keys().cloned());
169 if self.synthetic {
170 available.push(BackendId::SYNTHETIC);
171 }
172 available.sort();
173 available.dedup();
174 available
175 }
176
177 pub fn backend_availability(&self, backend: &BackendId) -> BackendAvailability {
179 if (backend == &BackendId::SYNTHETIC && self.synthetic)
180 || self.registered.contains_key(backend)
181 {
182 return BackendAvailability::Available;
183 }
184 let Some(backend) = backends::backend_type(backend) else {
185 return BackendAvailability::NotCompiled;
186 };
187 if !backend.is_compiled() {
188 BackendAvailability::NotCompiled
189 } else if !backend.supports_target() {
190 BackendAvailability::UnsupportedTarget
191 } else {
192 BackendAvailability::Available
193 }
194 }
195
196 pub fn permission_status(&self, backend: &BackendId) -> CameraResult<crate::PermissionStatus> {
198 if backend == &BackendId::AV_FOUNDATION {
199 #[cfg(all(
200 feature = "backend-avfoundation",
201 any(target_os = "macos", target_os = "ios")
202 ))]
203 {
204 return Ok(
205 crate::backends::avfoundation::camera::AVFoundationCamera::permission_status(),
206 );
207 }
208 #[cfg(not(all(
209 feature = "backend-avfoundation",
210 any(target_os = "macos", target_os = "ios")
211 )))]
212 return Err(CameraError::unsupported_target(
213 backend.clone(),
214 std::env::consts::OS,
215 ));
216 }
217 if backend == &BackendId::CAMERA2 || backend == &BackendId::UVC {
218 Ok(crate::PermissionStatus::ManagedExternally)
219 } else {
220 Ok(crate::PermissionStatus::NotRequired)
221 }
222 }
223
224 pub async fn request_permission(&self, backend: &BackendId) -> CameraResult<bool> {
227 if backend == &BackendId::AV_FOUNDATION {
228 #[cfg(all(
229 feature = "backend-avfoundation",
230 any(target_os = "macos", target_os = "ios")
231 ))]
232 return crate::backends::avfoundation::camera::AVFoundationCamera::request_permission()
233 .await;
234 #[cfg(not(all(
235 feature = "backend-avfoundation",
236 any(target_os = "macos", target_os = "ios")
237 )))]
238 return Err(CameraError::unsupported_target(
239 backend.clone(),
240 std::env::consts::OS,
241 ));
242 }
243 Err(CameraError::permission_managed_externally(backend.clone()))
244 }
245
246 fn policy_backends(&self) -> Vec<BackendId> {
247 match &self.policy {
248 BackendPolicy::PlatformDefault => {
249 vec![BackendId::from(BackendType::default_for_platform())]
250 }
251 BackendPolicy::Require(backend) => vec![backend.clone()],
252 BackendPolicy::Prefer(backends) => backends.clone(),
253 }
254 }
255
256 fn requiring(&self, backend: BackendId) -> Self {
257 Self {
258 policy: BackendPolicy::Require(backend.clone()),
259 synthetic: self.synthetic,
260 registered: self.registered.clone(),
261 backend_options: Arc::new(
262 self.backend_options
263 .iter()
264 .filter(|options| options.backend() == backend)
265 .cloned()
266 .collect(),
267 ),
268 }
269 }
270
271 fn devices_for_backend(&self, backend: &BackendId) -> CameraResult<Vec<DeviceInfo>> {
272 if backend == &BackendId::SYNTHETIC && self.synthetic {
273 return Ok(vec![device_info_synthetic()]);
274 }
275 if let Some(registered) = self.registered.get(backend) {
276 return registered
277 .provider
278 .enumerate()?
279 .into_iter()
280 .enumerate()
281 .map(|(index, info)| {
282 Ok(DeviceInfo {
283 id: DeviceId::new(backend.clone(), info.native_id)?,
284 name: info.name,
285 description: info.description,
286 stability: info.stability,
287 index: index as u32,
288 facing: info.facing,
289 usb: info.usb,
290 })
291 })
292 .collect();
293 }
294 let backend_type = backends::backend_type(backend)
295 .ok_or_else(|| CameraError::backend_not_compiled(backend.clone()))?;
296 let backend_type = resolve_backend(backend_type)?;
297 backends::list_devices(backend_type).map(|devices| {
298 devices
299 .into_iter()
300 .map(|device| device_info(backend_type, device))
301 .collect()
302 })
303 }
304
305 pub fn devices_blocking(&self) -> CameraResult<Vec<DeviceInfo>> {
307 let mut devices = Vec::new();
308 let mut errors = Vec::new();
309 for backend in self.policy_backends() {
310 match self.devices_for_backend(&backend) {
311 Ok(mut found) => devices.append(&mut found),
312 Err(error) => errors.push(error),
313 }
314 }
315 if devices.is_empty() && !errors.is_empty() {
316 return Err(errors.remove(0));
317 }
318 Ok(devices)
319 }
320
321 pub fn capabilities_blocking(&self, id: &DeviceId) -> CameraResult<DeviceCapabilities> {
323 let devices = self.devices_for_backend(&id.backend)?;
324 let matching: Vec<_> = devices.iter().filter(|d| d.id == *id).collect();
325 if matching.len() > 1 {
326 return Err(CameraError::ambiguous_device(id.to_string()));
327 }
328 let device = matching
329 .first()
330 .ok_or_else(|| CameraError::device_not_found(id.to_string()))?;
331 if id.backend == BackendId::SYNTHETIC {
332 return Ok(DeviceCapabilities::single_native(
333 CaptureFormat::Rgb8,
334 640,
335 480,
336 FrameRate::default(),
337 ));
338 }
339 if let Some(registered) = self.registered.get(&id.backend) {
340 return registered.provider.open(id.native_id())?.capabilities();
341 }
342 let backend = backends::backend_type(&id.backend)
343 .ok_or_else(|| CameraError::backend_not_compiled(id.backend.clone()))?;
344 backends::get_device_capabilities(backend, device.index)
345 }
346
347 pub async fn devices(&self) -> CameraResult<Vec<DeviceInfo>> {
349 let system = self.clone();
350 tokio::task::spawn_blocking(move || system.devices_blocking())
351 .await
352 .map_err(join_error)?
353 }
354
355 pub async fn capabilities(&self, id: &DeviceId) -> CameraResult<DeviceCapabilities> {
357 let system = self.clone();
358 let id = id.clone();
359 tokio::task::spawn_blocking(move || system.capabilities_blocking(&id))
360 .await
361 .map_err(join_error)?
362 }
363
364 pub async fn capabilities_timeout(
366 &self,
367 id: &DeviceId,
368 timeout: Duration,
369 ) -> CameraResult<DeviceCapabilities> {
370 if timeout.is_zero() {
371 return Err(CameraError::invalid_config(
372 "Capability query timeout must be positive".into(),
373 ));
374 }
375 tokio::time::timeout(timeout, self.capabilities(id))
376 .await
377 .map_err(|_| CameraError::timeout(crate::OperationStage::Capabilities))?
378 }
379
380 async fn resolve_selector(&self, selector: DeviceSelector) -> CameraResult<DeviceInfo> {
381 let devices = self.devices().await?;
382 let mut matches = devices.into_iter().filter(|device| match &selector {
383 DeviceSelector::Default => true,
384 DeviceSelector::Id(id) => device.id == *id,
385 DeviceSelector::Facing(facing) => device.facing == *facing,
386 DeviceSelector::Usb {
387 vendor_id,
388 product_id,
389 serial_number,
390 } => device.usb.as_ref().is_some_and(|usb| {
391 usb.vendor_id == *vendor_id
392 && usb.product_id == *product_id
393 && serial_number
394 .as_ref()
395 .is_none_or(|serial| usb.serial_number.as_ref() == Some(serial))
396 }),
397 DeviceSelector::Name(name) => device.name.contains(name),
398 });
399 let selected = matches
400 .next()
401 .ok_or_else(|| CameraError::device_not_found(format!("{selector:?}")))?;
402 if !matches!(selector, DeviceSelector::Default) && matches.next().is_some() {
403 return Err(CameraError::ambiguous_device(format!("{selector:?}")));
404 }
405 Ok(selected)
406 }
407
408 pub async fn open(&self, selector: DeviceSelector) -> CameraResult<Device> {
410 self.open_inner(selector).await
411 }
412
413 pub async fn open_timeout(
415 &self,
416 selector: DeviceSelector,
417 timeout: Duration,
418 ) -> CameraResult<Device> {
419 if timeout.is_zero() {
420 return Err(CameraError::invalid_config(
421 "Device open timeout must be positive".into(),
422 ));
423 }
424 self.open_until(selector, StartupDeadline::new(timeout))
425 .await
426 }
427
428 async fn open_until(
429 &self,
430 selector: DeviceSelector,
431 deadline: StartupDeadline,
432 ) -> CameraResult<Device> {
433 deadline
434 .run(crate::OperationStage::Open, self.open_inner(selector))
435 .await
436 }
437
438 async fn open_inner(&self, selector: DeviceSelector) -> CameraResult<Device> {
439 let device = self.resolve_selector(selector).await?;
440 let source = if device.id.backend == BackendId::SYNTHETIC {
441 Source::Synthetic(FrameHub::default())
442 } else if let Some(registered) = self.registered.get(&device.id.backend) {
443 let provider = registered.provider.clone();
444 let native_id = device.id.native.clone();
445 let handle = tokio::task::spawn_blocking(move || provider.open(&native_id))
446 .await
447 .map_err(join_error)??;
448 Source::custom(handle)
449 } else {
450 let backend = backends::backend_type(&device.id.backend)
451 .ok_or_else(|| CameraError::backend_not_compiled(device.id.backend.clone()))?;
452 let index = device.index;
453 let expected = device.id.native.clone();
454 Source::native(
455 tokio::task::spawn_blocking(move || {
456 backends::create_selected_camera(
457 index,
458 backend,
459 FrameHub::default(),
460 Some(&expected),
461 )
462 })
463 .await
464 .map_err(join_error)??,
465 Some(device.id.clone()),
466 )?
467 };
468 let hub = source.hub()?;
469 Ok(Device {
470 device,
471 source,
472 hub,
473 busy: Arc::new(AtomicBool::new(false)),
474 backend_options: self.backend_options.clone(),
475 })
476 }
477 #[cfg(all(
480 unix,
481 all(
482 feature = "backend-uvc",
483 any(target_os = "linux", target_os = "macos", target_os = "android")
484 )
485 ))]
486 pub async fn open_usb_fd(&self, fd: std::os::fd::BorrowedFd<'_>) -> CameraResult<Device> {
487 use std::os::fd::AsRawFd;
488 let owned = fd.try_clone_to_owned()?;
489 let backend =
490 tokio::task::spawn_blocking(move || backends::create_camera_from_fd(owned.as_raw_fd()))
491 .await
492 .map_err(join_error)??;
493 let source = Source::native(backend, None)?;
494 let hub = source.hub()?;
495 Ok(Device {
496 device: DeviceInfo {
497 id: DeviceId {
498 backend: BackendId::UVC,
499 native: "authorized-usb-fd".into(),
500 },
501 name: "Authorized USB camera".into(),
502 description: String::new(),
503 stability: IdentityStability::EnumerationOnly,
504 index: 0,
505 facing: CameraFacing::External,
506 usb: None,
507 },
508 source,
509 hub,
510 busy: Arc::new(AtomicBool::new(false)),
511 backend_options: self.backend_options.clone(),
512 })
513 }
514
515 pub fn capture(&self, selector: DeviceSelector) -> CaptureBuilder {
517 CaptureBuilder::new(self.clone(), selector)
518 }
519
520 pub async fn plan(
522 &self,
523 selector: DeviceSelector,
524 request: CaptureRequest,
525 ) -> CameraResult<CapturePlan> {
526 let device = self.resolve_selector(selector).await?;
527 let mut configurations = if device.id.backend == BackendId::SYNTHETIC {
528 vec![CameraConfig::new(
529 request
530 .preferred_formats
531 .first()
532 .copied()
533 .unwrap_or(CaptureFormat::Rgb8)
534 .into(),
535 request.width,
536 request.height,
537 request.preferred_fps.numerator(),
538 )
539 .with_frame_rate(request.preferred_fps)]
540 } else {
541 let capabilities = self.capabilities(&device.id).await?;
542 let mut configurations = capabilities.configurations()?;
543 if let Some(backend) = backends::backend_type(&device.id.backend) {
544 let index = device.index;
545 let requested = request.clone();
546 let advertised = configurations.clone();
547 configurations.extend(
548 tokio::task::spawn_blocking(move || {
549 backends::requested_configs(backend, index, &requested, &advertised)
550 })
551 .await
552 .map_err(join_error)??,
553 );
554 }
555 configurations
556 };
557 rank_capture_configurations(&request, &mut configurations)?;
558 let selected = configurations[0].clone();
559 let per_frame = selected
560 .frame_size()
561 .unwrap_or_else(|| (selected.width as usize * selected.height as usize).max(1));
562 let estimated_pool_bytes = per_frame
563 .saturating_mul(request.memory.buffers)
564 .min(request.memory.bytes);
565 Ok(CapturePlan {
566 selected: NegotiatedCapture {
567 capture: CaptureMode::from_config(&selected)?,
568 adjustments: Vec::new(),
569 first_frame_latency: std::time::Duration::ZERO,
570 },
571 alternatives: configurations
572 .iter()
573 .skip(1)
574 .map(CaptureMode::from_config)
575 .collect::<CameraResult<Vec<_>>>()?,
576 ranking_reason: format!(
577 "ranked for {:?}: format preference, required FPS, resolution, then frame rate",
578 request.priority
579 ),
580 estimated_pool_bytes,
581 adjustments: Vec::new(),
582 })
583 }
584}
585fn device_info_synthetic() -> DeviceInfo {
586 DeviceInfo {
587 id: DeviceId {
588 backend: BackendId::SYNTHETIC,
589 native: "test-pattern".into(),
590 },
591 name: "Synthetic RGB camera".into(),
592 description: "Deterministic capture source".into(),
593 stability: IdentityStability::Native,
594 index: 0,
595 facing: CameraFacing::Unknown,
596 usb: None,
597 }
598}
599fn device_info(backend: BackendType, d: CameraDeviceInfo) -> DeviceInfo {
600 let stable = d.device_path.as_ref().is_some_and(|s| {
601 !s.is_empty() && !(backend == BackendType::V4l2 && s.starts_with("/dev/video"))
602 }) || d.serial_number.as_ref().is_some_and(|s| !s.is_empty());
603 let facing = infer_facing(&d.name, &d.description);
604 let usb = d
605 .vendor_id
606 .zip(d.product_id)
607 .map(|(vendor_id, product_id)| UsbIdentity {
608 vendor_id,
609 product_id,
610 serial_number: d.serial_number.clone(),
611 });
612 DeviceInfo {
613 id: DeviceId {
614 backend: backend.into(),
615 native: d.unique_id(),
616 },
617 name: d.name,
618 description: d.description,
619 index: d.index,
620 stability: if stable {
621 IdentityStability::Native
622 } else {
623 IdentityStability::EnumerationOnly
624 },
625 facing,
626 usb,
627 }
628}
629
630fn infer_facing(name: &str, description: &str) -> CameraFacing {
631 let value = format!("{name} {description}").to_ascii_lowercase();
632 if value.contains("front") || value.contains("user") {
633 CameraFacing::Front
634 } else if value.contains("back") || value.contains("rear") || value.contains("environment") {
635 CameraFacing::Back
636 } else if value.contains("usb") || value.contains("external") {
637 CameraFacing::External
638 } else {
639 CameraFacing::Unknown
640 }
641}
642fn resolve_backend(b: BackendType) -> CameraResult<BackendType> {
643 let b = if b == BackendType::Auto {
644 BackendType::default_for_platform()
645 } else {
646 b
647 };
648 if !b.is_compiled() {
649 Err(CameraError::backend_not_compiled(b.into()))
650 } else if !b.supports_target() {
651 Err(CameraError::unsupported_target(
652 b.into(),
653 std::env::consts::OS,
654 ))
655 } else {
656 Ok(b)
657 }
658}
659fn join_error(e: tokio::task::JoinError) -> CameraError {
660 CameraError::worker_failure(format!("Camera worker: {e}")).with_source(e)
661}
662#[derive(Clone)]
663enum Source {
664 Native(Arc<NativeSource>),
665 Custom(Arc<CustomSource>),
666 Synthetic(FrameHub),
667}
668
669struct CustomSource {
670 device: parking_lot::Mutex<Box<dyn crate::extension::BackendDevice>>,
671 hub: FrameHub,
672 actual: parking_lot::Mutex<Option<CameraConfig>>,
673}
674
675struct NativeSource {
676 camera: parking_lot::Mutex<Option<BackendCamera>>,
677 id: Option<DeviceId>,
678 hub: FrameHub,
679}
680impl NativeSource {
681 fn current(&self) -> CameraResult<BackendCamera> {
682 self.camera
683 .lock()
684 .clone()
685 .ok_or_else(|| CameraError::disconnected("Device is reconnecting".into()))
686 }
687 fn backend_type(&self) -> BackendType {
688 self.id
689 .as_ref()
690 .and_then(|id| backends::backend_type(&id.backend))
691 .unwrap_or(BackendType::Uvc)
692 }
693 fn get_config(&self) -> Option<CameraConfig> {
694 self.current().ok().and_then(|c| c.get_config())
695 }
696 async fn start_stream(&self, c: CameraConfig) -> CameraResult<()> {
697 self.current()?.start_stream(c).await
698 }
699 async fn stop_stream(&self) -> CameraResult<()> {
700 if let Ok(c) = self.current() {
701 c.stop_stream().await
702 } else {
703 Ok(())
704 }
705 }
706 async fn set_buffer_size(&self, n: usize) -> CameraResult<()> {
707 self.current()?.set_buffer_size(n).await
708 }
709 async fn reopen(self: &Arc<Self>) -> CameraResult<()> {
710 let id = self.id.clone().ok_or_else(|| {
711 CameraError::disconnected(
712 "USB permission descriptor requires explicit reopening".into(),
713 )
714 })?;
715 let this = self.clone();
716 tokio::task::spawn_blocking(move || {
717 let _ = this.camera.lock().take();
718 let backend = backends::backend_type(&id.backend)
719 .ok_or_else(|| CameraError::backend_not_compiled(id.backend.clone()))?;
720 let mut matches = backends::list_devices(backend)?
721 .into_iter()
722 .filter(|d| d.unique_id() == id.native);
723 let device = matches
724 .next()
725 .ok_or_else(|| CameraError::disconnected(id.to_string()))?;
726 if matches.next().is_some() {
727 return Err(CameraError::ambiguous_device(id.to_string()));
728 }
729 let camera = backends::create_selected_camera(
730 device.index,
731 backend,
732 this.hub.clone(),
733 Some(&id.native),
734 )?;
735 *this.camera.lock() = Some(camera);
736 Ok(())
737 })
738 .await
739 .map_err(join_error)?
740 }
741}
742impl Source {
743 fn native(camera: BackendCamera, id: Option<DeviceId>) -> CameraResult<Self> {
744 let hub = camera
745 .frame_hub()
746 .cloned()
747 .ok_or_else(|| CameraError::invalid_state("Backend has no frame source".into()))?;
748 Ok(Self::Native(Arc::new(NativeSource {
749 camera: parking_lot::Mutex::new(Some(camera)),
750 id,
751 hub,
752 })))
753 }
754
755 fn custom(device: Box<dyn crate::extension::BackendDevice>) -> Self {
756 Self::Custom(Arc::new(CustomSource {
757 device: parking_lot::Mutex::new(device),
758 hub: FrameHub::default(),
759 actual: parking_lot::Mutex::new(None),
760 }))
761 }
762
763 fn hub(&self) -> CameraResult<FrameHub> {
764 match self {
765 Self::Native(c) => Ok(c.hub.clone()),
766 Self::Custom(c) => Ok(c.hub.clone()),
767 Self::Synthetic(h) => Ok(h.clone()),
768 }
769 }
770 async fn start(&self, config: CameraConfig) -> CameraResult<()> {
771 match self {
772 Self::Native(c) => c.start_stream(config).await,
773 Self::Custom(c) => {
774 let c = c.clone();
775 tokio::task::spawn_blocking(move || {
776 let sink = crate::extension::FrameSink::begin(c.hub.clone());
777 let plan = CapturePlan {
778 selected: NegotiatedCapture {
779 capture: CaptureMode::from_config(&config)?,
780 adjustments: Vec::new(),
781 first_frame_latency: Duration::ZERO,
782 },
783 alternatives: Vec::new(),
784 ranking_reason: "selected by the camera negotiation layer".into(),
785 estimated_pool_bytes: 0,
786 adjustments: Vec::new(),
787 };
788 c.device.lock().start(sink, &plan)?;
789 *c.actual.lock() = Some(config);
790 Ok(())
791 })
792 .await
793 .map_err(join_error)?
794 }
795 Self::Synthetic(h) => {
796 let session = h.start();
797 let hub = h.clone();
798 let interval = config.frame_rate()?.interval();
799 tokio::spawn(async move {
800 let mut ticker = tokio::time::interval(interval);
801 ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
802 let mut seq = 0u64;
803 while hub.is_streaming() && hub.session() == session {
804 ticker.tick().await;
805 if !hub.is_streaming() || hub.session() != session {
806 break;
807 }
808 seq += 1;
809 if hub.wants_native() {
810 let _ = hub.publish_native_into(
811 session,
812 FrameLayout::rgb(config.width, config.height),
813 None,
814 |out| {
815 out.fill((seq % 255) as u8);
816 Ok(())
817 },
818 );
819 } else {
820 let _ =
821 hub.publish_rgb(session, config.width, config.height, None, |b| {
822 b.fill((seq % 255) as u8);
823 Ok(())
824 });
825 }
826 }
827 });
828 Ok(())
829 }
830 }
831 }
832 async fn stop(&self) -> CameraResult<()> {
833 match self {
834 Self::Native(c) => c.stop_stream().await,
835 Self::Custom(c) => {
836 let c = c.clone();
837 tokio::task::spawn_blocking(move || {
838 c.hub.stop();
839 let result = c.device.lock().stop();
840 *c.actual.lock() = None;
841 result
842 })
843 .await
844 .map_err(join_error)?
845 }
846 Self::Synthetic(h) => {
847 h.stop();
848 Ok(())
849 }
850 }
851 }
852 fn actual(&self, fallback: &CameraConfig) -> CameraResult<CameraConfig> {
853 match self {
854 Self::Native(c) => c.get_config().ok_or_else(|| {
855 CameraError::invalid_state("Backend did not report negotiated configuration".into())
856 }),
857 Self::Synthetic(_) => Ok(fallback.clone()),
858 Self::Custom(c) => c.actual.lock().clone().ok_or_else(|| {
859 CameraError::invalid_state("External backend did not start a capture".into())
860 }),
861 }
862 }
863
864 async fn external_capabilities(&self) -> CameraResult<Option<DeviceCapabilities>> {
865 match self {
866 Self::Custom(c) => {
867 let c = c.clone();
868 tokio::task::spawn_blocking(move || c.device.lock().capabilities())
869 .await
870 .map_err(join_error)?
871 .map(Some)
872 }
873 _ => Ok(None),
874 }
875 }
876}
877
878async fn apply_backend_options(
879 source: &Source,
880 selected_backend: &BackendId,
881 options: &[BackendOptions],
882) -> CameraResult<()> {
883 for option in options {
884 if &option.backend() != selected_backend {
885 return Err(CameraError::backend_option_mismatch(
886 option.backend(),
887 selected_backend.clone(),
888 ));
889 }
890 let Source::Native(native) = source else {
891 continue;
892 };
893 match option {
894 BackendOptions::V4l2(options) => {
895 if native.backend_type() != BackendType::V4l2 {
896 return Err(CameraError::invalid_config(
897 "Driver queue configuration is only supported by V4L2".into(),
898 ));
899 }
900 if let Some(count) = options.mmap_buffers {
901 native.set_buffer_size(count).await?;
902 }
903 }
904 BackendOptions::Camera2(options) => {
905 native.current()?.set_camera2_options(options.clone())?;
906 }
907 BackendOptions::AvFoundation(options) => {
908 native
909 .current()?
910 .set_avfoundation_options(options.clone())?;
911 }
912 }
913 }
914 Ok(())
915}
916pub struct Camera {
918 device: DeviceInfo,
919 source: Source,
920 hub: FrameHub,
921 busy: Arc<AtomicBool>,
922 backend_options: Arc<Vec<BackendOptions>>,
923}
924pub type Device = Camera;
926impl std::fmt::Debug for Camera {
927 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
928 f.debug_struct("Camera")
929 .field("device", &self.device)
930 .field("busy", &self.busy.load(Ordering::Acquire))
931 .finish()
932 }
933}
934impl Camera {
935 pub fn device(&self) -> &DeviceInfo {
936 &self.device
937 }
938
939 pub async fn capabilities(&self) -> CameraResult<DeviceCapabilities> {
943 match &self.source {
944 Source::Synthetic(_) => Ok(DeviceCapabilities::single_native(
945 CaptureFormat::Rgb8,
946 640,
947 480,
948 FrameRate::default(),
949 )),
950 Source::Custom(source) => {
951 let source = source.clone();
952 tokio::task::spawn_blocking(move || source.device.lock().capabilities())
953 .await
954 .map_err(join_error)?
955 }
956 Source::Native(source) if source.id.is_none() => {
957 let camera = source.current()?;
958 tokio::task::spawn_blocking(move || camera.opened_device_capabilities())
959 .await
960 .map_err(join_error)?
961 }
962 Source::Native(_) => {
963 let backend =
964 backends::backend_type(self.device.id.backend()).ok_or_else(|| {
965 CameraError::backend_not_compiled(self.device.id.backend().clone())
966 })?;
967 let index = self.device.index;
968 tokio::task::spawn_blocking(move || {
969 backends::get_device_capabilities(backend, index)
970 })
971 .await
972 .map_err(join_error)?
973 }
974 }
975 }
976
977 pub async fn start(&mut self, request: CaptureRequest) -> CameraResult<Session> {
978 let deadline = StartupDeadline::new(request.startup_timeout);
979 self.start_until(request, deadline).await
980 }
981
982 async fn start_until(
983 &mut self,
984 request: CaptureRequest,
985 deadline: StartupDeadline,
986 ) -> CameraResult<Session> {
987 let mut raw = request.to_stream_request()?;
988 for options in self.backend_options.iter() {
989 match options {
990 BackendOptions::V4l2(options) if self.device.id.backend == BackendId::V4L2 => {
991 raw.driver_buffers = options.mmap_buffers;
992 }
993 BackendOptions::Camera2(_) if self.device.id.backend == BackendId::CAMERA2 => {}
994 BackendOptions::AvFoundation(_)
995 if self.device.id.backend == BackendId::AV_FOUNDATION => {}
996 options if options.backend() != self.device.id.backend => {
997 return Err(CameraError::backend_option_mismatch(
998 options.backend(),
999 self.device.id.backend.clone(),
1000 ));
1001 }
1002 _ => {}
1003 }
1004 }
1005 let candidates = if matches!(self.source, Source::Synthetic(_)) {
1006 vec![CameraConfig::new(
1007 VideoFormat::RGB,
1008 request.width,
1009 request.height,
1010 request.preferred_fps.numerator(),
1011 )
1012 .with_frame_rate(request.preferred_fps)]
1013 } else if matches!(&self.source, Source::Native(native) if native.id.is_none()) {
1014 direct_capture_candidates(&request)
1015 } else {
1016 let external = deadline
1017 .run(
1018 crate::OperationStage::Startup,
1019 self.source.external_capabilities(),
1020 )
1021 .await?;
1022 let mut configurations = if let Some(capabilities) = external {
1023 capabilities.configurations()?
1024 } else {
1025 let backend = backends::backend_type(&self.device.id.backend).ok_or_else(|| {
1026 CameraError::backend_not_compiled(self.device.id.backend.clone())
1027 })?;
1028 let index = self.device.index;
1029 let requested = request.clone();
1030 deadline
1031 .run(crate::OperationStage::Startup, async move {
1032 tokio::task::spawn_blocking(move || {
1033 let capabilities = backends::get_device_capabilities(backend, index)?;
1034 let mut configurations = capabilities.configurations()?;
1035 let requested_configurations = backends::requested_configs(
1036 backend,
1037 index,
1038 &requested,
1039 &configurations,
1040 )?;
1041 configurations.extend(requested_configurations);
1042 Ok(configurations)
1043 })
1044 .await
1045 .map_err(join_error)?
1046 })
1047 .await?
1048 };
1049 rank_capture_configurations(&request, &mut configurations)?;
1050 configurations
1051 };
1052 self.start_raw(raw, candidates, request, deadline).await
1053 }
1054
1055 async fn start_raw(
1056 &mut self,
1057 request: StreamRequest,
1058 candidates: Vec<CameraConfig>,
1059 intent: CaptureRequest,
1060 deadline: StartupDeadline,
1061 ) -> CameraResult<CaptureSession> {
1062 self.start_inner(request, candidates, intent, deadline)
1063 .await
1064 }
1065 async fn start_inner(
1066 &mut self,
1067 request: StreamRequest,
1068 mut candidates: Vec<CameraConfig>,
1069 intent: CaptureRequest,
1070 deadline: StartupDeadline,
1071 ) -> CameraResult<CaptureSession> {
1072 let startup_started = std::time::Instant::now();
1073 if request.output == OutputFormat::Rgb8 && request.format == Some(VideoFormat::H264) {
1074 return Err(CameraError::unsupported_format(
1075 "H264 requires native output and an external decoder".into(),
1076 ));
1077 }
1078 if matches!(self.source, Source::Synthetic(_))
1079 && request.format.is_some_and(|f| f != VideoFormat::RGB)
1080 {
1081 return Err(CameraError::unsupported_format(
1082 "Synthetic source produces RGB only".into(),
1083 ));
1084 }
1085 if request.reconnect.is_some()
1086 && self.device.stability == IdentityStability::EnumerationOnly
1087 {
1088 return Err(CameraError::invalid_config(
1089 "Automatic reconnect requires a stable native identity".into(),
1090 ));
1091 }
1092 if self.busy.swap(true, Ordering::AcqRel) {
1093 return Err(CameraError::invalid_state(
1094 "Camera is already starting, capturing, or closing".into(),
1095 ));
1096 }
1097 if let Err(e) = self.hub.configure(&request) {
1098 self.busy.store(false, Ordering::Release);
1099 return Err(e);
1100 }
1101 let (state, _) = watch::channel(SessionState::Starting);
1102 let (events, _) = broadcast::channel(64);
1103 self.hub.set_event_sender(events.clone());
1104 let inner = Arc::new(SessionInner {
1105 device: self.device.clone(),
1106 source: self.source.clone(),
1107 hub: self.hub.clone(),
1108 busy: self.busy.clone(),
1109 state,
1110 events,
1111 stopping: Arc::new(AtomicBool::new(false)),
1112 closed: AtomicBool::new(false),
1113 terminal_metrics: parking_lot::Mutex::new(None),
1114 requested_controls: std::sync::Mutex::new(std::collections::HashMap::new()),
1115 lifecycle: Mutex::new(()),
1116 intent: intent.clone(),
1117 backend_options: self.backend_options.clone(),
1118 });
1119 let guard = StartGuard(Some(inner.clone()));
1120 deadline
1121 .run(
1122 crate::OperationStage::Startup,
1123 apply_backend_options(
1124 &inner.source,
1125 &inner.device.id.backend,
1126 inner.backend_options.as_ref(),
1127 ),
1128 )
1129 .await?;
1130 let (tx, rx) = tokio::sync::oneshot::channel();
1131 let task_inner = inner.clone();
1132 let task_intent = intent.clone();
1133 tokio::spawn(async move {
1134 let _lock = task_inner.lifecycle.lock().await;
1135 let mut result = Err(CameraError::unsupported_format(
1136 "No usable capture configuration".into(),
1137 ));
1138 for config in candidates.drain(..) {
1139 if task_inner.stopping.load(Ordering::Acquire) || tx.is_closed() {
1140 break;
1141 }
1142 let attempt = async {
1143 let previous_session = task_inner.hub.session();
1144 task_inner.source.start(config.clone()).await?;
1145 let session = task_inner.hub.session();
1146 if session == previous_session {
1147 return Err(CameraError::invalid_state(
1148 "Backend did not begin a new frame session".into(),
1149 ));
1150 }
1151 let frame = task_inner
1152 .hub
1153 .wait_first_in(session, deadline.remaining())
1154 .await?;
1155 let actual = task_inner.source.actual(&config)?;
1156 validate_requested_capture(&task_intent, &actual, frame.layout())?;
1157 Ok::<_, CameraError>((actual, frame))
1158 }
1159 .await;
1160 match attempt {
1161 Ok(ok) => {
1162 result = Ok(ok);
1163 break;
1164 }
1165 Err(e) => {
1166 result = Err(e);
1167 let _ = task_inner.source.stop().await;
1168 }
1169 }
1170 }
1171 if task_inner.stopping.load(Ordering::Acquire) || tx.is_closed() || result.is_err() {
1172 let _ = task_inner.source.stop().await;
1173 task_inner.stopping.store(true, Ordering::Release);
1174 task_inner.closed.store(true, Ordering::Release);
1175 task_inner.set_state(SessionState::Stopped);
1176 task_inner.busy.store(false, Ordering::Release);
1177 }
1178 let _ = tx.send(result);
1179 });
1180 let (actual, _first) = tokio::time::timeout_at(deadline.0, rx)
1181 .await
1182 .map_err(|_| CameraError::timeout(crate::OperationStage::FirstFrame))?
1183 .map_err(|_| CameraError::stream_stopped())??;
1184 let frame_rate = actual.frame_rate()?;
1185 let mut adjustments = Vec::new();
1186 if (actual.width, actual.height) != (request.width, request.height) {
1187 adjustments.push(format!(
1188 "Resolution {}x{} selected",
1189 actual.width, actual.height
1190 ));
1191 }
1192 if frame_rate != request.fps {
1193 adjustments.push(format!(
1194 "Requested {} fps, selected {} fps",
1195 request.fps.as_f64(),
1196 frame_rate.as_f64()
1197 ));
1198 }
1199 if request.format.is_some_and(|f| f != actual.format) {
1200 adjustments.push(format!("Capture format {:?} selected", actual.format));
1201 }
1202 inner.set_state(SessionState::Streaming);
1203 for adjustment in &adjustments {
1204 let _ = inner
1205 .events
1206 .send(SessionEvent::ConfigurationAdjusted(adjustment.clone()));
1207 }
1208 let session = CaptureSession {
1209 inner,
1210 negotiated: NegotiatedCapture {
1211 capture: CaptureMode::from_config(&actual)?,
1212 first_frame_latency: startup_started.elapsed(),
1213 adjustments,
1214 },
1215 actual,
1216 };
1217 if let Some(policy) = request.reconnect {
1218 session.inner.hub.enable_recovery();
1219 session.spawn_recovery(policy);
1220 }
1221 guard.disarm();
1222 Ok(session)
1223 }
1224}
1225
1226#[derive(Clone, Debug)]
1227pub struct CaptureBuilder {
1229 system: CameraSystem,
1230 selector: DeviceSelector,
1231 request: CaptureRequest,
1232}
1233
1234impl CaptureBuilder {
1235 fn new(system: CameraSystem, selector: DeviceSelector) -> Self {
1236 Self {
1237 system,
1238 selector,
1239 request: CaptureRequest::builder()
1240 .build()
1241 .expect("default capture request is valid"),
1242 }
1243 }
1244
1245 pub fn profile(mut self, profile: CaptureProfile) -> Self {
1247 let builder = match profile {
1248 CaptureProfile::Preview => CaptureRequest::builder()
1249 .preferred_resolution(1280, 720)
1250 .priority(crate::NegotiationPriority::LowLatency),
1251 CaptureProfile::Recognition => CaptureRequest::builder()
1252 .preferred_resolution(1280, 720)
1253 .priority(crate::NegotiationPriority::LowCpu),
1254 CaptureProfile::Recording => CaptureRequest::builder()
1255 .preferred_resolution(1920, 1080)
1256 .priority(crate::NegotiationPriority::Fidelity),
1257 CaptureProfile::NativeRelay => CaptureRequest::builder()
1258 .preferred_resolution(1280, 720)
1259 .priority(crate::NegotiationPriority::LowBandwidth)
1260 .preferred_formats([CaptureFormat::Mjpeg, CaptureFormat::H264]),
1261 };
1262 self.request = builder
1263 .build()
1264 .expect("built-in capture profiles are valid");
1265 self
1266 }
1267
1268 pub fn resolution(mut self, width: u32, height: u32) -> Self {
1270 self.request.width = width;
1271 self.request.height = height;
1272 self
1273 }
1274
1275 pub fn frame_rate(mut self, frame_rate: FrameRate) -> Self {
1277 self.request.preferred_fps = frame_rate;
1278 self
1279 }
1280
1281 pub fn request(mut self, request: CaptureRequest) -> Self {
1283 self.request = request;
1284 self
1285 }
1286
1287 pub async fn start(self) -> CameraResult<Capture> {
1289 self.request.to_stream_request()?;
1290 let deadline = StartupDeadline::new(self.request.startup_timeout);
1291 if matches!(self.selector, DeviceSelector::Default) {
1292 if let BackendPolicy::Prefer(backends) = self.system.backend_policy() {
1293 let mut attempts = Vec::new();
1294 for backend in backends.clone() {
1295 let system = self.system.requiring(backend.clone());
1296 let attempt: CameraResult<Capture> = async {
1297 let mut device =
1298 system.open_until(DeviceSelector::Default, deadline).await?;
1299 let session = device.start_until(self.request.clone(), deadline).await?;
1300 let receiver = session.subscribe(SubscriptionOptions::latest())?;
1301 Ok(Capture { session, receiver })
1302 }
1303 .await;
1304 match attempt {
1305 Ok(capture) => return Ok(capture),
1306 Err(error)
1307 if matches!(
1308 error.kind(),
1309 crate::CameraErrorKind::PermissionDenied
1310 | crate::CameraErrorKind::PermissionRequired
1311 ) =>
1312 {
1313 return Err(error)
1314 }
1315 Err(error) => {
1316 attempts.push(crate::BackendAttempt::new(backend, Arc::new(error)))
1317 }
1318 }
1319 }
1320 return Err(CameraError::all_backends_failed(attempts));
1321 }
1322 }
1323 let mut device = self.system.open_until(self.selector, deadline).await?;
1324 let session = device.start_until(self.request, deadline).await?;
1325 let receiver = session.subscribe(SubscriptionOptions::latest())?;
1326 Ok(Capture { session, receiver })
1327 }
1328}
1329
1330#[derive(Debug)]
1331pub struct Capture {
1333 session: Session,
1334 receiver: FrameReceiver,
1335}
1336
1337impl Capture {
1338 pub async fn next_frame(&mut self) -> CameraResult<Frame> {
1340 self.receiver.next().await
1341 }
1342
1343 pub fn negotiated(&self) -> &NegotiatedCapture {
1345 self.session.negotiated()
1346 }
1347
1348 pub fn session(&self) -> &Session {
1350 &self.session
1351 }
1352
1353 pub async fn close(&self) -> CameraResult<()> {
1355 self.session.close().await
1356 }
1357}
1358
1359fn validate_requested_capture(
1360 request: &CaptureRequest,
1361 actual: &CameraConfig,
1362 first_layout: &crate::FrameLayout,
1363) -> CameraResult<()> {
1364 actual.validate()?;
1365 let frame_rate = actual.frame_rate()?;
1366 if request
1367 .minimum_fps
1368 .is_some_and(|minimum| frame_rate < minimum)
1369 {
1370 return Err(CameraError::unsupported_format(format!(
1371 "Driver negotiated {} fps below required {} fps",
1372 frame_rate.as_f64(),
1373 request.minimum_fps.unwrap().as_f64()
1374 )));
1375 }
1376 if !request.preferred_formats.is_empty()
1377 && !request
1378 .preferred_formats
1379 .iter()
1380 .any(|format| VideoFormat::from(*format) == actual.format)
1381 {
1382 return Err(CameraError::unsupported_format(format!(
1383 "Driver negotiated unaccepted format {:?}",
1384 actual.format
1385 )));
1386 }
1387 if request.exact_resolution && (actual.width, actual.height) != (request.width, request.height)
1388 {
1389 return Err(CameraError::unsupported_format(format!(
1390 "Driver negotiated {}x{} instead of the required {}x{} resolution",
1391 actual.width, actual.height, request.width, request.height
1392 )));
1393 }
1394 if request
1395 .required_aspect_ratio
1396 .is_some_and(|(width, height)| {
1397 u64::from(actual.width) * u64::from(height)
1398 != u64::from(actual.height) * u64::from(width)
1399 })
1400 {
1401 return Err(CameraError::unsupported_format(format!(
1402 "Driver negotiated {}x{} outside the required aspect ratio",
1403 actual.width, actual.height
1404 )));
1405 }
1406 if first_layout.width != actual.width || first_layout.height != actual.height {
1407 return Err(CameraError::invalid_frame(
1408 "First frame differs from negotiated dimensions".into(),
1409 ));
1410 }
1411 Ok(())
1412}
1413
1414fn rank_capture_configurations(
1415 request: &CaptureRequest,
1416 configurations: &mut Vec<CameraConfig>,
1417) -> CameraResult<()> {
1418 configurations.retain(|config| {
1419 config.validate().is_ok()
1420 && (request.preferred_formats.is_empty()
1421 || request
1422 .preferred_formats
1423 .iter()
1424 .any(|format| VideoFormat::from(*format) == config.format))
1425 && request
1426 .minimum_fps
1427 .is_none_or(|minimum| config.frame_rate().is_ok_and(|rate| rate >= minimum))
1428 && (!request.exact_resolution
1429 || (config.width, config.height) == (request.width, request.height))
1430 && request.required_aspect_ratio.is_none_or(|(width, height)| {
1431 u64::from(config.width) * u64::from(height)
1432 == u64::from(config.height) * u64::from(width)
1433 })
1434 });
1435 let preference = |format: VideoFormat| {
1436 request
1437 .preferred_formats
1438 .iter()
1439 .position(|preferred| VideoFormat::from(*preferred) == format)
1440 .unwrap_or(request.preferred_formats.len())
1441 };
1442 let cost = |format: VideoFormat| match request.priority {
1443 crate::NegotiationPriority::LowCpu => match format {
1444 VideoFormat::RGB | VideoFormat::NV12 | VideoFormat::YUYV | VideoFormat::UYVY => 0,
1445 VideoFormat::Gray => 1,
1446 VideoFormat::MJPEG => 2,
1447 VideoFormat::H264 => 3,
1448 },
1449 crate::NegotiationPriority::LowBandwidth => match format {
1450 VideoFormat::H264 => 0,
1451 VideoFormat::MJPEG => 1,
1452 VideoFormat::NV12 | VideoFormat::Gray => 2,
1453 _ => 3,
1454 },
1455 crate::NegotiationPriority::LowLatency => match format {
1456 VideoFormat::NV12 | VideoFormat::YUYV | VideoFormat::UYVY | VideoFormat::RGB => 0,
1457 VideoFormat::Gray => 1,
1458 _ => 2,
1459 },
1460 crate::NegotiationPriority::Fidelity => 0,
1461 };
1462 configurations.sort_by_key(|config| {
1463 let resolution_delta = (i64::from(config.width) - i64::from(request.width)).unsigned_abs()
1464 + (i64::from(config.height) - i64::from(request.height)).unsigned_abs();
1465 let frame_rate_delta = config.frame_rate().map_or(u64::MAX, |rate| {
1466 ((rate.as_f64() - request.preferred_fps.as_f64()).abs() * 1_000_000.0) as u64
1467 });
1468 (
1469 preference(config.format),
1470 cost(config.format),
1471 resolution_delta,
1472 frame_rate_delta,
1473 )
1474 });
1475 configurations.dedup();
1476 if configurations.is_empty() {
1477 return Err(CameraError::unsupported_format(
1478 "No capture mode satisfies the required constraints".into(),
1479 ));
1480 }
1481 Ok(())
1482}
1483
1484fn direct_capture_candidates(request: &CaptureRequest) -> Vec<CameraConfig> {
1485 let formats = if request.preferred_formats.is_empty() {
1486 vec![CaptureFormat::Mjpeg]
1487 } else {
1488 request.preferred_formats.clone()
1489 };
1490 formats
1491 .into_iter()
1492 .map(|format| {
1493 CameraConfig::new(
1494 format.into(),
1495 request.width,
1496 request.height,
1497 request.preferred_fps.numerator(),
1498 )
1499 .with_frame_rate(request.preferred_fps)
1500 })
1501 .collect()
1502}
1503
1504#[derive(Clone, Debug, PartialEq, Eq)]
1505pub enum SessionState {
1507 Starting,
1509 Streaming,
1511 Recovering {
1513 attempt: u32,
1515 },
1516 Stopping,
1518 Stopped,
1520 Failed(Arc<CameraError>),
1522}
1523impl SessionState {
1524 pub fn is_streaming(&self) -> bool {
1526 matches!(self, Self::Streaming)
1527 }
1528
1529 pub fn is_terminal(&self) -> bool {
1531 matches!(self, Self::Stopped | Self::Failed(_))
1532 }
1533
1534 pub fn error(&self) -> Option<&CameraError> {
1536 match self {
1537 Self::Failed(error) => Some(error),
1538 _ => None,
1539 }
1540 }
1541}
1542
1543#[derive(Clone, Debug, PartialEq, Eq)]
1544pub enum SessionEvent {
1546 Lifecycle(SessionState),
1548 ConfigurationAdjusted(String),
1550 ResourcePressure {
1552 dropped_frames: u64,
1554 },
1555 Missed {
1557 count: u64,
1559 },
1560}
1561
1562impl SessionEvent {
1563 pub fn is_lifecycle(&self) -> bool {
1565 matches!(self, Self::Lifecycle(_))
1566 }
1567}
1568
1569#[derive(Debug)]
1570pub struct StateWatcher {
1572 receiver: watch::Receiver<SessionState>,
1573}
1574
1575impl StateWatcher {
1576 pub fn current(&self) -> SessionState {
1578 self.receiver.borrow().clone()
1579 }
1580
1581 pub async fn changed(&mut self) -> CameraResult<SessionState> {
1583 self.receiver
1584 .changed()
1585 .await
1586 .map_err(|_| CameraError::stream_stopped())?;
1587 Ok(self.current())
1588 }
1589}
1590
1591#[derive(Debug)]
1592pub struct EventReceiver {
1594 receiver: broadcast::Receiver<SessionEvent>,
1595}
1596
1597impl EventReceiver {
1598 pub async fn next(&mut self) -> CameraResult<SessionEvent> {
1600 match self.receiver.recv().await {
1601 Ok(event) => Ok(event),
1602 Err(broadcast::error::RecvError::Lagged(count)) => Ok(SessionEvent::Missed { count }),
1603 Err(broadcast::error::RecvError::Closed) => Err(CameraError::stream_stopped()),
1604 }
1605 }
1606}
1607struct SessionInner {
1608 device: DeviceInfo,
1609 source: Source,
1610 hub: FrameHub,
1611 busy: Arc<AtomicBool>,
1612 state: watch::Sender<SessionState>,
1613 events: broadcast::Sender<SessionEvent>,
1614 stopping: Arc<AtomicBool>,
1615 closed: AtomicBool,
1616 terminal_metrics: parking_lot::Mutex<Option<(FrameMetrics, StreamStats)>>,
1617 requested_controls:
1618 std::sync::Mutex<std::collections::HashMap<crate::ControlId, crate::ControlValue>>,
1619 lifecycle: Mutex<()>,
1620 intent: CaptureRequest,
1621 backend_options: Arc<Vec<BackendOptions>>,
1622}
1623impl SessionInner {
1624 fn set_state(&self, state: SessionState) {
1625 self.state.send_replace(state.clone());
1626 let _ = self.events.send(SessionEvent::Lifecycle(state));
1627 }
1628
1629 fn request_stop(self: &Arc<Self>) {
1630 if self.stopping.swap(true, Ordering::AcqRel) || self.closed.load(Ordering::Acquire) {
1631 return;
1632 }
1633 self.set_state(SessionState::Stopping);
1634 self.hub.disable_recovery();
1635 self.hub.end_recovery();
1636 self.hub.stop();
1637 let inner = self.clone();
1638 if let Ok(runtime) = tokio::runtime::Handle::try_current() {
1639 runtime.spawn(async move {
1640 let _ = inner.stop().await;
1641 });
1642 } else {
1643 std::thread::spawn(move || {
1644 if let Ok(rt) = tokio::runtime::Builder::new_current_thread()
1645 .enable_all()
1646 .build()
1647 {
1648 let _ = rt.block_on(inner.stop());
1649 }
1650 });
1651 }
1652 }
1653 async fn stop(&self) -> CameraResult<()> {
1654 let _guard = self.lifecycle.lock().await;
1655 if self.closed.load(Ordering::Acquire) {
1656 return match &*self.state.borrow() {
1657 SessionState::Failed(error) => Err((**error).clone()),
1658 _ => Ok(()),
1659 };
1660 }
1661 self.stopping.store(true, Ordering::Release);
1662 self.hub.disable_recovery();
1663 self.hub.end_recovery();
1664 self.hub.stop();
1665 let result = self.source.stop().await;
1666 *self.terminal_metrics.lock() = Some((self.hub.metrics(), self.hub.stats()));
1667 self.closed.store(true, Ordering::Release);
1668 self.set_state(match &result {
1669 Ok(_) => SessionState::Stopped,
1670 Err(e) => SessionState::Failed(Arc::new(e.clone())),
1671 });
1672 self.busy.store(false, Ordering::Release);
1673 result
1674 }
1675}
1676struct StartGuard(Option<Arc<SessionInner>>);
1677impl StartGuard {
1678 fn disarm(mut self) {
1679 self.0.take();
1680 }
1681}
1682impl Drop for StartGuard {
1683 fn drop(&mut self) {
1684 if let Some(i) = self.0.take() {
1685 i.request_stop();
1686 }
1687 }
1688}
1689pub struct CaptureSession {
1692 inner: Arc<SessionInner>,
1693 negotiated: NegotiatedCapture,
1694 actual: CameraConfig,
1695}
1696pub type Session = CaptureSession;
1698impl std::fmt::Debug for CaptureSession {
1699 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1700 f.debug_struct("CaptureSession")
1701 .field("negotiated", &self.negotiated)
1702 .field("state", &self.state())
1703 .finish()
1704 }
1705}
1706impl CaptureSession {
1707 pub fn device(&self) -> &DeviceInfo {
1708 &self.inner.device
1709 }
1710 pub fn is_streaming(&self) -> bool {
1711 !self.inner.stopping.load(Ordering::Acquire) && self.inner.hub.is_streaming()
1712 }
1713 pub fn negotiated(&self) -> &NegotiatedCapture {
1714 &self.negotiated
1715 }
1716 pub fn subscribe(&self, options: SubscriptionOptions) -> CameraResult<FrameReceiver> {
1717 Ok(self
1718 .inner
1719 .hub
1720 .subscribe_with(options)?
1721 .with_stop_flag(self.inner.stopping.clone()))
1722 }
1723 pub fn latest(&self) -> Option<Frame> {
1724 if self.inner.stopping.load(Ordering::Acquire) {
1725 None
1726 } else {
1727 self.inner.hub.latest()
1728 }
1729 }
1730 pub fn metrics(&self) -> FrameMetrics {
1731 self.inner
1732 .terminal_metrics
1733 .lock()
1734 .as_ref()
1735 .map(|(metrics, _)| metrics.clone())
1736 .unwrap_or_else(|| self.inner.hub.metrics())
1737 }
1738 pub fn stats(&self) -> StreamStats {
1739 self.inner
1740 .terminal_metrics
1741 .lock()
1742 .as_ref()
1743 .map(|(_, stats)| stats.clone())
1744 .unwrap_or_else(|| self.inner.hub.stats())
1745 }
1746 pub fn state(&self) -> SessionState {
1747 self.inner.state.borrow().clone()
1748 }
1749 pub fn watch_state(&self) -> StateWatcher {
1750 StateWatcher {
1751 receiver: self.inner.state.subscribe(),
1752 }
1753 }
1754 pub fn events(&self) -> EventReceiver {
1755 EventReceiver {
1756 receiver: self.inner.events.subscribe(),
1757 }
1758 }
1759 pub async fn close(&self) -> CameraResult<()> {
1760 self.inner.request_stop();
1761 self.inner.stop().await
1762 }
1763 fn spawn_recovery(&self, policy: ReconnectPolicy) {
1764 let weak = Arc::downgrade(&self.inner);
1765 let config = self.actual.clone();
1766 tokio::spawn(async move {
1767 let mut attempt = 0;
1768 let mut pending_trigger = None;
1769 loop {
1770 let Some(inner) = weak.upgrade() else {
1771 break;
1772 };
1773 let retrying = pending_trigger.is_some();
1774 let trigger = if let Some(trigger) = pending_trigger {
1775 tokio::time::sleep(recovery_retry_delay(policy.delay, attempt)).await;
1776 trigger
1777 } else {
1778 inner
1779 .hub
1780 .wait_for_recovery_trigger(policy.stall_timeout)
1781 .await
1782 };
1783 if inner.stopping.load(Ordering::Acquire) {
1784 break;
1785 }
1786 let _guard = inner.lifecycle.lock().await;
1787 if inner.stopping.load(Ordering::Acquire) {
1788 break;
1789 }
1790 if !retrying && inner.hub.recovery_trigger(policy.stall_timeout).is_none() {
1791 continue;
1792 }
1793 attempt += 1;
1794 inner.set_state(SessionState::Recovering { attempt });
1795 inner.hub.begin_recovery();
1796 let force_reopen =
1797 attempt > 1 || matches!(trigger, crate::frame::RecoveryTrigger::SourceStopped);
1798 let result = async {
1799 let stop_result = inner.source.stop().await;
1800 let reopen = force_reopen
1801 || matches!((&inner.source, &stop_result), (Source::Native(_), Err(_)));
1802 if let Source::Native(native) = &inner.source {
1803 if reopen {
1804 native.reopen().await?;
1805 apply_backend_options(
1806 &inner.source,
1807 &inner.device.id.backend,
1808 inner.backend_options.as_ref(),
1809 )
1810 .await?;
1811 inner.requested_controls.lock().unwrap().clear();
1812 } else {
1813 stop_result?;
1814 }
1815 } else {
1816 stop_result?;
1817 }
1818 if inner.stopping.load(Ordering::Acquire) {
1819 return Err(CameraError::stream_stopped());
1820 }
1821 inner.source.start(config.clone()).await?;
1822 let session = inner.hub.session();
1823 let first_frame_timeout = if reopen {
1824 policy.stall_timeout
1825 } else {
1826 fast_restart_first_frame_timeout(&config, policy.stall_timeout)?
1827 };
1828 let first = inner
1829 .hub
1830 .wait_first_in(session, first_frame_timeout)
1831 .await?;
1832 let actual = inner.source.actual(&config)?;
1833 validate_recovered_capture(&config, &actual, first.layout())?;
1834 validate_requested_capture(&inner.intent, &actual, first.layout())?;
1835 Ok::<_, CameraError>(())
1836 }
1837 .await;
1838 if inner.stopping.load(Ordering::Acquire) {
1839 let _ = inner.source.stop().await;
1840 break;
1841 }
1842 match result {
1843 Ok(()) => {
1844 inner.hub.end_recovery();
1845 inner.set_state(SessionState::Streaming);
1846 attempt = 0;
1847 pending_trigger = None;
1848 }
1849 Err(e) => {
1850 let _ = inner.source.stop().await;
1853 inner.hub.stop();
1854 pending_trigger = Some(trigger);
1855 if policy.max_attempts.is_some_and(|n| attempt >= n) {
1856 inner.set_state(SessionState::Failed(Arc::new(e)));
1857 inner.stopping.store(true, Ordering::Release);
1858 inner.hub.disable_recovery();
1859 inner.hub.end_recovery();
1860 inner.hub.stop();
1861 let _ = inner.source.stop().await;
1862 *inner.terminal_metrics.lock() =
1863 Some((inner.hub.metrics(), inner.hub.stats()));
1864 inner.closed.store(true, Ordering::Release);
1865 inner.busy.store(false, Ordering::Release);
1866 break;
1867 }
1868 }
1869 }
1870 }
1871 });
1872 }
1873 pub(crate) async fn with_native<T: Send + 'static>(
1874 &self,
1875 operation: impl FnOnce(BackendCamera) -> CameraResult<T> + Send + 'static,
1876 ) -> CameraResult<T> {
1877 let inner = self.inner.clone();
1878 tokio::spawn(async move {
1879 let _guard = inner.lifecycle.lock().await;
1880 if inner.stopping.load(Ordering::Acquire) {
1881 return Err(CameraError::stream_stopped());
1882 }
1883 let Source::Native(camera) = &inner.source else {
1884 return Err(CameraError::control_not_supported(
1885 "Synthetic source has no device controls".into(),
1886 ));
1887 };
1888 let camera = camera.current()?;
1889 tokio::task::spawn_blocking(move || operation(camera))
1890 .await
1891 .map_err(join_error)?
1892 })
1893 .await
1894 .map_err(join_error)?
1895 }
1896 pub(crate) fn requested_control(&self, id: crate::ControlId) -> Option<crate::ControlValue> {
1897 self.inner
1898 .requested_controls
1899 .lock()
1900 .unwrap()
1901 .get(&id)
1902 .cloned()
1903 }
1904 pub(crate) fn record_control(&self, id: crate::ControlId, value: crate::ControlValue) {
1905 self.inner
1906 .requested_controls
1907 .lock()
1908 .unwrap()
1909 .insert(id, value);
1910 }
1911}
1912
1913fn recovery_retry_delay(configured: Duration, failed_attempts: u32) -> Duration {
1914 let fast = match failed_attempts {
1915 0 | 1 => Duration::from_millis(100),
1916 2 => Duration::from_millis(200),
1917 3 => Duration::from_millis(400),
1918 _ => configured,
1919 };
1920 configured.min(fast)
1921}
1922
1923fn fast_restart_first_frame_timeout(
1924 config: &CameraConfig,
1925 recovery_timeout: Duration,
1926) -> CameraResult<Duration> {
1927 let three_frames = config.frame_rate()?.interval().saturating_mul(3);
1928 Ok(recovery_timeout.min(Duration::from_millis(750).max(three_frames)))
1929}
1930
1931fn validate_recovered_capture(
1932 expected: &CameraConfig,
1933 actual: &CameraConfig,
1934 first: &crate::FrameLayout,
1935) -> CameraResult<()> {
1936 if (actual.width, actual.height, actual.format)
1937 != (expected.width, expected.height, expected.format)
1938 || actual.frame_rate()? != expected.frame_rate()?
1939 {
1940 return Err(CameraError::unsupported_format(
1941 "Recovered capture differs from the negotiated configuration".into(),
1942 ));
1943 }
1944 if (first.width, first.height) != (actual.width, actual.height) {
1945 return Err(CameraError::invalid_frame(
1946 "Recovered first frame differs from negotiated dimensions".into(),
1947 ));
1948 }
1949 Ok(())
1950}
1951
1952impl Drop for CaptureSession {
1953 fn drop(&mut self) {
1954 if !self.inner.stopping.load(Ordering::Acquire) {
1955 self.inner.request_stop();
1956 }
1957 }
1958}
1959
1960#[cfg(test)]
1961mod recovery_tests {
1962 use super::*;
1963
1964 #[test]
1965 fn recovery_requires_the_original_mode_and_a_matching_frame() {
1966 let config = CameraConfig::new(VideoFormat::RGB, 8, 8, 30);
1967 let hub = FrameHub::default();
1968 let epoch = hub.start();
1969 hub.publish_rgb(epoch, 8, 8, None, |_| Ok(())).unwrap();
1970 let first = hub.latest().unwrap();
1971 assert!(validate_recovered_capture(&config, &config, first.layout()).is_ok());
1972 let equivalent = CameraConfig {
1973 fps: 60,
1974 fps_denominator: 2,
1975 ..config.clone()
1976 };
1977 assert!(validate_recovered_capture(&config, &equivalent, first.layout()).is_ok());
1978 for changed in [
1979 CameraConfig {
1980 fps: 15,
1981 ..config.clone()
1982 },
1983 CameraConfig {
1984 width: 16,
1985 ..config.clone()
1986 },
1987 CameraConfig {
1988 format: VideoFormat::NV12,
1989 ..config.clone()
1990 },
1991 ] {
1992 assert!(matches!(
1993 validate_recovered_capture(&config, &changed, first.layout()),
1994 Err(ref error) if error.kind() == crate::CameraErrorKind::UnsupportedFormat
1995 ));
1996 }
1997 let mut layout = first.layout().clone();
1998 layout.height = 16;
1999 assert!(matches!(
2000 validate_recovered_capture(&config, &config, &layout),
2001 Err(ref error) if error.kind() == crate::CameraErrorKind::InvalidFrame
2002 ));
2003 }
2004
2005 #[test]
2006 fn descriptor_candidates_preserve_all_format_preferences() {
2007 let request = CaptureRequest::builder()
2008 .exact_resolution(800, 600)
2009 .preferred_frame_rate(FrameRate::new(30000, 1001).unwrap())
2010 .preferred_formats([CaptureFormat::Mjpeg, CaptureFormat::Yuyv])
2011 .build()
2012 .unwrap();
2013 let candidates = direct_capture_candidates(&request);
2014 assert_eq!(candidates.len(), 2);
2015 assert_eq!(candidates[0].format, VideoFormat::MJPEG);
2016 assert_eq!(candidates[1].format, VideoFormat::YUYV);
2017 assert!(candidates.iter().all(|candidate| {
2018 candidate.width == 800
2019 && candidate.height == 600
2020 && candidate.frame_rate().unwrap() == FrameRate::new(30000, 1001).unwrap()
2021 }));
2022 }
2023
2024 #[test]
2025 fn reconnect_retries_are_fast_first_and_then_respect_the_policy_delay() {
2026 let configured = Duration::from_secs(1);
2027 assert_eq!(
2028 recovery_retry_delay(configured, 1),
2029 Duration::from_millis(100)
2030 );
2031 assert_eq!(
2032 recovery_retry_delay(configured, 2),
2033 Duration::from_millis(200)
2034 );
2035 assert_eq!(
2036 recovery_retry_delay(configured, 3),
2037 Duration::from_millis(400)
2038 );
2039 assert_eq!(recovery_retry_delay(configured, 4), configured);
2040 assert_eq!(
2041 recovery_retry_delay(Duration::from_millis(3), 3),
2042 Duration::from_millis(3)
2043 );
2044 }
2045
2046 #[test]
2047 fn soft_restart_first_frame_deadline_tracks_slow_frame_rates() {
2048 assert_eq!(
2049 fast_restart_first_frame_timeout(
2050 &CameraConfig::new(VideoFormat::RGB, 8, 8, 30),
2051 Duration::from_secs(5),
2052 )
2053 .unwrap(),
2054 Duration::from_millis(750)
2055 );
2056 assert_eq!(
2057 fast_restart_first_frame_timeout(
2058 &CameraConfig::new(VideoFormat::RGB, 8, 8, 2),
2059 Duration::from_secs(5),
2060 )
2061 .unwrap(),
2062 Duration::from_millis(1500)
2063 );
2064 }
2065}