Skip to main content

ferroui_media_camera_controls/
lib.rs

1//! Camera-only presentation through the normal composition texture presenter.
2#![forbid(unsafe_code)]
3pub mod automation;
4mod camera_capture_ui;
5pub use camera_capture_ui::{CameraCaptureResult, CameraCaptureUI};
6#[cfg(feature = "qr")]
7mod qr_preview;
8#[cfg(feature = "qr")]
9pub use qr_preview::QrPreview;
10
11use ferroui_base::{
12    ferro_class, ferro_class_info, ferro_impl_classes, ferro_properties,
13    input::InputElementImpl,
14    instantiate,
15    interactivity::InteractiveImpl,
16    layout::LayoutableImpl,
17    media::Stretch,
18    reactive::{IDisposable, Subscription},
19    rendering::composition::{ElementComposition, ICompositionGpuInterop, RenderContextState},
20    threading::{Dispatcher, DispatcherPriority, DispatcherTimer},
21    FerroObjectImpl, FerroObjectImplExt, FerroProperty, FerroPropertyChangedEventArgs, Ref,
22    StyledElementImpl, StyledProperty, VisualImpl, VisualImplExt, VisualTreeAttachmentEventArgs,
23};
24use ferroui_controls::{ControlImpl, Panel, PanelImpl, TextBlock};
25use ferroui_media::{MediaEvent, MediaSubscription};
26use ferroui_media_camera::*;
27pub use ferroui_media_controls::{TextureFrameImporter, TexturePresenter};
28use std::{
29    cell::{Cell, RefCell},
30    rc::Rc,
31    time::Duration,
32};
33/// Recreates a camera provider/importer for the restored compositor. Capture
34/// the original shared frame budget when the native producer needs it.
35pub type CameraProviderFactory =
36    dyn Fn(
37        &dyn ICompositionGpuInterop,
38    ) -> Result<(Rc<dyn CameraProvider>, Rc<dyn TextureFrameImporter>), CameraError>;
39
40#[repr(C)]
41pub struct CameraPreview {
42    base: Panel,
43    presenter: RefCell<Option<Ref<TexturePresenter>>>,
44    placeholder: RefCell<Option<Ref<TextBlock>>>,
45    owned: RefCell<Option<CameraSession>>,
46    shared: RefCell<Option<WeakCameraSession>>,
47    capture: RefCell<Option<CameraCapture>>,
48    timer: RefCell<Option<Subscription>>,
49    generation: Cell<u64>,
50    errors: MediaEvent<CameraError>,
51    error_subscription: RefCell<Option<MediaSubscription>>,
52    frame_subscription: RefCell<Option<MediaSubscription>>,
53    provider_factory: RefCell<Option<Rc<CameraProviderFactory>>>,
54    context_subscription: RefCell<Option<Rc<dyn IDisposable>>>,
55    requested_options: RefCell<Option<CameraStartOptions>>,
56    device_suspended: Cell<bool>,
57}
58ferro_class!(CameraPreview:Panel);
59ferro_class_info!(CameraPreview {
60    new: CameraPreview::new
61});
62ferro_impl_classes!(CameraPreview:StyledElementImpl,LayoutableImpl,InteractiveImpl,InputElementImpl,ControlImpl,PanelImpl);
63ferro_properties! {impl CameraPreview {
64    pub fn stretch_property()->StyledProperty<Stretch> {FerroProperty::register::<Self,_>("Stretch",Stretch::Uniform)}
65}}
66impl FerroObjectImpl for CameraPreview {
67    fn constructed(this: &Self) {
68        Self::parent_constructed(this);
69        let presenter = TexturePresenter::new();
70        let placeholder = TextBlock::new();
71        placeholder.set_text(Some("Camera stopped"));
72        let weak = this.to_ref().downgrade();
73        this.error_subscription
74            .replace(Some(presenter.errors().subscribe(move |error| {
75                if let Some(view) = weak.upgrade() {
76                    let kind = if error.kind == ferroui_media::MediaErrorKind::DeviceLost {
77                        view.suspend_device();
78                        CameraErrorKind::DeviceLost
79                    } else {
80                        view.stop();
81                        CameraErrorKind::NativeFailure
82                    };
83                    view.errors.emit(&CameraError::new(kind, error.to_string()));
84                }
85            })));
86        let weak = this.to_ref().downgrade();
87        this.frame_subscription
88            .replace(Some(presenter.frame_presented().subscribe(move |_| {
89                if let Some(view) = weak.upgrade() {
90                    if let Some(text) = view.placeholder.borrow().as_ref() {
91                        text.set_is_visible(false);
92                    }
93                }
94            })));
95        this.children().add(&presenter);
96        this.children().add(&placeholder);
97        this.presenter.replace(Some(presenter));
98        this.placeholder.replace(Some(placeholder));
99    }
100    fn on_property_changed(this: &Self, args: &FerroPropertyChangedEventArgs<'_>) {
101        Self::parent_on_property_changed(this, args);
102        if args.property() == Self::stretch_property().as_property() {
103            if let Some(presenter) = this.presenter.borrow().as_ref() {
104                presenter.set_stretch(this.stretch());
105            }
106        }
107    }
108}
109impl VisualImpl for CameraPreview {
110    fn on_attached_to_visual_tree(this: &Self, args: &VisualTreeAttachmentEventArgs) {
111        Self::parent_on_attached_to_visual_tree(this, args);
112        this.subscribe_context();
113        this.start_polling();
114    }
115    fn on_detached_from_visual_tree(this: &Self, args: &VisualTreeAttachmentEventArgs) {
116        if this.owned.borrow().is_some() || this.capture.borrow().is_some() {
117            // An owned session (or its pending start) ends with the view.
118            this.stop();
119        } else if let Some(timer) = this.timer.borrow_mut().take() {
120            // A caller-owned session stays borrowed: the view only stops polling it (the
121            // presenter releases its own leases when it detaches) and shows it again when
122            // it is attached again.
123            timer.dispose();
124        }
125        if let Some(subscription) = this.context_subscription.borrow_mut().take() {
126            subscription.dispose();
127        }
128        Self::parent_on_detached_from_visual_tree(this, args);
129    }
130}
131impl CameraPreview {
132    pub fn construct() -> Self {
133        Self {
134            base: Panel::construct(),
135            presenter: RefCell::new(None),
136            placeholder: RefCell::new(None),
137            owned: RefCell::new(None),
138            shared: RefCell::new(None),
139            capture: RefCell::new(None),
140            timer: RefCell::new(None),
141            generation: Cell::new(0),
142            errors: MediaEvent::default(),
143            error_subscription: RefCell::new(None),
144            frame_subscription: RefCell::new(None),
145            provider_factory: RefCell::new(None),
146            context_subscription: RefCell::new(None),
147            requested_options: RefCell::new(None),
148            device_suspended: Cell::new(false),
149        }
150    }
151    pub fn new() -> Ref<Self> {
152        instantiate(Self::construct())
153    }
154    pub fn stretch(&self) -> Stretch {
155        self.get_value(Self::stretch_property())
156    }
157    pub fn set_stretch(&self, value: Stretch) {
158        self.set_value(Self::stretch_property(), value);
159    }
160    pub fn errors(&self) -> MediaEvent<CameraError> {
161        self.errors.clone()
162    }
163    pub fn session(&self) -> Option<CameraSession> {
164        self.owned.borrow().clone().or_else(|| {
165            self.shared
166                .borrow()
167                .as_ref()
168                .and_then(WeakCameraSession::upgrade)
169        })
170    }
171    /// Borrows a caller-owned session. Detach releases this view's leases only.
172    pub fn set_session(
173        &self,
174        session: Option<&CameraSession>,
175        importer: Option<Rc<dyn TextureFrameImporter>>,
176    ) {
177        self.stop();
178        self.shared.replace(session.map(CameraSession::downgrade));
179        self.attach_source(importer);
180        self.start_polling();
181    }
182    /// Explicitly creates an owned session. Detach cancels preparation and
183    /// stops this session; construction/attachment never starts a camera.
184    pub fn start(
185        &self,
186        capture: CameraCapture,
187        options: CameraStartOptions,
188        importer: Rc<dyn TextureFrameImporter>,
189    ) -> CameraFuture<()> {
190        self.stop();
191        let generation = self.generation.get();
192        self.requested_options.replace(Some(options.clone()));
193        let pending = capture.start(options);
194        self.capture.replace(Some(capture));
195        let weak = self.to_ref().downgrade();
196        Box::pin(async move {
197            let session = pending.await?;
198            let Some(view) = weak.upgrade().filter(|v| v.generation.get() == generation) else {
199                session.stop();
200                return Err(CameraError::new(
201                    CameraErrorKind::Cancelled,
202                    "Camera preview detached",
203                ));
204            };
205            view.owned.replace(Some(session));
206            view.attach_source(Some(importer));
207            view.start_polling();
208            Ok(())
209        })
210    }
211    fn attach_source(&self, importer: Option<Rc<dyn TextureFrameImporter>>) {
212        if let Some(presenter) = self.presenter.borrow().as_ref() {
213            presenter.set_frame_source(self.session().map(|v| v.frames()), importer);
214        }
215    }
216    pub fn stop(&self) {
217        self.generation.set(
218            self.generation
219                .get()
220                .checked_add(1)
221                .expect("camera preview generation exhausted"),
222        );
223        self.device_suspended.set(false);
224        self.requested_options.borrow_mut().take();
225        if let Some(timer) = self.timer.borrow_mut().take() {
226            timer.dispose();
227        }
228        if let Some(capture) = self.capture.borrow_mut().take() {
229            capture.cancel_pending_start();
230        }
231        if let Some(session) = self.owned.borrow_mut().take() {
232            session.stop();
233        }
234        self.shared.borrow_mut().take();
235        if let Some(presenter) = self.presenter.borrow().as_ref() {
236            presenter.set_frame_source(None, None);
237        }
238        if let Some(text) = self.placeholder.borrow().as_ref() {
239            text.set_is_visible(true);
240        }
241    }
242    /// Enables automatic restart only for this control's explicitly started,
243    /// active session. Caller-supplied sessions are recovered by their owner.
244    pub fn set_provider_factory(&self, factory: Option<Rc<CameraProviderFactory>>) {
245        if let Some(subscription) = self.context_subscription.borrow_mut().take() {
246            subscription.dispose();
247        }
248        *self.provider_factory.borrow_mut() = factory;
249        self.subscribe_context();
250    }
251    fn subscribe_context(&self) {
252        if self.context_subscription.borrow().is_some()
253            || self.provider_factory.borrow().is_none()
254            || !self.is_attached_to_visual_tree()
255        {
256            return;
257        }
258        let Some(visual) = ElementComposition::get_element_visual(&self.to_ref()) else {
259            return;
260        };
261        let weak = self.to_ref().downgrade();
262        *self.context_subscription.borrow_mut() =
263            Some(visual.compositor().render_context_changed(move |state| {
264                if let Some(view) = weak.upgrade() {
265                    match state {
266                        RenderContextState::Lost => view.suspend_device(),
267                        RenderContextState::Created if view.device_suspended.get() => {
268                            view.recover_device()
269                        }
270                        _ => {}
271                    }
272                }
273            }));
274    }
275    fn suspend_device(&self) {
276        self.suspend_device_inner(false);
277    }
278    fn suspend_device_inner(&self, stopped_by_device_loss: bool) {
279        if self.device_suspended.get() {
280            return;
281        }
282        let capture = self.capture.borrow().clone();
283        let options = self.requested_options.borrow().clone();
284        let was_active = self
285            .owned
286            .borrow()
287            .as_ref()
288            .is_some_and(|s| stopped_by_device_loss || !s.is_stopped());
289        // Pending starts are cancelled without silently retrying consent.
290        self.stop();
291        if was_active && self.provider_factory.borrow().is_some() {
292            self.capture.replace(capture);
293            self.requested_options.replace(options);
294            self.device_suspended.set(true);
295        }
296    }
297    fn recover_device(&self) {
298        let Some(factory) = self.provider_factory.borrow().clone() else {
299            return;
300        };
301        let Some(capture) = self.capture.borrow().clone() else {
302            return;
303        };
304        let Some(options) = self.requested_options.borrow().clone() else {
305            return;
306        };
307        let Some(visual) = ElementComposition::get_element_visual(&self.to_ref()) else {
308            return;
309        };
310        let compositor = visual.compositor().clone();
311        let generation = self.generation.get();
312        let weak = self.to_ref().downgrade();
313        self.device_suspended.set(false);
314        Dispatcher::ui_thread().spawn_local(async move {
315            let interop = compositor.try_get_composition_gpu_interop().await;
316            let Some(view) = weak
317                .upgrade()
318                .filter(|v| v.generation.get() == generation && v.is_attached_to_visual_tree())
319            else {
320                return;
321            };
322            let result = match interop {
323                Some(interop) if !interop.is_lost() => factory(&*interop),
324                _ => Err(CameraError::new(
325                    CameraErrorKind::DeviceLost,
326                    "No restored camera GPU context",
327                )),
328            };
329            let (provider, importer) = match result {
330                Ok(value) => value,
331                Err(error) => {
332                    view.device_suspended.set(true);
333                    view.errors.emit(&error);
334                    return;
335                }
336            };
337            if view.generation.get() != generation {
338                return;
339            }
340            capture.rebind_provider(provider);
341            let pending = view.start(capture, options, importer);
342            let active_generation = view.generation.get();
343            drop(view);
344            if let Err(error) = pending.await {
345                if error.kind != CameraErrorKind::Cancelled {
346                    if let Some(view) = weak
347                        .upgrade()
348                        .filter(|v| v.generation.get() == active_generation)
349                    {
350                        view.errors.emit(&error);
351                    }
352                }
353            }
354        });
355    }
356    fn start_polling(&self) {
357        if !self.is_attached_to_visual_tree()
358            || self.timer.borrow().is_some()
359            || self.session().is_none()
360        {
361            return;
362        }
363        let weak = self.to_ref().downgrade();
364        let timer = DispatcherTimer::run(
365            move || {
366                let Some(view) = weak.upgrade() else {
367                    return false;
368                };
369                let Some(session) = view.session().filter(|v| !v.is_stopped()) else {
370                    view.stop();
371                    return false;
372                };
373                if let Err(error) = session.poll() {
374                    if error.kind == CameraErrorKind::DeviceLost {
375                        // CameraSession.poll has already retired the failed native session.
376                        view.suspend_device_inner(true);
377                    } else {
378                        view.stop();
379                    }
380                    view.errors.emit(&error);
381                    return false;
382                }
383                let presenter = view.presenter.borrow().clone();
384                if let Some(presenter) = presenter {
385                    presenter.present_latest();
386                }
387                true
388            },
389            Duration::from_millis(16),
390            DispatcherPriority::NORMAL,
391        );
392        self.timer.replace(Some(timer));
393    }
394}
395impl Drop for CameraPreview {
396    fn drop(&mut self) {
397        self.stop();
398        if let Some(subscription) = self.context_subscription.get_mut().take() {
399            subscription.dispose();
400        }
401    }
402}
403// Shared by runtime registration and the build-time XAML metadata scanner.
404const NAMESPACES: &[(&str, &str)] = &[(
405    "ferroui_media_camera_controls",
406    "FerroUI.Media.Camera.Controls",
407)];
408pub static ASSEMBLY: ferroui_base::metadata::MarkupAssembly =
409    ferroui_base::metadata::MarkupAssembly {
410        name: "FerroUI.Media.Camera.Controls",
411        crate_name: "ferroui_media_camera_controls",
412        xmlns_definitions: &[ferroui_base::metadata::XmlnsDefinition {
413            xml_namespace: ferroui_base::metadata::FERRO_XML_NAMESPACE,
414            namespace: "FerroUI.Media.Camera.Controls",
415        }],
416        xmlns_prefixes: &[],
417        metadata: &[],
418    };
419ferroui_base::ferro_rust_paths! {
420    classes: [crate::CameraCaptureUI, crate::CameraPreview],
421    types: [], contracts: [], generics: [], generic_contracts: [],
422}
423const TYPES: &[&ferroui_base::TypeInfo] =
424    &[CameraPreview::TYPE, CameraCaptureUI::TYPE, automation::CameraCaptureUIAutomationPeer::TYPE];
425#[cfg(feature = "qr")]
426const QR_TYPES: &[&ferroui_base::TypeInfo] = &[QrPreview::TYPE];
427/// Registers the namespaces, the assembly and the table of types of this crate (and of the
428/// crates it is built on). Cheap and idempotent. The themes of the WinUI family call it, so the
429/// table is trimmed as that of every crate of the framework (`TypeInfo::register_table`): an
430/// application that shows no camera control links none.
431pub fn register_types() {
432    static ONCE: std::sync::Once = std::sync::Once::new();
433    ONCE.call_once(|| {
434        ferroui_media_controls::register_types();
435        ferroui_base::TypeInfo::register_namespaces(NAMESPACES);
436        ferroui_base::metadata::MarkupAssembly::register_table(&ASSEMBLY);
437        register_rust_paths();
438        ferroui_base::TypeInfo::register_table(TYPES);
439        #[cfg(feature = "qr")]
440        ferroui_base::TypeInfo::register_table(QR_TYPES);
441    });
442}
443
444#[cfg(test)]
445mod recovery_tests {
446    use super::*;
447    use ferroui_controls::testing::{TestServices, UnitTestApplication};
448    use ferroui_media::frames::{FrameBudget, FrameLease};
449    use std::{
450        pin::Pin,
451        task::{Context, Poll, Waker},
452    };
453    struct Native(Rc<Cell<u32>>);
454    impl NativeCameraSession for Native {
455        fn settings(&self) -> CameraSettings {
456            CameraSettings {
457                device: CameraDevice {
458                    id: "fixture".into(),
459                    label: None,
460                    facing: CameraFacing::Unknown,
461                    formats: vec![],
462                },
463                format: CameraFormat {
464                    width: 16,
465                    height: 16,
466                    frames_per_second: 30.,
467                },
468                native_still: false,
469                torch: false,
470                zoom: None,
471            }
472        }
473        fn take_frame(&self) -> Result<Option<FrameLease>, CameraError> {
474            Ok(None)
475        }
476        fn photo(&self) -> CameraFuture<CameraPhoto> {
477            Box::pin(async { Err(CameraError::new(CameraErrorKind::Unsupported, "No photo")) })
478        }
479        fn set_control(&self, _: CameraControl) -> Result<(), CameraError> {
480            Ok(())
481        }
482        fn stop(&self) {
483            self.0.set(self.0.get() + 1);
484        }
485    }
486    struct Provider(Rc<Cell<u32>>);
487    impl CameraProvider for Provider {
488        fn devices(&self) -> CameraFuture<Vec<CameraDevice>> {
489            Box::pin(async { Ok(vec![]) })
490        }
491        fn permission(&self) -> CameraPermission {
492            CameraPermission::Granted
493        }
494        fn start(
495            &self,
496            _: CameraStartOptions,
497            _: u64,
498            _: u64,
499            _: FrameBudget,
500        ) -> CameraFuture<Rc<dyn NativeCameraSession>> {
501            let native: Rc<dyn NativeCameraSession> = Rc::new(Native(self.0.clone()));
502            Box::pin(async { Ok(native) })
503        }
504    }
505    struct Importer;
506    impl TextureFrameImporter for Importer {
507        fn import(
508            &self,
509            _: &dyn ICompositionGpuInterop,
510            _: &FrameLease,
511        ) -> Result<ferroui_media_controls::ImportedTextureFrame, ferroui_media::MediaError>
512        {
513            unreachable!("no frame fixture")
514        }
515    }
516    fn ready<T>(mut future: Pin<Box<dyn std::future::Future<Output = T>>>) -> T {
517        match future
518            .as_mut()
519            .poll(&mut Context::from_waker(Waker::noop()))
520        {
521            Poll::Ready(value) => value,
522            Poll::Pending => panic!("expected ready"),
523        }
524    }
525    #[test]
526    fn owned_camera_suspension_preserves_restart_request_until_explicit_stop() {
527        let _app = UnitTestApplication::start(TestServices::styled_window());
528        let stops = Rc::new(Cell::new(0));
529        let capture = CameraCapture::new(
530            Rc::new(Provider(stops.clone())),
531            FrameBudget::new(Default::default()),
532        );
533        let view = CameraPreview::new();
534        view.set_provider_factory(Some(Rc::new(|_| {
535            Err(CameraError::new(
536                CameraErrorKind::Unsupported,
537                "No GPU fixture",
538            ))
539        })));
540        ready(view.start(capture, CameraStartOptions::default(), Rc::new(Importer))).unwrap();
541        let old = view.session().unwrap();
542        view.suspend_device();
543        assert!(old.is_stopped());
544        assert_eq!(stops.get(), 1);
545        assert!(view.device_suspended.get());
546        assert!(view.requested_options.borrow().is_some());
547        assert!(view.session().is_none());
548        view.stop();
549        assert!(!view.device_suspended.get());
550        assert!(view.requested_options.borrow().is_none());
551        assert!(view.capture.borrow().is_none());
552    }
553    #[test]
554    fn borrowed_camera_is_released_without_stopping_callers_session() {
555        let _app = UnitTestApplication::start(TestServices::styled_window());
556        let stops = Rc::new(Cell::new(0));
557        let capture = CameraCapture::new(
558            Rc::new(Provider(stops.clone())),
559            FrameBudget::new(Default::default()),
560        );
561        let session = ready(capture.start(Default::default())).unwrap();
562        let view = CameraPreview::new();
563        view.set_session(Some(&session), Some(Rc::new(Importer)));
564        view.suspend_device();
565        assert!(!session.is_stopped());
566        assert_eq!(stops.get(), 0);
567        assert!(view.session().is_none());
568        session.stop();
569        assert_eq!(stops.get(), 1);
570    }
571    #[test]
572    fn a_borrowed_session_is_shown_again_after_the_view_is_reattached() {
573        let _app = UnitTestApplication::start(TestServices::styled_window());
574        let stops = Rc::new(Cell::new(0));
575        let capture = CameraCapture::new(
576            Rc::new(Provider(stops.clone())),
577            FrameBudget::new(Default::default()),
578        );
579        let session = ready(capture.start(Default::default())).unwrap();
580        let view = CameraPreview::new();
581        view.set_session(Some(&session), Some(Rc::new(Importer)));
582        let window = ferroui_controls::Window::new();
583        window.set_content(Some(ferroui_controls::Control::boxed(view.clone())));
584        window.show();
585        window.set_content(None);
586        // Detached: the caller's session is neither stopped nor forgotten.
587        assert!(!session.is_stopped());
588        assert!(view.session().is_some());
589        assert!(view.timer.borrow().is_none());
590        window.set_content(Some(ferroui_controls::Control::boxed(view.clone())));
591        assert!(view.session().is_some());
592        assert!(
593            view.timer.borrow().is_some(),
594            "polling resumes when attached again"
595        );
596        assert_eq!(stops.get(), 0);
597        session.stop();
598    }
599}
600
601/// Optional media controls whose control themes belong to the WinUI family.
602pub const WINUI_ONLY_CONTROLS: &[(&ferroui_base::TypeInfo, &str)] = &[(
603    CameraCaptureUI::TYPE,
604    "Portable media UI styled by the WinUI theme and its derived themes",
605)];