azul_layout/widgets/capture_common.rs
1//! Shared core for the "video-ish" widgets (camera / screencap / video).
2//!
3//! All three are identical in architecture (RefAny dataset + AfterMount
4//! background capture/decode thread + writeback that uploads each frame into a
5//! stable external GL texture + recomposites). Only the *config* and the
6//! *worker* differ. This module holds the duplicated pieces - the [`VideoFrame`]
7//! the worker produces and [`present_frame`], the GL writeback core - so each
8//! widget is a thin config+worker wrapper and there's a single place for GL
9//! fixes + the real platform workers (AVFoundation / ScreenCaptureKit /
10//! vk-video) to plug in.
11//!
12//! NOTE: GL code - compile-verified here; the actual texture rendering must be
13//! verified on a machine with a window + GPU.
14
15use azul_core::animation::UpdateImageType;
16use azul_core::callbacks::Update;
17use azul_core::gl::gl::{RGBA, TEXTURE_2D, UNSIGNED_BYTE};
18use azul_core::gl::{GlContextPtr, OptionU8VecRef, Texture, U8VecRef};
19use azul_core::geom::PhysicalSizeU32;
20use azul_core::refany::RefAny;
21use azul_core::resources::ImageRef;
22use azul_core::video::VideoFrame;
23use azul_css::impl_option_inner; // brought into scope for impl_widget_callback!'s impl_option!
24use azul_css::props::basic::ColorU;
25
26use crate::callbacks::CallbackInfo;
27
28/// User hook fired once per captured/decoded frame - the backreference
29/// dependency-injection pattern (see `architecture.md`). A capture widget's
30/// private writeback invokes it with each [`VideoFrame`], so application code
31/// can apply effects, save the frame into its own data model, or send it over
32/// the network (azul-meet). Returns `Update` like any callback. Wired via
33/// `CameraWidget::with_on_frame` / `ScreenCaptureWidget::with_on_frame` /
34/// `VideoWidget::with_on_frame`.
35pub type OnVideoFrameCallbackType = extern "C" fn(RefAny, CallbackInfo, VideoFrame) -> Update;
36impl_widget_callback!(
37 OnVideoFrame,
38 OptionOnVideoFrame,
39 OnVideoFrameCallback,
40 OnVideoFrameCallbackType
41);
42
43// Host-invoker plumbing for managed-FFI bindings - see core/src/host_invoker.rs.
44azul_core::impl_managed_callback! {
45 wrapper: OnVideoFrameCallback,
46 info_ty: CallbackInfo,
47 return_ty: Update,
48 default_ret: Update::DoNothing,
49 invoker_static: ON_VIDEO_FRAME_INVOKER,
50 invoker_ty: AzOnVideoFrameCallbackInvoker,
51 thunk_fn: az_on_video_frame_callback_thunk,
52 setter_fn: AzApp_setOnVideoFrameCallbackInvoker,
53 from_handle_fn: AzOnVideoFrameCallback_createFromHostHandle,
54 extra_args: [ frame: VideoFrame ],
55}
56
57/// Invoke a capture widget's optional `on_frame` hook with `frame`, returning
58/// the user's `Update` (`DoNothing` when no hook is set). Shared by all three
59/// capture widgets' writebacks.
60pub fn invoke_on_frame(
61 hook: &OptionOnVideoFrame,
62 info: &mut CallbackInfo,
63 frame: &VideoFrame,
64) -> Update {
65 match hook {
66 OptionOnVideoFrame::Some(h) => {
67 (h.callback.cb)(h.refany.clone(), info.clone(), frame.clone())
68 }
69 OptionOnVideoFrame::None => Update::DoNothing,
70 }
71}
72
73/// Present `frame` for a video-ish widget and return the (stable) GL texture
74/// id to store back in the widget's state.
75///
76/// - First frame (`current_id` is `None`): allocate a GL texture, upload, wrap
77/// in an external-texture `ImageRef`, and install it on the widget's node
78/// **once** via `change_node_image` (the node is found via
79/// `get_node_id_of_root_dataset(dataset)`). Returns `Some(new_id)`.
80/// - Every frame after: re-upload into the same texture id + recomposite
81/// (`update_all_image_callbacks` -> `ShouldReRenderCurrentWindow`) - no
82/// relayout, no display-list rebuild, since the external texture's wr key
83/// (= the `ImageRef` data pointer) stays stable. Returns `current_id`.
84/// - No GL context (cpurender): returns `current_id` unchanged (a CPU upload
85/// path is a follow-up).
86pub fn present_frame(
87 info: &mut CallbackInfo,
88 dataset: RefAny,
89 current_id: Option<u32>,
90 frame: &VideoFrame,
91) -> Option<u32> {
92 let gl = match info.get_gl_context().into_option() {
93 Some(g) => g,
94 None => return current_id,
95 };
96
97 match current_id {
98 Some(id) => {
99 upload_rgba(&gl, id, frame);
100 info.update_all_image_callbacks();
101 Some(id)
102 }
103 None => {
104 let tex = Texture::allocate_rgba8(
105 gl.clone(),
106 PhysicalSizeU32 {
107 width: frame.width,
108 height: frame.height,
109 },
110 ColorU {
111 r: 0,
112 g: 0,
113 b: 0,
114 a: 0,
115 },
116 );
117 let id = tex.texture_id;
118 upload_rgba(&gl, id, frame);
119 let image = ImageRef::new_gltexture(tex);
120 if let Some(node) = info.get_node_id_of_root_dataset(dataset) {
121 if let Some(nid) = node.node.into_crate_internal() {
122 info.change_node_image(node.dom, nid, image, UpdateImageType::Content);
123 }
124 }
125 Some(id)
126 }
127 }
128}
129
130/// Upload tightly-packed RGBA8 pixels into the GL texture `texture_id`.
131pub fn upload_rgba(gl: &GlContextPtr, texture_id: u32, frame: &VideoFrame) {
132 gl.bind_texture(TEXTURE_2D, texture_id);
133 gl.tex_image_2d(
134 TEXTURE_2D,
135 0,
136 RGBA as i32,
137 frame.width as i32,
138 frame.height as i32,
139 0,
140 RGBA,
141 UNSIGNED_BYTE,
142 OptionU8VecRef::Some(U8VecRef::from(frame.bytes.as_ref())),
143 );
144}
145
146/// A platform frame-capture backend (camera / screen), registered by the dll at
147/// startup so the cross-platform capture widgets can pull **real** frames
148/// instead of their built-in test pattern. The dll provides one per OS (v4l2 on
149/// Linux, AVFoundation on macOS, Media Foundation on Windows, ScreenCaptureKit /
150/// PipeWire / DXGI for screens, ...). These are plain Rust fn pointers - the dll
151/// links azul-layout statically, so registering + calling is a Rust-to-Rust
152/// call, no `extern "C"`/trait-object dance.
153#[derive(Clone, Copy)]
154pub struct CaptureVTable {
155 /// Open source `index` (camera device / display index) at the requested
156 /// `width` x `height`. Returns an opaque handle, or `0` on failure (the
157 /// worker then falls back to the test pattern).
158 pub open: fn(index: u32, width: u32, height: u32) -> u64,
159 /// Block for the next frame, writing tightly-packed RGBA8 into `out`
160 /// (resized as needed). Returns the actual frame `(width, height)`, or
161 /// `(0, 0)` on end-of-stream / error (the worker then stops + closes).
162 pub read: fn(handle: u64, out: &mut alloc::vec::Vec<u8>) -> (u32, u32),
163 /// Close + free the source.
164 pub close: fn(handle: u64),
165}
166
167static CAMERA_BACKEND: std::sync::OnceLock<CaptureVTable> = std::sync::OnceLock::new();
168static SCREEN_BACKEND: std::sync::OnceLock<CaptureVTable> = std::sync::OnceLock::new();
169
170/// Register the platform **camera** capture backend (called once by the dll at
171/// startup; the first registration wins). Without it, `CameraWidget` shows its
172/// test pattern.
173pub fn register_camera_backend(vtable: CaptureVTable) {
174 let _ = CAMERA_BACKEND.set(vtable);
175}
176
177/// Register the platform **screen** capture backend (for `ScreenCaptureWidget`).
178pub fn register_screen_backend(vtable: CaptureVTable) {
179 let _ = SCREEN_BACKEND.set(vtable);
180}
181
182/// The registered camera backend, if the dll provided one for this platform.
183pub fn camera_backend() -> Option<CaptureVTable> {
184 CAMERA_BACKEND.get().copied()
185}
186
187/// The registered screen-capture backend, if any.
188pub fn screen_backend() -> Option<CaptureVTable> {
189 SCREEN_BACKEND.get().copied()
190}
191
192/// A platform **audio**-capture backend (microphone), registered by the dll so
193/// `MicrophoneWidget` can pull real samples instead of the test tone. Like
194/// [`CaptureVTable`] but yields interleaved `f32` audio rather than RGBA video.
195#[derive(Clone, Copy)]
196pub struct AudioCaptureVTable {
197 /// Open the default mic at `sample_rate` x `channels`. Opaque handle, or
198 /// `0` on failure.
199 pub open: fn(sample_rate: u32, channels: u16) -> u64,
200 /// Block for the next chunk, writing interleaved `f32` into `out` (resized).
201 /// Returns the frame count (`out.len() / channels`), or `0` on error / EOF
202 /// (the worker then stops + closes).
203 pub read: fn(handle: u64, out: &mut alloc::vec::Vec<f32>) -> u32,
204 /// Close + free the source.
205 pub close: fn(handle: u64),
206}
207
208static MIC_BACKEND: std::sync::OnceLock<AudioCaptureVTable> = std::sync::OnceLock::new();
209
210/// Register the platform microphone-capture backend (called once by the dll).
211pub fn register_mic_backend(vtable: AudioCaptureVTable) {
212 let _ = MIC_BACKEND.set(vtable);
213}
214
215/// The registered mic-capture backend, if the dll provided one for this platform.
216pub fn mic_backend() -> Option<AudioCaptureVTable> {
217 MIC_BACKEND.get().copied()
218}