Skip to main content

ferroui_media_browser/
lib.rs

1//! Immutable WebGL2 storage shared by independent producers, video and camera.
2//! Two routes, chosen at run time by where the compositor's WebGL2 context lives:
3//! on the page's thread, native media transfers use TexImageSource directly into
4//! a shared texture; on the render worker of a module with threads, each frame is
5//! a WebCodecs `VideoFrame` transferred to the worker and uploaded there when the
6//! compositor imports it (#601). No framework preview pixel array or readback is
7//! admitted on either route. Browser decoder/driver internals are opaque.
8#![deny(unsafe_code)]
9#[allow(unsafe_code)]
10mod native;
11use ferroui_base::{
12    platform::KnownPlatformGraphicsExternalImageHandleTypes,
13    rendering::composition::ICompositionGpuInterop,
14};
15use ferroui_browser::interop::{
16    completion_helper::{await_promise, PromiseFuture},
17    JsObject,
18};
19use ferroui_media::{frames::*, *};
20use std::{
21    cell::{Cell, RefCell},
22    time::Duration,
23};
24use std::{
25    future::Future,
26    pin::Pin,
27    task::{Context, Poll, Waker},
28};
29use wasm_bindgen::JsValue;
30
31#[cfg(feature = "qr")]
32mod qr;
33#[cfg(feature = "qr")]
34pub use qr::recognize_qr;
35
36/// Allocation policy for the active compositor context. Old generations remain
37/// charged to the same budget until their native storage actually retires.
38#[derive(Clone)]
39pub struct WebGlFrameProducer {
40    context: u64,
41    generation: u64,
42    budget: FrameBudget,
43    graphics: Acceleration,
44    /// Frames go to the render worker as transferred `VideoFrame`s (#601).
45    transfer: bool,
46    analysis: bool,
47}
48impl WebGlFrameProducer {
49    pub fn from_interop(
50        interop: &dyn ICompositionGpuInterop,
51        budget: FrameBudget,
52    ) -> Result<Self, MediaError> {
53        if !cfg!(target_os = "emscripten") {
54            return Err(unsupported("Shared WebGL frames need the browser platform"));
55        }
56        let types = interop.supported_image_handle_types();
57        let has = |kind: &str| types.iter().any(|v| v == kind);
58        // The compositor renders on the render worker: its context cannot sample the page's
59        // media elements, so the frames are transferred to it (#601).
60        if interop.shared_context_id().is_none()
61            && has(KnownPlatformGraphicsExternalImageHandleTypes::BROWSER_TRANSFERRED_FRAME)
62        {
63            if !native::can_transfer() {
64                return Err(unsupported(
65                    "Media frames on the render worker need WebCodecs VideoFrame and a started render worker",
66                ));
67            }
68            return Ok(Self {
69                context: 0,
70                generation: interop.shared_context_generation(),
71                budget,
72                // The renderer of the worker's context is not observable from the page.
73                graphics: Acceleration::Unknown,
74                transfer: true,
75                analysis: false,
76            });
77        }
78        let context = interop
79            .shared_context_id()
80            .ok_or_else(|| unsupported("The compositor has no admitted shared WebGL context"))?;
81        if context > i32::MAX as u64
82            || !interop
83                .supported_image_handle_types()
84                .iter()
85                .any(|v| v == KnownPlatformGraphicsExternalImageHandleTypes::SHARED_GL_TEXTURE)
86        {
87            return Err(unsupported(
88                "The compositor has no lease-backed WebGL importer",
89            ));
90        }
91        let graphics = match native::graphics(context as i32) {
92            2 => Acceleration::Software,
93            0 => Acceleration::Unknown,
94            _ => return Err(unsupported("The WebGL context is unavailable")),
95        };
96        if graphics == Acceleration::Software {
97            return Err(unsupported(
98                "Software graphics has no admitted GPU media route",
99            ));
100        }
101        Ok(Self {
102            context,
103            generation: interop.shared_context_generation(),
104            budget,
105            graphics,
106            transfer: false,
107            analysis: false,
108        })
109    }
110    pub fn budget(&self) -> FrameBudget {
111        self.budget.clone()
112    }
113    pub fn device(&self) -> DeviceIdentity {
114        DeviceIdentity::Context(self.context)
115    }
116    pub fn device_generation(&self) -> u64 {
117        self.generation
118    }
119    pub fn graphics(&self) -> Acceleration {
120        self.graphics
121    }
122    pub fn with_budget(&self, budget: FrameBudget) -> Self {
123        Self {
124            budget,
125            ..self.clone()
126        }
127    }
128    pub fn with_analysis(&self, enabled: bool) -> Self {
129        Self {
130            analysis: enabled,
131            ..self.clone()
132        }
133    }
134    pub fn stream(&self, source: u64, generation: u64) -> BrowserFrameStream {
135        BrowserFrameStream {
136            producer: self.clone(),
137            source,
138            generation,
139            last_sequence: Cell::new(0),
140            pending: RefCell::new(None),
141            retrying: Cell::new(false),
142        }
143    }
144    pub fn is_current(&self) -> bool {
145        // A transferred frame is bound to the worker's context only when it is imported.
146        self.transfer || native::current(self.context as i32, self.generation)
147    }
148    /// Whether frames are transferred to the render worker (#601) rather than written into a
149    /// texture of a context of this thread.
150    pub fn transfers_frames(&self) -> bool {
151        self.transfer
152    }
153    /// Independent GPU producer. Its framebuffer is bottom-left oriented;
154    /// presentation must preserve that origin rather than flip CPU pixels.
155    pub fn quadrants(
156        &self,
157        identity: FrameIdentity,
158        width: u32,
159        height: u32,
160    ) -> Result<Option<PendingWebGlFrame>, MediaError> {
161        if self.transfer {
162            return Err(unsupported(
163                "A generated GPU test pattern needs the compositor's context on the page's thread",
164            ));
165        }
166        self.produce(identity, width, height, Duration::ZERO, false, |texture| {
167            native::quadrants(texture, width, height)
168        })
169    }
170    /// Transfers a browser-owned HTMLVideoElement image directly to the active GL
171    /// context. CORS/security failures remain errors; there is no CPU fallback.
172    pub fn upload(
173        &self,
174        identity: FrameIdentity,
175        width: u32,
176        height: u32,
177        timestamp: Duration,
178        source: &JsValue,
179    ) -> Result<Option<PendingWebGlFrame>, MediaError> {
180        if !native::is_video_source(source) {
181            let bytes = u64::from(width)
182                .saturating_mul(u64::from(height))
183                .saturating_mul(4);
184            let _ = self.budget.counters().cpu_transfer(
185                PixelTransferPurpose::Preview,
186                bytes,
187                PresentationRoute::SharedTexture,
188            );
189            return Err(unsupported(
190                "CPU-backed image/canvas upload is forbidden in shared GPU preview",
191            ));
192        }
193        if self.transfer {
194            return self.produce_transferred(identity, width, height, timestamp, source);
195        }
196        self.produce(identity, width, height, timestamp, true, |texture| {
197            self.budget
198                .counters()
199                .gpu_copy(GpuTransferReason::NativeImageTransfer);
200            native::upload(texture, source, width, height)
201        })
202    }
203    /// The route of the render worker (#601): the frame is reserved here and transferred to the
204    /// worker as a WebCodecs `VideoFrame`, a GPU-backed image; the compositor uploads it into its
205    /// context when it imports it. No pixel array is made on either side.
206    fn produce_transferred(
207        &self,
208        identity: FrameIdentity,
209        width: u32,
210        height: u32,
211        timestamp: Duration,
212        source: &JsValue,
213    ) -> Result<Option<PendingWebGlFrame>, MediaError> {
214        let metadata = self.metadata(identity, width, height, timestamp, true, None)?;
215        let reservation = match self.budget.reserve(metadata) {
216            Ok(v) => v,
217            Err(FrameError::BudgetExceeded) => return Ok(None),
218            Err(e) => return Err(unsupported(e.to_string())),
219        };
220        let (frame, receipt) = native::transfer(
221            source,
222            width,
223            height,
224            self.analysis,
225            reservation.retirement(),
226        )?;
227        self.budget
228            .counters()
229            .gpu_copy(GpuTransferReason::NativeImageTransfer);
230        let frame = reservation.lease(frame);
231        let ready = FrameReadiness::Receipt {
232            pending: RefCell::new(Some(await_promise(&receipt))),
233            result: RefCell::new(None),
234        };
235        Ok(Some(PendingWebGlFrame { frame, ready }))
236    }
237    /// The metadata of a frame of this producer, within its budget. A transferred frame has no
238    /// device until the compositor imports it (#601).
239    fn metadata(
240        &self,
241        identity: FrameIdentity,
242        width: u32,
243        height: u32,
244        timestamp: Duration,
245        top_left_origin: bool,
246        device: Option<DeviceIdentity>,
247    ) -> Result<FrameMetadata, MediaError> {
248        let bytes = u64::from(width)
249            .checked_mul(u64::from(height))
250            .and_then(|v| v.checked_mul(4))
251            .and_then(|bytes| bytes.checked_mul(if self.transfer && self.analysis { 2 } else { 1 }))
252            .ok_or_else(|| unsupported("Texture dimensions overflow"))?;
253        let metadata = FrameMetadata {
254            identity,
255            width,
256            height,
257            visible: FrameRegion {
258                x: 0,
259                y: 0,
260                width,
261                height,
262            },
263            format: FrameFormat::Rgba8,
264            color: FrameColor::Srgb,
265            alpha: FrameAlpha::Opaque,
266            rotation: FrameRotation::None,
267            mirrored: false,
268            top_left_origin,
269            timestamp,
270            device,
271            device_generation: self.generation,
272            planes: vec![FramePlane {
273                offset: 0,
274                stride: u64::from(width) * 4,
275                length: u64::from(width) * u64::from(height) * 4,
276            }],
277            // RGBA8 storage payload is known. Browser allocation overhead and
278            // decoder-owned storage are not observable through WebGL.
279            retained_bytes: bytes,
280        };
281        if self.budget.limits().frames == 0 || bytes > self.budget.limits().bytes {
282            return Err(unsupported("Frame exceeds the configured storage budget"));
283        }
284        Ok(metadata)
285    }
286    fn produce(
287        &self,
288        identity: FrameIdentity,
289        width: u32,
290        height: u32,
291        timestamp: Duration,
292        top_left_origin: bool,
293        write: impl FnOnce(u32) -> Result<(), MediaError>,
294    ) -> Result<Option<PendingWebGlFrame>, MediaError> {
295        if !self.is_current() {
296            return Err(unsupported(
297                "The WebGL context generation was lost or replaced",
298            ));
299        }
300        let metadata = self.metadata(
301            identity,
302            width,
303            height,
304            timestamp,
305            top_left_origin,
306            Some(self.device()),
307        )?;
308        let reservation = match self.budget.reserve(metadata) {
309            Ok(v) => v,
310            Err(FrameError::BudgetExceeded) => return Ok(None),
311            Err(e) => return Err(unsupported(e.to_string())),
312        };
313        let resource = native::create(
314            self.context as i32,
315            self.generation,
316            reservation.retirement(),
317        )?;
318        let texture = resource.texture();
319        let frame = reservation.lease(resource);
320        let ready = GpuCompletion::new();
321        // Retain before the first producer GPU command, including error paths.
322        if let Err(e) = ready.retain(frame.clone()) {
323            ready.complete();
324            if e == FrameError::BudgetExceeded {
325                return Ok(None);
326            }
327            return Err(unsupported(e.to_string()));
328        }
329        let result = write(texture);
330        ferroui_browser::rendering::schedule_webgl_completion(self.context as i32, ready.clone());
331        result?;
332        Ok(Some(PendingWebGlFrame {
333            frame,
334            ready: FrameReadiness::Gpu(ready),
335        }))
336    }
337}
338/// Cancellation drops the caller's view; the fence retains submitted storage.
339pub struct PendingWebGlFrame {
340    frame: FrameLease,
341    ready: FrameReadiness,
342}
343enum FrameReadiness {
344    Gpu(GpuCompletion),
345    Receipt {
346        pending: RefCell<Option<PromiseFuture<JsObject>>>,
347        result: RefCell<Option<Result<(), MediaError>>>,
348    },
349}
350impl FrameReadiness {
351    fn poll_receipt(&self, cx: &mut Context<'_>) -> Poll<Result<(), MediaError>> {
352        let Self::Receipt { pending, result } = self else {
353            unreachable!()
354        };
355        if let Some(value) = result.borrow().as_ref() {
356            return Poll::Ready(value.clone());
357        }
358        let mut pending = pending.borrow_mut();
359        match Pin::new(pending.as_mut().unwrap()).poll(cx) {
360            Poll::Pending => Poll::Pending,
361            Poll::Ready(value) => {
362                let value = value.map(|_| ()).map_err(|e| unsupported(e.to_string()));
363                pending.take();
364                *result.borrow_mut() = Some(value.clone());
365                Poll::Ready(value)
366            }
367        }
368    }
369}
370/// Native browser image and its source-frame timestamp, sampled synchronously
371/// with the subsequent TexImageSource transfer. No framework pixel buffer.
372pub struct BrowserImageFrame {
373    pub image: JsValue,
374    pub sequence: u64,
375    pub timestamp: Duration,
376    pub width: u32,
377    pub height: u32,
378}
379/// One producer transfer in flight, with latest-native-frame dropping. A
380/// second view reads the shared mailbox; it does not call this acquisition.
381pub struct BrowserFrameStream {
382    producer: WebGlFrameProducer,
383    source: u64,
384    generation: u64,
385    last_sequence: Cell<u64>,
386    pending: RefCell<Option<PendingWebGlFrame>>,
387    retrying: Cell<bool>,
388}
389impl BrowserFrameStream {
390    /// Retire queued producer output on seek/stop. Submitted GPU storage is
391    /// still held by its completion; the last native sequence stays rejected.
392    pub fn clear(&self) {
393        self.pending.borrow_mut().take();
394        self.retrying.set(false);
395    }
396    pub fn has_pending_frame_work(&self) -> bool {
397        self.pending.borrow().is_some() || self.retrying.get()
398    }
399    pub fn poll(
400        &self,
401        sample: impl FnOnce() -> Result<Option<BrowserImageFrame>, MediaError>,
402    ) -> Result<Option<FrameLease>, MediaError> {
403        let ready = self
404            .pending
405            .borrow()
406            .as_ref()
407            .map(PendingWebGlFrame::poll)
408            .transpose()?;
409        let frame = match ready {
410            Some(None) => return Ok(None),
411            Some(Some(frame)) => {
412                self.pending.borrow_mut().take();
413                Some(frame)
414            }
415            None => None,
416        };
417        let Some(sample) = sample()? else {
418            return Ok(frame);
419        };
420        if sample.sequence <= self.last_sequence.get() {
421            return Ok(frame);
422        }
423        if let Some(pending) = self.producer.upload(
424            FrameIdentity {
425                source: self.source,
426                generation: self.generation,
427                sequence: sample.sequence,
428            },
429            sample.width,
430            sample.height,
431            sample.timestamp,
432            &sample.image,
433        )? {
434            self.last_sequence.set(sample.sequence);
435            *self.pending.borrow_mut() = Some(pending);
436            self.retrying.set(false);
437        } else {
438            self.retrying.set(true);
439        }
440        Ok(frame)
441    }
442}
443impl PendingWebGlFrame {
444    pub fn poll(&self) -> Result<Option<FrameLease>, MediaError> {
445        let FrameReadiness::Gpu(ready) = &self.ready else {
446            return match self
447                .ready
448                .poll_receipt(&mut Context::from_waker(Waker::noop()))
449            {
450                Poll::Pending => Ok(None),
451                Poll::Ready(value) => value.map(|_| Some(self.frame.clone())),
452            };
453        };
454        match ready.status() {
455            GpuCompletionStatus::Pending => Ok(None),
456            GpuCompletionStatus::Complete => Ok(Some(self.frame.clone())),
457            GpuCompletionStatus::Indeterminate => Err(unsupported(
458                "GPU completion failed; storage remains quarantined",
459            )),
460        }
461    }
462    pub async fn finish(self) -> Result<FrameLease, MediaError> {
463        match &self.ready {
464            FrameReadiness::Gpu(ready) => {
465                ready.wait().await.map_err(|e| unsupported(e.to_string()))?
466            }
467            _ => std::future::poll_fn(|cx| self.ready.poll_receipt(cx)).await?,
468        }
469        Ok(self.frame)
470    }
471}
472pub fn unsupported(message: impl Into<String>) -> MediaError {
473    MediaError::new(MediaErrorKind::Unsupported, message)
474}
475
476#[cfg(feature = "controls")]
477pub struct BrowserTextureImporter;
478#[cfg(feature = "controls")]
479impl ferroui_media_controls::TextureFrameImporter for BrowserTextureImporter {
480    fn import(
481        &self,
482        interop: &dyn ICompositionGpuInterop,
483        frame: &FrameLease,
484    ) -> Result<ferroui_media_controls::ImportedTextureFrame, MediaError> {
485        use ferroui_base::platform::{
486            PlatformGraphicsExternalImageFormat, PlatformGraphicsExternalImageProperties,
487        };
488        let m = frame.metadata();
489        // #601: a frame transferred to the render worker is uploaded into the compositor's
490        // context when the compositor imports it.
491        let handle: std::rc::Rc<dyn ferroui_base::platform::IPlatformHandle> =
492            if let Some(transferred) = frame.resource::<native::TransferredFrame>() {
493                std::rc::Rc::new(ferroui_base::platform::LeasedPlatformHandle::new(
494                    transferred.id() as isize,
495                    Some(KnownPlatformGraphicsExternalImageHandleTypes::BROWSER_TRANSFERRED_FRAME),
496                    frame.clone(),
497                ))
498            } else {
499                if m.device != interop.shared_context_id().map(DeviceIdentity::Context)
500                    || m.device_generation != interop.shared_context_generation()
501                {
502                    return Err(unsupported("Frame and renderer contexts differ"));
503                }
504                ferroui_opengl::SharedFrameTexture::handle(frame)
505                    .map_err(|e| unsupported(e.to_string()))?
506            };
507        let image = interop.import_image(
508            handle,
509            PlatformGraphicsExternalImageProperties {
510                width: m.width as i32,
511                height: m.height as i32,
512                format: PlatformGraphicsExternalImageFormat::R8G8B8A8UNorm,
513                top_left_origin: m.top_left_origin,
514                memory_size: m.retained_bytes,
515                ..Default::default()
516            },
517        );
518        // PendingWebGlFrame only exposes a frame after the producer fence.
519        let producer_ready = GpuCompletion::new();
520        producer_ready.complete();
521        Ok(ferroui_media_controls::ImportedTextureFrame {
522            image,
523            producer_ready,
524        })
525    }
526}