1#![deny(unsafe_code)]
9#[allow(unsafe_code)]
10mod native;
11use ferroui_base::{
12 platform::KnownPlatformGraphicsExternalImageHandleTypes,
13 rendering::composition::ICompositionGpuInterop,
14};
15use ferroui_browser::interop::{
16 completion_helper::{await_promise, PromiseFuture},
17 JsObject,
18};
19use ferroui_media::{frames::*, *};
20use std::{
21 cell::{Cell, RefCell},
22 time::Duration,
23};
24use std::{
25 future::Future,
26 pin::Pin,
27 task::{Context, Poll, Waker},
28};
29use wasm_bindgen::JsValue;
30
31#[cfg(feature = "qr")]
32mod qr;
33#[cfg(feature = "qr")]
34pub use qr::recognize_qr;
35
36#[derive(Clone)]
39pub struct WebGlFrameProducer {
40 context: u64,
41 generation: u64,
42 budget: FrameBudget,
43 graphics: Acceleration,
44 transfer: bool,
46 analysis: bool,
47}
48impl WebGlFrameProducer {
49 pub fn from_interop(
50 interop: &dyn ICompositionGpuInterop,
51 budget: FrameBudget,
52 ) -> Result<Self, MediaError> {
53 if !cfg!(target_os = "emscripten") {
54 return Err(unsupported("Shared WebGL frames need the browser platform"));
55 }
56 let types = interop.supported_image_handle_types();
57 let has = |kind: &str| types.iter().any(|v| v == kind);
58 if interop.shared_context_id().is_none()
61 && has(KnownPlatformGraphicsExternalImageHandleTypes::BROWSER_TRANSFERRED_FRAME)
62 {
63 if !native::can_transfer() {
64 return Err(unsupported(
65 "Media frames on the render worker need WebCodecs VideoFrame and a started render worker",
66 ));
67 }
68 return Ok(Self {
69 context: 0,
70 generation: interop.shared_context_generation(),
71 budget,
72 graphics: Acceleration::Unknown,
74 transfer: true,
75 analysis: false,
76 });
77 }
78 let context = interop
79 .shared_context_id()
80 .ok_or_else(|| unsupported("The compositor has no admitted shared WebGL context"))?;
81 if context > i32::MAX as u64
82 || !interop
83 .supported_image_handle_types()
84 .iter()
85 .any(|v| v == KnownPlatformGraphicsExternalImageHandleTypes::SHARED_GL_TEXTURE)
86 {
87 return Err(unsupported(
88 "The compositor has no lease-backed WebGL importer",
89 ));
90 }
91 let graphics = match native::graphics(context as i32) {
92 2 => Acceleration::Software,
93 0 => Acceleration::Unknown,
94 _ => return Err(unsupported("The WebGL context is unavailable")),
95 };
96 if graphics == Acceleration::Software {
97 return Err(unsupported(
98 "Software graphics has no admitted GPU media route",
99 ));
100 }
101 Ok(Self {
102 context,
103 generation: interop.shared_context_generation(),
104 budget,
105 graphics,
106 transfer: false,
107 analysis: false,
108 })
109 }
110 pub fn budget(&self) -> FrameBudget {
111 self.budget.clone()
112 }
113 pub fn device(&self) -> DeviceIdentity {
114 DeviceIdentity::Context(self.context)
115 }
116 pub fn device_generation(&self) -> u64 {
117 self.generation
118 }
119 pub fn graphics(&self) -> Acceleration {
120 self.graphics
121 }
122 pub fn with_budget(&self, budget: FrameBudget) -> Self {
123 Self {
124 budget,
125 ..self.clone()
126 }
127 }
128 pub fn with_analysis(&self, enabled: bool) -> Self {
129 Self {
130 analysis: enabled,
131 ..self.clone()
132 }
133 }
134 pub fn stream(&self, source: u64, generation: u64) -> BrowserFrameStream {
135 BrowserFrameStream {
136 producer: self.clone(),
137 source,
138 generation,
139 last_sequence: Cell::new(0),
140 pending: RefCell::new(None),
141 retrying: Cell::new(false),
142 }
143 }
144 pub fn is_current(&self) -> bool {
145 self.transfer || native::current(self.context as i32, self.generation)
147 }
148 pub fn transfers_frames(&self) -> bool {
151 self.transfer
152 }
153 pub fn quadrants(
156 &self,
157 identity: FrameIdentity,
158 width: u32,
159 height: u32,
160 ) -> Result<Option<PendingWebGlFrame>, MediaError> {
161 if self.transfer {
162 return Err(unsupported(
163 "A generated GPU test pattern needs the compositor's context on the page's thread",
164 ));
165 }
166 self.produce(identity, width, height, Duration::ZERO, false, |texture| {
167 native::quadrants(texture, width, height)
168 })
169 }
170 pub fn upload(
173 &self,
174 identity: FrameIdentity,
175 width: u32,
176 height: u32,
177 timestamp: Duration,
178 source: &JsValue,
179 ) -> Result<Option<PendingWebGlFrame>, MediaError> {
180 if !native::is_video_source(source) {
181 let bytes = u64::from(width)
182 .saturating_mul(u64::from(height))
183 .saturating_mul(4);
184 let _ = self.budget.counters().cpu_transfer(
185 PixelTransferPurpose::Preview,
186 bytes,
187 PresentationRoute::SharedTexture,
188 );
189 return Err(unsupported(
190 "CPU-backed image/canvas upload is forbidden in shared GPU preview",
191 ));
192 }
193 if self.transfer {
194 return self.produce_transferred(identity, width, height, timestamp, source);
195 }
196 self.produce(identity, width, height, timestamp, true, |texture| {
197 self.budget
198 .counters()
199 .gpu_copy(GpuTransferReason::NativeImageTransfer);
200 native::upload(texture, source, width, height)
201 })
202 }
203 fn produce_transferred(
207 &self,
208 identity: FrameIdentity,
209 width: u32,
210 height: u32,
211 timestamp: Duration,
212 source: &JsValue,
213 ) -> Result<Option<PendingWebGlFrame>, MediaError> {
214 let metadata = self.metadata(identity, width, height, timestamp, true, None)?;
215 let reservation = match self.budget.reserve(metadata) {
216 Ok(v) => v,
217 Err(FrameError::BudgetExceeded) => return Ok(None),
218 Err(e) => return Err(unsupported(e.to_string())),
219 };
220 let (frame, receipt) = native::transfer(
221 source,
222 width,
223 height,
224 self.analysis,
225 reservation.retirement(),
226 )?;
227 self.budget
228 .counters()
229 .gpu_copy(GpuTransferReason::NativeImageTransfer);
230 let frame = reservation.lease(frame);
231 let ready = FrameReadiness::Receipt {
232 pending: RefCell::new(Some(await_promise(&receipt))),
233 result: RefCell::new(None),
234 };
235 Ok(Some(PendingWebGlFrame { frame, ready }))
236 }
237 fn metadata(
240 &self,
241 identity: FrameIdentity,
242 width: u32,
243 height: u32,
244 timestamp: Duration,
245 top_left_origin: bool,
246 device: Option<DeviceIdentity>,
247 ) -> Result<FrameMetadata, MediaError> {
248 let bytes = u64::from(width)
249 .checked_mul(u64::from(height))
250 .and_then(|v| v.checked_mul(4))
251 .and_then(|bytes| bytes.checked_mul(if self.transfer && self.analysis { 2 } else { 1 }))
252 .ok_or_else(|| unsupported("Texture dimensions overflow"))?;
253 let metadata = FrameMetadata {
254 identity,
255 width,
256 height,
257 visible: FrameRegion {
258 x: 0,
259 y: 0,
260 width,
261 height,
262 },
263 format: FrameFormat::Rgba8,
264 color: FrameColor::Srgb,
265 alpha: FrameAlpha::Opaque,
266 rotation: FrameRotation::None,
267 mirrored: false,
268 top_left_origin,
269 timestamp,
270 device,
271 device_generation: self.generation,
272 planes: vec![FramePlane {
273 offset: 0,
274 stride: u64::from(width) * 4,
275 length: u64::from(width) * u64::from(height) * 4,
276 }],
277 retained_bytes: bytes,
280 };
281 if self.budget.limits().frames == 0 || bytes > self.budget.limits().bytes {
282 return Err(unsupported("Frame exceeds the configured storage budget"));
283 }
284 Ok(metadata)
285 }
286 fn produce(
287 &self,
288 identity: FrameIdentity,
289 width: u32,
290 height: u32,
291 timestamp: Duration,
292 top_left_origin: bool,
293 write: impl FnOnce(u32) -> Result<(), MediaError>,
294 ) -> Result<Option<PendingWebGlFrame>, MediaError> {
295 if !self.is_current() {
296 return Err(unsupported(
297 "The WebGL context generation was lost or replaced",
298 ));
299 }
300 let metadata = self.metadata(
301 identity,
302 width,
303 height,
304 timestamp,
305 top_left_origin,
306 Some(self.device()),
307 )?;
308 let reservation = match self.budget.reserve(metadata) {
309 Ok(v) => v,
310 Err(FrameError::BudgetExceeded) => return Ok(None),
311 Err(e) => return Err(unsupported(e.to_string())),
312 };
313 let resource = native::create(
314 self.context as i32,
315 self.generation,
316 reservation.retirement(),
317 )?;
318 let texture = resource.texture();
319 let frame = reservation.lease(resource);
320 let ready = GpuCompletion::new();
321 if let Err(e) = ready.retain(frame.clone()) {
323 ready.complete();
324 if e == FrameError::BudgetExceeded {
325 return Ok(None);
326 }
327 return Err(unsupported(e.to_string()));
328 }
329 let result = write(texture);
330 ferroui_browser::rendering::schedule_webgl_completion(self.context as i32, ready.clone());
331 result?;
332 Ok(Some(PendingWebGlFrame {
333 frame,
334 ready: FrameReadiness::Gpu(ready),
335 }))
336 }
337}
338pub struct PendingWebGlFrame {
340 frame: FrameLease,
341 ready: FrameReadiness,
342}
343enum FrameReadiness {
344 Gpu(GpuCompletion),
345 Receipt {
346 pending: RefCell<Option<PromiseFuture<JsObject>>>,
347 result: RefCell<Option<Result<(), MediaError>>>,
348 },
349}
350impl FrameReadiness {
351 fn poll_receipt(&self, cx: &mut Context<'_>) -> Poll<Result<(), MediaError>> {
352 let Self::Receipt { pending, result } = self else {
353 unreachable!()
354 };
355 if let Some(value) = result.borrow().as_ref() {
356 return Poll::Ready(value.clone());
357 }
358 let mut pending = pending.borrow_mut();
359 match Pin::new(pending.as_mut().unwrap()).poll(cx) {
360 Poll::Pending => Poll::Pending,
361 Poll::Ready(value) => {
362 let value = value.map(|_| ()).map_err(|e| unsupported(e.to_string()));
363 pending.take();
364 *result.borrow_mut() = Some(value.clone());
365 Poll::Ready(value)
366 }
367 }
368 }
369}
370pub struct BrowserImageFrame {
373 pub image: JsValue,
374 pub sequence: u64,
375 pub timestamp: Duration,
376 pub width: u32,
377 pub height: u32,
378}
379pub struct BrowserFrameStream {
382 producer: WebGlFrameProducer,
383 source: u64,
384 generation: u64,
385 last_sequence: Cell<u64>,
386 pending: RefCell<Option<PendingWebGlFrame>>,
387 retrying: Cell<bool>,
388}
389impl BrowserFrameStream {
390 pub fn clear(&self) {
393 self.pending.borrow_mut().take();
394 self.retrying.set(false);
395 }
396 pub fn has_pending_frame_work(&self) -> bool {
397 self.pending.borrow().is_some() || self.retrying.get()
398 }
399 pub fn poll(
400 &self,
401 sample: impl FnOnce() -> Result<Option<BrowserImageFrame>, MediaError>,
402 ) -> Result<Option<FrameLease>, MediaError> {
403 let ready = self
404 .pending
405 .borrow()
406 .as_ref()
407 .map(PendingWebGlFrame::poll)
408 .transpose()?;
409 let frame = match ready {
410 Some(None) => return Ok(None),
411 Some(Some(frame)) => {
412 self.pending.borrow_mut().take();
413 Some(frame)
414 }
415 None => None,
416 };
417 let Some(sample) = sample()? else {
418 return Ok(frame);
419 };
420 if sample.sequence <= self.last_sequence.get() {
421 return Ok(frame);
422 }
423 if let Some(pending) = self.producer.upload(
424 FrameIdentity {
425 source: self.source,
426 generation: self.generation,
427 sequence: sample.sequence,
428 },
429 sample.width,
430 sample.height,
431 sample.timestamp,
432 &sample.image,
433 )? {
434 self.last_sequence.set(sample.sequence);
435 *self.pending.borrow_mut() = Some(pending);
436 self.retrying.set(false);
437 } else {
438 self.retrying.set(true);
439 }
440 Ok(frame)
441 }
442}
443impl PendingWebGlFrame {
444 pub fn poll(&self) -> Result<Option<FrameLease>, MediaError> {
445 let FrameReadiness::Gpu(ready) = &self.ready else {
446 return match self
447 .ready
448 .poll_receipt(&mut Context::from_waker(Waker::noop()))
449 {
450 Poll::Pending => Ok(None),
451 Poll::Ready(value) => value.map(|_| Some(self.frame.clone())),
452 };
453 };
454 match ready.status() {
455 GpuCompletionStatus::Pending => Ok(None),
456 GpuCompletionStatus::Complete => Ok(Some(self.frame.clone())),
457 GpuCompletionStatus::Indeterminate => Err(unsupported(
458 "GPU completion failed; storage remains quarantined",
459 )),
460 }
461 }
462 pub async fn finish(self) -> Result<FrameLease, MediaError> {
463 match &self.ready {
464 FrameReadiness::Gpu(ready) => {
465 ready.wait().await.map_err(|e| unsupported(e.to_string()))?
466 }
467 _ => std::future::poll_fn(|cx| self.ready.poll_receipt(cx)).await?,
468 }
469 Ok(self.frame)
470 }
471}
472pub fn unsupported(message: impl Into<String>) -> MediaError {
473 MediaError::new(MediaErrorKind::Unsupported, message)
474}
475
476#[cfg(feature = "controls")]
477pub struct BrowserTextureImporter;
478#[cfg(feature = "controls")]
479impl ferroui_media_controls::TextureFrameImporter for BrowserTextureImporter {
480 fn import(
481 &self,
482 interop: &dyn ICompositionGpuInterop,
483 frame: &FrameLease,
484 ) -> Result<ferroui_media_controls::ImportedTextureFrame, MediaError> {
485 use ferroui_base::platform::{
486 PlatformGraphicsExternalImageFormat, PlatformGraphicsExternalImageProperties,
487 };
488 let m = frame.metadata();
489 let handle: std::rc::Rc<dyn ferroui_base::platform::IPlatformHandle> =
492 if let Some(transferred) = frame.resource::<native::TransferredFrame>() {
493 std::rc::Rc::new(ferroui_base::platform::LeasedPlatformHandle::new(
494 transferred.id() as isize,
495 Some(KnownPlatformGraphicsExternalImageHandleTypes::BROWSER_TRANSFERRED_FRAME),
496 frame.clone(),
497 ))
498 } else {
499 if m.device != interop.shared_context_id().map(DeviceIdentity::Context)
500 || m.device_generation != interop.shared_context_generation()
501 {
502 return Err(unsupported("Frame and renderer contexts differ"));
503 }
504 ferroui_opengl::SharedFrameTexture::handle(frame)
505 .map_err(|e| unsupported(e.to_string()))?
506 };
507 let image = interop.import_image(
508 handle,
509 PlatformGraphicsExternalImageProperties {
510 width: m.width as i32,
511 height: m.height as i32,
512 format: PlatformGraphicsExternalImageFormat::R8G8B8A8UNorm,
513 top_left_origin: m.top_left_origin,
514 memory_size: m.retained_bytes,
515 ..Default::default()
516 },
517 );
518 let producer_ready = GpuCompletion::new();
520 producer_ready.complete();
521 Ok(ferroui_media_controls::ImportedTextureFrame {
522 image,
523 producer_ready,
524 })
525 }
526}