Skip to main content

ferroui_media_android/
lib.rs

1//! Shared Android native SurfaceTexture output. No video/camera peer dependency.
2#![cfg(target_os = "android")]
3#![forbid(unsafe_code)]
4use ferroui_android::interop::java::*;
5use ferroui_base::{
6    reactive::Subscription,
7    threading::{DispatcherPriority, DispatcherTimer},
8};
9use ferroui_media::{frames::*, *};
10pub use ferroui_media_opengl::NativeGlFrameProducer;
11#[cfg(feature = "controls")]
12pub use ferroui_media_opengl::NativeGlTextureImporter;
13use ferroui_media_opengl::ReadyGlFrame;
14use ferroui_opengl::SharedGlTextureRetirementQueue;
15use std::{
16    cell::{Cell, RefCell},
17    rc::Rc,
18    sync::Arc,
19    task::{Context, Poll, Waker},
20    time::Duration,
21};
22
23/// Independent producer used to verify sharing before attaching video/camera.
24pub fn gpu_quadrants(
25    producer: &NativeGlFrameProducer,
26    identity: FrameIdentity,
27    width: u32,
28    height: u32,
29) -> MediaFuture<Option<ReadyGlFrame>> {
30    let class = JavaClass::find("org/ferroui/media/SurfaceFrames");
31    let bytes = width as u64 * height as u64 * 4;
32    let metadata = FrameMetadata {
33        identity,
34        width,
35        height,
36        visible: FrameRegion {
37            x: 0,
38            y: 0,
39            width,
40            height,
41        },
42        format: FrameFormat::Rgba8,
43        color: FrameColor::Srgb,
44        alpha: FrameAlpha::Opaque,
45        rotation: FrameRotation::None,
46        mirrored: false,
47        top_left_origin: false,
48        timestamp: Duration::ZERO,
49        device: Some(producer.device()),
50        device_generation: producer.device_generation(),
51        planes: vec![FramePlane {
52            offset: 0,
53            stride: width as u64 * 4,
54            length: bytes,
55        }],
56        retained_bytes: bytes,
57    };
58    producer.produce(metadata, move |_, output| {
59        let error = call_static_object(
60            &class,
61            "quadrants",
62            "(III)Ljava/lang/String;",
63            &[
64                JavaValue::Int(output as i32),
65                JavaValue::Int(width as i32),
66                JavaValue::Int(height as i32),
67            ],
68        )
69        .map(|v| string_of(&v))
70        .unwrap_or_else(|| "No independent producer result".into());
71        if error.is_empty() {
72            Ok(())
73        } else {
74            Err(native_error(error))
75        }
76    })
77}
78
79pub fn native_error(message: impl Into<String>) -> MediaError {
80    MediaError::new(MediaErrorKind::NativeFailure, message)
81}
82/// Live native source objects, distinct from immutable output frame leases and
83/// opaque codec/driver allocations. Destruction is counted after surface return.
84pub fn live_surface_sources() -> u32 {
85    call_static_int(
86        &JavaClass::find("org/ferroui/media/SurfaceFrames"),
87        "liveSources",
88        "()I",
89        &[],
90    )
91    .max(0) as u32
92}
93/// Poll native callbacks without busy-waiting or retaining canceled futures.
94pub fn poll_native<T: 'static, E: 'static>(
95    mut read: impl FnMut() -> Poll<Result<T, E>> + 'static,
96) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<T, E>>>> {
97    let mut timer: Option<Subscription> = None;
98    Box::pin(std::future::poll_fn(move |cx| {
99        timer.take();
100        match read() {
101            Poll::Ready(value) => Poll::Ready(value),
102            Poll::Pending => {
103                let waker = cx.waker().clone();
104                timer = Some(DispatcherTimer::run_once(
105                    move || waker.wake_by_ref(),
106                    Duration::from_millis(16),
107                    DispatcherPriority::NORMAL,
108                ));
109                Poll::Pending
110            }
111        }
112    }))
113}
114pub fn text(object: &JavaObject, method: &str) -> String {
115    call_object(object, method, "()Ljava/lang/String;", &[])
116        .map(|s| string_of(&s))
117        .unwrap_or_default()
118}
119struct SourceOwner {
120    java: JavaObject,
121    retirement: SharedGlTextureRetirementQueue,
122    _wake: Arc<dyn Fn() + Send + Sync>,
123}
124impl Drop for SourceOwner {
125    fn drop(&mut self) {
126        let java = self.java.clone();
127        self.retirement.defer_native(move |context| {
128            call_boolean(
129                &java,
130                "release",
131                "(Z)Z",
132                &[JavaValue::Boolean(context.is_some())],
133            )
134        });
135    }
136}
137/// One source acquisition shared by every 2D/3D consumer of its mailbox.
138#[derive(Clone)]
139pub struct AndroidSurfaceSource(Arc<SourceOwner>);
140impl AndroidSurfaceSource {
141    pub fn create(producer: &NativeGlFrameProducer) -> MediaFuture<Self> {
142        // Cache application classes on the Android UI thread before JNI calls
143        // from a native render thread (FindClass there has a system loader).
144        let class = JavaClass::find("org/ferroui/media/SurfaceFrames");
145        let wake = producer.retirement_wakeup();
146        producer.invoke(move |context| {
147            let java = new_object(&class, "()V", &[]).to_global();
148            let error = text(&java, "error");
149            if !error.is_empty() {
150                return Err(native_error(error));
151            }
152            let retirement = context
153                .shared_frame_retirement_queue()
154                .ok_or_else(|| native_error("No native GL retirement"))?;
155            let wake = retirement.register_wakeup(wake);
156            Ok(Self(Arc::new(SourceOwner {
157                java,
158                retirement,
159                _wake: wake,
160            })))
161        })
162    }
163    pub fn java(&self) -> JavaObject {
164        self.0.java.clone()
165    }
166    pub fn surface(&self) -> Result<JavaObject, MediaError> {
167        call_object(&self.0.java, "surface", "()Landroid/view/Surface;", &[])
168            .map(|v| v.to_global())
169            .ok_or_else(|| native_error("Surface source was released"))
170    }
171}
172pub struct AndroidFrameStream {
173    producer: NativeGlFrameProducer,
174    source: RefCell<Option<AndroidSurfaceSource>>,
175    identity: FrameIdentity,
176    last_sequence: Arc<std::sync::atomic::AtomicU64>,
177    job: RefCell<Option<MediaFuture<Option<ReadyGlFrame>>>>,
178    pending: RefCell<Option<ReadyGlFrame>>,
179    stopped: Cell<bool>,
180    epoch: Arc<std::sync::atomic::AtomicU64>,
181    job_epoch: Rc<Cell<u64>>,
182    frame_clock: Option<JavaObject>,
183    capture_identity: bool,
184    clock_nanoseconds: bool,
185}
186impl AndroidFrameStream {
187    pub fn new(
188        producer: NativeGlFrameProducer,
189        source: AndroidSurfaceSource,
190        id: u64,
191        generation: u64,
192    ) -> Self {
193        Self {
194            producer,
195            source: RefCell::new(Some(source)),
196            identity: FrameIdentity {
197                source: id,
198                generation,
199                sequence: 0,
200            },
201            last_sequence: Arc::new(std::sync::atomic::AtomicU64::new(0)),
202            job: RefCell::new(None),
203            pending: RefCell::new(None),
204            stopped: Cell::new(false),
205            epoch: Arc::new(std::sync::atomic::AtomicU64::new(0)),
206            job_epoch: Rc::new(Cell::new(0)),
207            frame_clock: None,
208            capture_identity: false,
209            clock_nanoseconds: false,
210        }
211    }
212    /// A bounded native frame-clock mapper for playback. Camera streams keep
213    /// their native capture timestamps and do not depend on a video crate.
214    pub fn with_frame_clock(mut self, clock: JavaObject) -> Self {
215        self.frame_clock = Some(clock);
216        self
217    }
218    /// Camera streams share sensor-timestamp identity with their optional
219    /// analysis stream; playback keeps monotonic acquisition sequence instead.
220    pub fn with_capture_identity(mut self) -> Self {
221        self.capture_identity = true;
222        self
223    }
224    /// Camera2 capture-result metadata belongs to the acquired sensor timestamp,
225    /// including across CameraX use-case rebinding. No guessed current size.
226    pub fn with_capture_clock(mut self, clock: JavaObject) -> Self {
227        self.frame_clock = Some(clock);
228        self.clock_nanoseconds = true;
229        self.capture_identity = true;
230        self
231    }
232    // Keep the current conversion until its producer fence retires. Canceling
233    // publication must not allow updateTexImage to reuse its input early.
234    pub fn clear(&self) {
235        self.epoch.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
236    }
237    pub fn stop(&self) {
238        self.stopped.set(true);
239        self.clear();
240        self.job.borrow_mut().take();
241        self.pending.borrow_mut().take();
242        self.source.borrow_mut().take();
243    }
244    pub fn has_pending_frame_work(&self) -> bool {
245        !self.stopped.get()
246            && (self.job.borrow().is_some()
247                || self.pending.borrow().is_some()
248                || self
249                    .source
250                    .borrow()
251                    .as_ref()
252                    .is_some_and(|source| call_boolean(&source.0.java, "pending", "()Z", &[])))
253    }
254    pub fn take_frame(&self, width: u32, height: u32) -> Result<Option<FrameLease>, MediaError> {
255        self.take_oriented_frame(width, height, FrameRotation::None, false)
256    }
257    pub fn take_oriented_frame(
258        &self,
259        width: u32,
260        height: u32,
261        rotation: FrameRotation,
262        mirrored: bool,
263    ) -> Result<Option<FrameLease>, MediaError> {
264        if self.stopped.get() || width == 0 || height == 0 {
265            return Ok(None);
266        }
267        let ready = self
268            .pending
269            .borrow()
270            .as_ref()
271            .map(ReadyGlFrame::take_if_ready);
272        if let Some(ready) = ready {
273            let frame = ready?;
274            if frame.is_none() {
275                return Ok(None);
276            }
277            self.pending.replace(None);
278            return Ok(
279                if self.epoch.load(std::sync::atomic::Ordering::Acquire) == self.job_epoch.get() {
280                    frame
281                } else {
282                    None
283                },
284            );
285        }
286        let result = {
287            let mut job = self.job.borrow_mut();
288            job.as_mut()
289                .map(|job| job.as_mut().poll(&mut Context::from_waker(Waker::noop())))
290        };
291        if let Some(result) = result {
292            match result {
293                Poll::Pending => return Ok(None),
294                Poll::Ready(result) => {
295                    self.job.borrow_mut().take();
296                    self.pending.replace(result?);
297                    return Ok(None);
298                }
299            }
300        }
301        let Some(source) = self.source.borrow().clone() else {
302            return Ok(None);
303        };
304        if !call_boolean(&source.0.java, "pending", "()Z", &[]) {
305            return Ok(None);
306        }
307        let producer = self.producer.clone();
308        let last = self.last_sequence.clone();
309        let mut identity = self.identity;
310        let frame_clock = self.frame_clock.clone();
311        let capture_identity = self.capture_identity;
312        let clock_nanoseconds = self.clock_nanoseconds;
313        let requested_epoch = self.epoch.load(std::sync::atomic::Ordering::Acquire);
314        self.job_epoch.set(requested_epoch);
315        let epoch = self.epoch.clone();
316        let job_epoch = self.job_epoch.clone();
317        self.job.replace(Some(Box::pin(async move {
318            let native = source.clone();
319            let acquisition_epoch = epoch.clone();
320            let acquired = producer
321                .invoke(move |_| {
322                    // An obsolete queued job must not consume a newly sought
323                    // paused frame: there may never be another frame callback.
324                    if acquisition_epoch.load(std::sync::atomic::Ordering::Acquire)
325                        != requested_epoch
326                    {
327                        return Ok(None);
328                    }
329                    if !call_boolean(&native.0.java, "pending", "()Z", &[]) {
330                        return Ok(None);
331                    }
332                    let error = text(&native.0.java, "acquire");
333                    if !error.is_empty() {
334                        return Err(if call_boolean(&native.0.java, "unsupported", "()Z", &[]) {
335                            MediaError::new(MediaErrorKind::Unsupported, error)
336                        } else {
337                            native_error(error)
338                        });
339                    }
340                    Ok(Some((
341                        call_long(&native.0.java, "sequence", "()J", &[]) as u64,
342                        call_long(&native.0.java, "timestamp", "()J", &[]).max(0) as u64,
343                        call_boolean(&native.0.java, "swapsAxes", "()Z", &[]),
344                    )))
345                })
346                .await?;
347            let Some((sequence, timestamp, swaps_axes)) = acquired else {
348                return Ok(None);
349            };
350            if sequence <= last.load(std::sync::atomic::Ordering::Acquire) {
351                return Ok(None);
352            }
353            identity.sequence = if capture_identity {
354                timestamp
355            } else {
356                sequence
357            };
358            let (timestamp, width, height, color, rotation, mirrored, visible) =
359                if let Some(clock) = frame_clock {
360                    let value = call_object(
361                        &clock,
362                        "frameInfo",
363                        "(J)Ljava/lang/String;",
364                        &[JavaValue::Long(timestamp as i64)],
365                    )
366                    .map(|v| string_of(&v))
367                    .unwrap_or_default();
368                    if value.is_empty() {
369                        return Ok(None);
370                    }
371                    let values = value
372                        .split(',')
373                        .map(str::parse::<u64>)
374                        .collect::<Result<Vec<_>, _>>()
375                        .map_err(|_| native_error("Invalid native frame-clock metadata"))?;
376                    if values.len() != if clock_nanoseconds { 6 } else { 10 }
377                        || values[1] > u32::MAX as u64
378                        || values[2] > u32::MAX as u64
379                    {
380                        return Err(native_error("Invalid native frame dimensions"));
381                    }
382                    (
383                        values[0]
384                            .checked_mul(if clock_nanoseconds { 1 } else { 1000 })
385                            .ok_or_else(|| native_error("Native timestamp overflow"))?,
386                        values[1] as u32,
387                        values[2] as u32,
388                        match values[3] {
389                            1 => FrameColor::Srgb,
390                            2 => FrameColor::Rec709Full,
391                            _ => return Err(native_error("Unsupported native video color")),
392                        },
393                        {
394                            match values[4] {
395                                0 => FrameRotation::None,
396                                90 => FrameRotation::Clockwise90,
397                                180 => FrameRotation::Clockwise180,
398                                270 => FrameRotation::Clockwise270,
399                                _ => {
400                                    return Err(native_error(
401                                        "Unsupported native frame orientation",
402                                    ))
403                                }
404                            }
405                        },
406                        {
407                            match values[5] {
408                                0 => false,
409                                1 => true,
410                                _ => return Err(native_error("Invalid native mirror flag")),
411                            }
412                        },
413                        if clock_nanoseconds {
414                            None
415                        } else {
416                            if values[6..].iter().any(|v| *v > u32::MAX as u64) {
417                                return Err(native_error("Native crop overflow"));
418                            }
419                            Some(FrameRegion {
420                                x: values[6] as u32,
421                                y: values[7] as u32,
422                                width: values[8] as u32,
423                                height: values[9] as u32,
424                            })
425                        },
426                    )
427                } else {
428                    if epoch.load(std::sync::atomic::Ordering::Acquire) != requested_epoch {
429                        return Ok(None);
430                    }
431                    (
432                        timestamp,
433                        width,
434                        height,
435                        FrameColor::Srgb,
436                        rotation,
437                        mirrored,
438                        None,
439                    )
440                };
441            // A seek can race the render-thread acquisition itself. The
442            // playback clock accepts only a frame belonging to its current
443            // epoch; keep that newly sought frame instead of discarding the
444            // sole paused callback just because the job was queued earlier.
445            job_epoch.set(epoch.load(std::sync::atomic::Ordering::Acquire));
446            // SurfaceTexture's native transform may already orient the image.
447            // Preserve its aspect instead of rotating it again in composition.
448            let (width, height) = if swaps_axes {
449                (height, width)
450            } else {
451                (width, height)
452            };
453            let bytes = width as u64 * height as u64 * 4;
454            let metadata = FrameMetadata {
455                identity,
456                width,
457                height,
458                visible: visible.unwrap_or(FrameRegion {
459                    x: 0,
460                    y: 0,
461                    width,
462                    height,
463                }),
464                format: FrameFormat::Rgba8,
465                color,
466                alpha: FrameAlpha::Opaque,
467                rotation,
468                mirrored,
469                top_left_origin: false,
470                timestamp: Duration::from_nanos(timestamp),
471                device: Some(producer.device()),
472                device_generation: producer.device_generation(),
473                planes: vec![FramePlane {
474                    offset: 0,
475                    stride: width as u64 * 4,
476                    length: bytes,
477                }],
478                retained_bytes: bytes,
479            };
480            let normalize_color = call_boolean(&source.0.java, "colorConversion", "()Z", &[]);
481            let counters = producer.budget().counters();
482            let owner = source.clone();
483            let output = producer
484                .produce_owned(metadata, owner, move |_, output| {
485                    counters.gpu_conversion(GpuTransferReason::PixelFormatConversion);
486                    if normalize_color {
487                        counters.gpu_conversion(GpuTransferReason::ColorSpaceConversion);
488                    }
489                    let error = call_object(
490                        &source.0.java,
491                        "draw",
492                        "(III)Ljava/lang/String;",
493                        &[
494                            JavaValue::Int(output as i32),
495                            JavaValue::Int(width as i32),
496                            JavaValue::Int(height as i32),
497                        ],
498                    )
499                    .map(|v| string_of(&v))
500                    .unwrap_or_default();
501                    if error.is_empty() {
502                        Ok(())
503                    } else {
504                        Err(native_error(error))
505                    }
506                })
507                .await?;
508            if output.is_some() {
509                last.store(sequence, std::sync::atomic::Ordering::Release);
510            }
511            Ok(output)
512        })));
513        Ok(None)
514    }
515}