1#![cfg(target_os = "android")]
3#![forbid(unsafe_code)]
4use ferroui_android::interop::java::*;
5use ferroui_base::{
6 reactive::Subscription,
7 threading::{DispatcherPriority, DispatcherTimer},
8};
9use ferroui_media::{frames::*, *};
10pub use ferroui_media_opengl::NativeGlFrameProducer;
11#[cfg(feature = "controls")]
12pub use ferroui_media_opengl::NativeGlTextureImporter;
13use ferroui_media_opengl::ReadyGlFrame;
14use ferroui_opengl::SharedGlTextureRetirementQueue;
15use std::{
16 cell::{Cell, RefCell},
17 rc::Rc,
18 sync::Arc,
19 task::{Context, Poll, Waker},
20 time::Duration,
21};
22
23pub fn gpu_quadrants(
25 producer: &NativeGlFrameProducer,
26 identity: FrameIdentity,
27 width: u32,
28 height: u32,
29) -> MediaFuture<Option<ReadyGlFrame>> {
30 let class = JavaClass::find("org/ferroui/media/SurfaceFrames");
31 let bytes = width as u64 * height as u64 * 4;
32 let metadata = FrameMetadata {
33 identity,
34 width,
35 height,
36 visible: FrameRegion {
37 x: 0,
38 y: 0,
39 width,
40 height,
41 },
42 format: FrameFormat::Rgba8,
43 color: FrameColor::Srgb,
44 alpha: FrameAlpha::Opaque,
45 rotation: FrameRotation::None,
46 mirrored: false,
47 top_left_origin: false,
48 timestamp: Duration::ZERO,
49 device: Some(producer.device()),
50 device_generation: producer.device_generation(),
51 planes: vec![FramePlane {
52 offset: 0,
53 stride: width as u64 * 4,
54 length: bytes,
55 }],
56 retained_bytes: bytes,
57 };
58 producer.produce(metadata, move |_, output| {
59 let error = call_static_object(
60 &class,
61 "quadrants",
62 "(III)Ljava/lang/String;",
63 &[
64 JavaValue::Int(output as i32),
65 JavaValue::Int(width as i32),
66 JavaValue::Int(height as i32),
67 ],
68 )
69 .map(|v| string_of(&v))
70 .unwrap_or_else(|| "No independent producer result".into());
71 if error.is_empty() {
72 Ok(())
73 } else {
74 Err(native_error(error))
75 }
76 })
77}
78
79pub fn native_error(message: impl Into<String>) -> MediaError {
80 MediaError::new(MediaErrorKind::NativeFailure, message)
81}
82pub fn live_surface_sources() -> u32 {
85 call_static_int(
86 &JavaClass::find("org/ferroui/media/SurfaceFrames"),
87 "liveSources",
88 "()I",
89 &[],
90 )
91 .max(0) as u32
92}
93pub fn poll_native<T: 'static, E: 'static>(
95 mut read: impl FnMut() -> Poll<Result<T, E>> + 'static,
96) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<T, E>>>> {
97 let mut timer: Option<Subscription> = None;
98 Box::pin(std::future::poll_fn(move |cx| {
99 timer.take();
100 match read() {
101 Poll::Ready(value) => Poll::Ready(value),
102 Poll::Pending => {
103 let waker = cx.waker().clone();
104 timer = Some(DispatcherTimer::run_once(
105 move || waker.wake_by_ref(),
106 Duration::from_millis(16),
107 DispatcherPriority::NORMAL,
108 ));
109 Poll::Pending
110 }
111 }
112 }))
113}
114pub fn text(object: &JavaObject, method: &str) -> String {
115 call_object(object, method, "()Ljava/lang/String;", &[])
116 .map(|s| string_of(&s))
117 .unwrap_or_default()
118}
119struct SourceOwner {
120 java: JavaObject,
121 retirement: SharedGlTextureRetirementQueue,
122 _wake: Arc<dyn Fn() + Send + Sync>,
123}
124impl Drop for SourceOwner {
125 fn drop(&mut self) {
126 let java = self.java.clone();
127 self.retirement.defer_native(move |context| {
128 call_boolean(
129 &java,
130 "release",
131 "(Z)Z",
132 &[JavaValue::Boolean(context.is_some())],
133 )
134 });
135 }
136}
137#[derive(Clone)]
139pub struct AndroidSurfaceSource(Arc<SourceOwner>);
140impl AndroidSurfaceSource {
141 pub fn create(producer: &NativeGlFrameProducer) -> MediaFuture<Self> {
142 let class = JavaClass::find("org/ferroui/media/SurfaceFrames");
145 let wake = producer.retirement_wakeup();
146 producer.invoke(move |context| {
147 let java = new_object(&class, "()V", &[]).to_global();
148 let error = text(&java, "error");
149 if !error.is_empty() {
150 return Err(native_error(error));
151 }
152 let retirement = context
153 .shared_frame_retirement_queue()
154 .ok_or_else(|| native_error("No native GL retirement"))?;
155 let wake = retirement.register_wakeup(wake);
156 Ok(Self(Arc::new(SourceOwner {
157 java,
158 retirement,
159 _wake: wake,
160 })))
161 })
162 }
163 pub fn java(&self) -> JavaObject {
164 self.0.java.clone()
165 }
166 pub fn surface(&self) -> Result<JavaObject, MediaError> {
167 call_object(&self.0.java, "surface", "()Landroid/view/Surface;", &[])
168 .map(|v| v.to_global())
169 .ok_or_else(|| native_error("Surface source was released"))
170 }
171}
172pub struct AndroidFrameStream {
173 producer: NativeGlFrameProducer,
174 source: RefCell<Option<AndroidSurfaceSource>>,
175 identity: FrameIdentity,
176 last_sequence: Arc<std::sync::atomic::AtomicU64>,
177 job: RefCell<Option<MediaFuture<Option<ReadyGlFrame>>>>,
178 pending: RefCell<Option<ReadyGlFrame>>,
179 stopped: Cell<bool>,
180 epoch: Arc<std::sync::atomic::AtomicU64>,
181 job_epoch: Rc<Cell<u64>>,
182 frame_clock: Option<JavaObject>,
183 capture_identity: bool,
184 clock_nanoseconds: bool,
185}
186impl AndroidFrameStream {
187 pub fn new(
188 producer: NativeGlFrameProducer,
189 source: AndroidSurfaceSource,
190 id: u64,
191 generation: u64,
192 ) -> Self {
193 Self {
194 producer,
195 source: RefCell::new(Some(source)),
196 identity: FrameIdentity {
197 source: id,
198 generation,
199 sequence: 0,
200 },
201 last_sequence: Arc::new(std::sync::atomic::AtomicU64::new(0)),
202 job: RefCell::new(None),
203 pending: RefCell::new(None),
204 stopped: Cell::new(false),
205 epoch: Arc::new(std::sync::atomic::AtomicU64::new(0)),
206 job_epoch: Rc::new(Cell::new(0)),
207 frame_clock: None,
208 capture_identity: false,
209 clock_nanoseconds: false,
210 }
211 }
212 pub fn with_frame_clock(mut self, clock: JavaObject) -> Self {
215 self.frame_clock = Some(clock);
216 self
217 }
218 pub fn with_capture_identity(mut self) -> Self {
221 self.capture_identity = true;
222 self
223 }
224 pub fn with_capture_clock(mut self, clock: JavaObject) -> Self {
227 self.frame_clock = Some(clock);
228 self.clock_nanoseconds = true;
229 self.capture_identity = true;
230 self
231 }
232 pub fn clear(&self) {
235 self.epoch.fetch_add(1, std::sync::atomic::Ordering::AcqRel);
236 }
237 pub fn stop(&self) {
238 self.stopped.set(true);
239 self.clear();
240 self.job.borrow_mut().take();
241 self.pending.borrow_mut().take();
242 self.source.borrow_mut().take();
243 }
244 pub fn has_pending_frame_work(&self) -> bool {
245 !self.stopped.get()
246 && (self.job.borrow().is_some()
247 || self.pending.borrow().is_some()
248 || self
249 .source
250 .borrow()
251 .as_ref()
252 .is_some_and(|source| call_boolean(&source.0.java, "pending", "()Z", &[])))
253 }
254 pub fn take_frame(&self, width: u32, height: u32) -> Result<Option<FrameLease>, MediaError> {
255 self.take_oriented_frame(width, height, FrameRotation::None, false)
256 }
257 pub fn take_oriented_frame(
258 &self,
259 width: u32,
260 height: u32,
261 rotation: FrameRotation,
262 mirrored: bool,
263 ) -> Result<Option<FrameLease>, MediaError> {
264 if self.stopped.get() || width == 0 || height == 0 {
265 return Ok(None);
266 }
267 let ready = self
268 .pending
269 .borrow()
270 .as_ref()
271 .map(ReadyGlFrame::take_if_ready);
272 if let Some(ready) = ready {
273 let frame = ready?;
274 if frame.is_none() {
275 return Ok(None);
276 }
277 self.pending.replace(None);
278 return Ok(
279 if self.epoch.load(std::sync::atomic::Ordering::Acquire) == self.job_epoch.get() {
280 frame
281 } else {
282 None
283 },
284 );
285 }
286 let result = {
287 let mut job = self.job.borrow_mut();
288 job.as_mut()
289 .map(|job| job.as_mut().poll(&mut Context::from_waker(Waker::noop())))
290 };
291 if let Some(result) = result {
292 match result {
293 Poll::Pending => return Ok(None),
294 Poll::Ready(result) => {
295 self.job.borrow_mut().take();
296 self.pending.replace(result?);
297 return Ok(None);
298 }
299 }
300 }
301 let Some(source) = self.source.borrow().clone() else {
302 return Ok(None);
303 };
304 if !call_boolean(&source.0.java, "pending", "()Z", &[]) {
305 return Ok(None);
306 }
307 let producer = self.producer.clone();
308 let last = self.last_sequence.clone();
309 let mut identity = self.identity;
310 let frame_clock = self.frame_clock.clone();
311 let capture_identity = self.capture_identity;
312 let clock_nanoseconds = self.clock_nanoseconds;
313 let requested_epoch = self.epoch.load(std::sync::atomic::Ordering::Acquire);
314 self.job_epoch.set(requested_epoch);
315 let epoch = self.epoch.clone();
316 let job_epoch = self.job_epoch.clone();
317 self.job.replace(Some(Box::pin(async move {
318 let native = source.clone();
319 let acquisition_epoch = epoch.clone();
320 let acquired = producer
321 .invoke(move |_| {
322 if acquisition_epoch.load(std::sync::atomic::Ordering::Acquire)
325 != requested_epoch
326 {
327 return Ok(None);
328 }
329 if !call_boolean(&native.0.java, "pending", "()Z", &[]) {
330 return Ok(None);
331 }
332 let error = text(&native.0.java, "acquire");
333 if !error.is_empty() {
334 return Err(if call_boolean(&native.0.java, "unsupported", "()Z", &[]) {
335 MediaError::new(MediaErrorKind::Unsupported, error)
336 } else {
337 native_error(error)
338 });
339 }
340 Ok(Some((
341 call_long(&native.0.java, "sequence", "()J", &[]) as u64,
342 call_long(&native.0.java, "timestamp", "()J", &[]).max(0) as u64,
343 call_boolean(&native.0.java, "swapsAxes", "()Z", &[]),
344 )))
345 })
346 .await?;
347 let Some((sequence, timestamp, swaps_axes)) = acquired else {
348 return Ok(None);
349 };
350 if sequence <= last.load(std::sync::atomic::Ordering::Acquire) {
351 return Ok(None);
352 }
353 identity.sequence = if capture_identity {
354 timestamp
355 } else {
356 sequence
357 };
358 let (timestamp, width, height, color, rotation, mirrored, visible) =
359 if let Some(clock) = frame_clock {
360 let value = call_object(
361 &clock,
362 "frameInfo",
363 "(J)Ljava/lang/String;",
364 &[JavaValue::Long(timestamp as i64)],
365 )
366 .map(|v| string_of(&v))
367 .unwrap_or_default();
368 if value.is_empty() {
369 return Ok(None);
370 }
371 let values = value
372 .split(',')
373 .map(str::parse::<u64>)
374 .collect::<Result<Vec<_>, _>>()
375 .map_err(|_| native_error("Invalid native frame-clock metadata"))?;
376 if values.len() != if clock_nanoseconds { 6 } else { 10 }
377 || values[1] > u32::MAX as u64
378 || values[2] > u32::MAX as u64
379 {
380 return Err(native_error("Invalid native frame dimensions"));
381 }
382 (
383 values[0]
384 .checked_mul(if clock_nanoseconds { 1 } else { 1000 })
385 .ok_or_else(|| native_error("Native timestamp overflow"))?,
386 values[1] as u32,
387 values[2] as u32,
388 match values[3] {
389 1 => FrameColor::Srgb,
390 2 => FrameColor::Rec709Full,
391 _ => return Err(native_error("Unsupported native video color")),
392 },
393 {
394 match values[4] {
395 0 => FrameRotation::None,
396 90 => FrameRotation::Clockwise90,
397 180 => FrameRotation::Clockwise180,
398 270 => FrameRotation::Clockwise270,
399 _ => {
400 return Err(native_error(
401 "Unsupported native frame orientation",
402 ))
403 }
404 }
405 },
406 {
407 match values[5] {
408 0 => false,
409 1 => true,
410 _ => return Err(native_error("Invalid native mirror flag")),
411 }
412 },
413 if clock_nanoseconds {
414 None
415 } else {
416 if values[6..].iter().any(|v| *v > u32::MAX as u64) {
417 return Err(native_error("Native crop overflow"));
418 }
419 Some(FrameRegion {
420 x: values[6] as u32,
421 y: values[7] as u32,
422 width: values[8] as u32,
423 height: values[9] as u32,
424 })
425 },
426 )
427 } else {
428 if epoch.load(std::sync::atomic::Ordering::Acquire) != requested_epoch {
429 return Ok(None);
430 }
431 (
432 timestamp,
433 width,
434 height,
435 FrameColor::Srgb,
436 rotation,
437 mirrored,
438 None,
439 )
440 };
441 job_epoch.set(epoch.load(std::sync::atomic::Ordering::Acquire));
446 let (width, height) = if swaps_axes {
449 (height, width)
450 } else {
451 (width, height)
452 };
453 let bytes = width as u64 * height as u64 * 4;
454 let metadata = FrameMetadata {
455 identity,
456 width,
457 height,
458 visible: visible.unwrap_or(FrameRegion {
459 x: 0,
460 y: 0,
461 width,
462 height,
463 }),
464 format: FrameFormat::Rgba8,
465 color,
466 alpha: FrameAlpha::Opaque,
467 rotation,
468 mirrored,
469 top_left_origin: false,
470 timestamp: Duration::from_nanos(timestamp),
471 device: Some(producer.device()),
472 device_generation: producer.device_generation(),
473 planes: vec![FramePlane {
474 offset: 0,
475 stride: width as u64 * 4,
476 length: bytes,
477 }],
478 retained_bytes: bytes,
479 };
480 let normalize_color = call_boolean(&source.0.java, "colorConversion", "()Z", &[]);
481 let counters = producer.budget().counters();
482 let owner = source.clone();
483 let output = producer
484 .produce_owned(metadata, owner, move |_, output| {
485 counters.gpu_conversion(GpuTransferReason::PixelFormatConversion);
486 if normalize_color {
487 counters.gpu_conversion(GpuTransferReason::ColorSpaceConversion);
488 }
489 let error = call_object(
490 &source.0.java,
491 "draw",
492 "(III)Ljava/lang/String;",
493 &[
494 JavaValue::Int(output as i32),
495 JavaValue::Int(width as i32),
496 JavaValue::Int(height as i32),
497 ],
498 )
499 .map(|v| string_of(&v))
500 .unwrap_or_default();
501 if error.is_empty() {
502 Ok(())
503 } else {
504 Err(native_error(error))
505 }
506 })
507 .await?;
508 if output.is_some() {
509 last.store(sequence, std::sync::atomic::Ordering::Release);
510 }
511 Ok(output)
512 })));
513 Ok(None)
514 }
515}