Skip to main content

ferroui_media_camera_android/
lib.rs

1//! CameraX preview, consent, still capture and silent video recording.
2#![cfg(target_os = "android")]
3#![forbid(unsafe_code)]
4use ferroui_android::{interop::java::*, FerroActivity, FerroAndroidApplication};
5use ferroui_media::{frames::*, MediaError};
6#[cfg(feature = "controls")]
7pub use ferroui_media_android::NativeGlTextureImporter;
8use ferroui_media_android::{
9    poll_native, text, AndroidFrameStream, AndroidSurfaceSource, NativeGlFrameProducer,
10};
11use ferroui_media_camera::*;
12use std::{rc::Rc, task::Poll, time::Duration};
13fn error(kind: CameraErrorKind, message: impl Into<String>) -> CameraError {
14    CameraError::new(kind, message)
15}
16fn native(error: MediaError) -> CameraError {
17    CameraError::new(
18        match error.kind {
19            ferroui_media::MediaErrorKind::Unsupported => CameraErrorKind::Unsupported,
20            ferroui_media::MediaErrorKind::Cancelled => CameraErrorKind::Cancelled,
21            ferroui_media::MediaErrorKind::DeviceLost => CameraErrorKind::DeviceLost,
22            _ => CameraErrorKind::NativeFailure,
23        },
24        error.to_string(),
25    )
26}
27fn class() -> JavaClass {
28    JavaClass::find("org/ferroui/media/camera/CameraController")
29}
30fn json(text: &str) -> Result<serde_json::Value, CameraError> {
31    serde_json::from_str(text).map_err(|e| error(CameraErrorKind::NativeFailure, e.to_string()))
32}
33fn facing(value: u64) -> CameraFacing {
34    match value {
35        1 => CameraFacing::Front,
36        2 => CameraFacing::Rear,
37        3 => CameraFacing::External,
38        _ => CameraFacing::Unknown,
39    }
40}
41fn dimensions(value: &serde_json::Value) -> CameraFormat {
42    CameraFormat {
43        width: value["width"].as_u64().unwrap_or(0) as u32,
44        height: value["height"].as_u64().unwrap_or(0) as u32,
45        frames_per_second: value["fps"].as_f64().unwrap_or(0.),
46    }
47}
48/// Discovery can be used without graphics; starting requires an admitted active
49/// compositor context. No CPU fallback is declared by this provider.
50pub struct AndroidCameraProvider {
51    pub frames: Option<NativeGlFrameProducer>,
52    /// Explicit source contract for untagged SDR buffers only. Native color
53    /// tags take priority. None rejects missing metadata rather than guessing.
54    pub untagged_sdr_color: Option<FrameColor>,
55}
56/// Narrow native analysis extension. A clone keeps its bridge object alive,
57/// while stopping/dropping the originating CameraSession still closes capture.
58#[derive(Clone)]
59pub struct AndroidCameraContext {
60    java: JavaObject,
61    source: u64,
62    generation: u64,
63    counters: FrameTransferCounters,
64    untagged_sdr_color: Option<FrameColor>,
65}
66impl AndroidCameraContext {
67    pub fn java(&self) -> JavaObject {
68        self.java.clone()
69    }
70    pub fn source(&self) -> u64 {
71        self.source
72    }
73    pub fn generation(&self) -> u64 {
74        self.generation
75    }
76    pub fn declared_untagged_color(&self) -> Option<FrameColor> {
77        self.untagged_sdr_color
78    }
79    pub fn counters(&self) -> FrameTransferCounters {
80        self.counters.clone()
81    }
82}
83impl CameraProvider for AndroidCameraProvider {
84    fn devices(&self) -> CameraFuture<Vec<CameraDevice>> {
85        let result = call_static_object(
86            &class(),
87            "devices",
88            "(Landroid/content/Context;)Ljava/lang/String;",
89            &[JavaValue::Object(Some(&FerroAndroidApplication::context()))],
90        )
91        .map(|v| string_of(&v))
92        .ok_or_else(|| error(CameraErrorKind::NativeFailure, "No discovery result"));
93        Box::pin(async move {
94            let value = json(&result?)?;
95            let values = value
96                .as_array()
97                .ok_or_else(|| error(CameraErrorKind::NativeFailure, "Camera discovery failed"))?;
98            Ok(values
99                .iter()
100                .map(|v| CameraDevice {
101                    id: v["id"].as_str().unwrap_or_default().into(),
102                    label: None,
103                    facing: facing(v["facing"].as_u64().unwrap_or(0)),
104                    formats: v["formats"]
105                        .as_array()
106                        .map(|v| v.iter().map(dimensions).collect())
107                        .unwrap_or_default(),
108                })
109                .collect())
110        })
111    }
112    fn permission(&self) -> CameraPermission {
113        match call_static_int(
114            &class(),
115            "permission",
116            "(Landroid/content/Context;)I",
117            &[JavaValue::Object(Some(&FerroAndroidApplication::context()))],
118        ) {
119            1 => CameraPermission::Granted,
120            2 => CameraPermission::Denied,
121            _ => CameraPermission::NotDetermined,
122        }
123    }
124    fn start(
125        &self,
126        options: CameraStartOptions,
127        id: u64,
128        generation: u64,
129        budget: FrameBudget,
130    ) -> CameraFuture<Rc<dyn NativeCameraSession>> {
131        let Some(frames) = self.frames.as_ref().map(|v| v.with_budget(budget.clone())) else {
132            return Box::pin(async {
133                Err(error(
134                    CameraErrorKind::Unsupported,
135                    "Camera preview needs an admitted GL context",
136                ))
137            });
138        };
139        let Some(activity) =
140            FerroActivity::current_main_activity().map(|v| v.java_object().clone())
141        else {
142            return Box::pin(async {
143                Err(error(
144                    CameraErrorKind::Disconnected,
145                    "No active Android activity",
146                ))
147            });
148        };
149        let untagged_sdr_color = self.untagged_sdr_color;
150        let color_profile = match untagged_sdr_color {
151            None => 0,
152            Some(FrameColor::Srgb) => 1,
153            Some(FrameColor::Rec709Full) => 2,
154            _ => {
155                return Box::pin(async {
156                    Err(error(
157                        CameraErrorKind::Unsupported,
158                        "Unsupported declared camera color profile",
159                    ))
160                })
161            }
162        };
163        let selection = options.selection;
164        if selection.exact && selection.format.is_some() {
165            return Box::pin(async {
166                Err(error(
167                    CameraErrorKind::Unsupported,
168                    "Exact FPS/size negotiation is not admitted on this CameraX profile",
169                ))
170            });
171        }
172        let class = class();
173        Box::pin(async move {
174            let source = AndroidSurfaceSource::create(&frames)
175                .await
176                .map_err(native)?;
177            let format = selection.format;
178            let java = new_object(
179                &class,
180                "(Landroid/app/Activity;Lorg/ferroui/media/SurfaceFrames;Ljava/lang/String;IIIZI)V",
181                &[
182                    JavaValue::Object(Some(&activity)),
183                    JavaValue::Object(Some(&source.java())),
184                    JavaValue::String(selection.device_id.as_deref().unwrap_or("")),
185                    JavaValue::Int(match selection.preferred_facing {
186                        Some(CameraFacing::Front) => 1,
187                        Some(CameraFacing::Rear) => 2,
188                        _ => 0,
189                    }),
190                    JavaValue::Int(format.map(|v| v.width as i32).unwrap_or(0)),
191                    JavaValue::Int(format.map(|v| v.height as i32).unwrap_or(0)),
192                    JavaValue::Boolean(selection.exact),
193                    JavaValue::Int(color_profile),
194                ],
195            )
196            .to_global();
197            let mut session = Some(Session {
198                context: AndroidCameraContext {
199                    java: java.clone(),
200                    source: id,
201                    generation,
202                    counters: budget.counters(),
203                    untagged_sdr_color,
204                },
205                java: java.clone(),
206                stream: AndroidFrameStream::new(frames, source, id, generation)
207                    .with_capture_clock(java),
208                budget,
209            });
210            poll_native(move || {
211                let value = session.as_ref().unwrap();
212                if let Err(error) = value.check() {
213                    return Poll::Ready(Err(error));
214                }
215                if !call_boolean(&value.java, "ready", "()Z", &[]) {
216                    return Poll::Pending;
217                }
218                Poll::Ready(Ok(
219                    Rc::new(session.take().unwrap()) as Rc<dyn NativeCameraSession>
220                ))
221            })
222            .await
223        })
224    }
225}
226struct Session {
227    context: AndroidCameraContext,
228    java: JavaObject,
229    stream: AndroidFrameStream,
230    budget: FrameBudget,
231}
232impl Session {
233    fn check(&self) -> Result<(), CameraError> {
234        if call_boolean(&self.java, "closed", "()Z", &[]) {
235            return Err(error(
236                CameraErrorKind::Disconnected,
237                "Camera activity/session closed",
238            ));
239        }
240        let message = text(&self.java, "error");
241        if message.is_empty() {
242            return Ok(());
243        }
244        Err(error(
245            match call_int(&self.java, "errorKind", "()I", &[]) {
246                4 => CameraErrorKind::Denied,
247                6 => CameraErrorKind::Disconnected,
248                7 => CameraErrorKind::Unsupported,
249                _ => CameraErrorKind::NativeFailure,
250            },
251            message,
252        ))
253    }
254    fn info(&self) -> serde_json::Value {
255        json(&text(&self.java, "settings")).unwrap_or_default()
256    }
257}
258impl Drop for Session {
259    fn drop(&mut self) {
260        self.stop();
261    }
262}
263impl NativeCameraSession for Session {
264    fn platform_context(&self) -> Option<&dyn std::any::Any> {
265        Some(&self.context)
266    }
267    fn settings(&self) -> CameraSettings {
268        let info = self.info();
269        let format = dimensions(&info);
270        CameraSettings {
271            device: CameraDevice {
272                id: info["id"].as_str().unwrap_or_default().into(),
273                label: None,
274                facing: facing(info["facing"].as_u64().unwrap_or(0)),
275                formats: vec![format],
276            },
277            format,
278            native_still: true,
279            torch: info["torch"].as_bool().unwrap_or(false),
280            zoom: info["minZoom"].as_f64().zip(info["maxZoom"].as_f64()),
281        }
282    }
283    fn take_frame(&self) -> Result<Option<FrameLease>, CameraError> {
284        self.check()?;
285        let info = self.info();
286        let size = dimensions(&info);
287        let rotation = match info["rotation"].as_u64() {
288            Some(90) => FrameRotation::Clockwise90,
289            Some(180) => FrameRotation::Clockwise180,
290            Some(270) => FrameRotation::Clockwise270,
291            _ => FrameRotation::None,
292        };
293        self.stream
294            .take_oriented_frame(
295                size.width,
296                size.height,
297                rotation,
298                info["mirrored"].as_bool().unwrap_or(false),
299            )
300            .map_err(native)
301    }
302    fn photo(&self) -> CameraFuture<CameraPhoto> {
303        let Some(java) = call_object(
304            &self.java,
305            "capture",
306            "()Lorg/ferroui/media/camera/CapturedPhoto;",
307            &[],
308        )
309        .map(|v| v.to_global()) else {
310            return Box::pin(async {
311                Err(error(
312                    CameraErrorKind::NativeFailure,
313                    "No native photo request",
314                ))
315            });
316        };
317        struct Request {
318            java: JavaObject,
319        }
320        impl Drop for Request {
321            fn drop(&mut self) {
322                call_void(&self.java, "abandon", "()V", &[]);
323            }
324        }
325        let request = Request { java };
326        let counters = self.budget.counters();
327        let mut copied: Option<
328            std::sync::Arc<std::sync::Mutex<Option<Result<CameraPhoto, CameraError>>>>,
329        > = None;
330        poll_native(move || {
331            if let Some(copied) = copied.as_ref() {
332                return copied
333                    .lock()
334                    .unwrap()
335                    .take()
336                    .map(Poll::Ready)
337                    .unwrap_or(Poll::Pending);
338            }
339            let java = &request.java;
340            if call_boolean(java, "pending", "()Z", &[]) {
341                return Poll::Pending;
342            }
343            let failure = text(java, "error");
344            if !failure.is_empty() {
345                return Poll::Ready(Err(error(CameraErrorKind::NativeFailure, failure)));
346            }
347            let length = call_int(java, "length", "()I", &[]);
348            if length <= 0 || length > 64 * 1024 * 1024 {
349                return Poll::Ready(Err(error(
350                    CameraErrorKind::Cancelled,
351                    "Photo unavailable or exceeds limit",
352                )));
353            }
354            let info = text(java, "info")
355                .split(',')
356                .map(|v| v.parse::<u64>().unwrap_or(0))
357                .collect::<Vec<_>>();
358            let java = java.clone();
359            let counters = counters.clone();
360            let result = std::sync::Arc::new(std::sync::Mutex::new(None));
361            let output = result.clone();
362            let spawned = std::thread::Builder::new()
363                .name("FerroUI.Media.Camera.Photo".into())
364                .spawn(move || {
365                    let value = (|| {
366                        let Some(bytes) = call_object(&java, "takeBytes", "()[B", &[]) else {
367                            return Err(error(CameraErrorKind::Cancelled, "Photo canceled"));
368                        };
369                        if let Err(e) = counters.cpu_transfer(
370                            PixelTransferPurpose::Photo,
371                            length as u64,
372                            PresentationRoute::SharedTexture,
373                        ) {
374                            return Err(error(CameraErrorKind::NativeFailure, e.to_string()));
375                        }
376                        let Some(bytes) = bounded_byte_array_of(&bytes, 64 * 1024 * 1024) else {
377                            return Err(error(
378                                CameraErrorKind::NativeFailure,
379                                "Photo exceeds limit",
380                            ));
381                        };
382                        Ok(CameraPhoto {
383                            encoded: bytes.into(),
384                            content_type: "image/jpeg".into(),
385                            width: info.first().copied().unwrap_or(0) as u32,
386                            height: info.get(1).copied().unwrap_or(0) as u32,
387                            rotation_degrees: info.get(2).copied().unwrap_or(0) as u16,
388                            timestamp: Duration::from_nanos(info.get(3).copied().unwrap_or(0)),
389                            mirrored: false,
390                            origin: PhotoOrigin::NativeStill,
391                        })
392                    })();
393                    *output.lock().unwrap() = Some(value);
394                });
395            if let Err(failure) = spawned {
396                return Poll::Ready(Err(error(
397                    CameraErrorKind::NativeFailure,
398                    failure.to_string(),
399                )));
400            }
401            copied = Some(result);
402            Poll::Pending
403        })
404    }
405    fn supports_video_recording(&self) -> bool {
406        call_boolean(&self.java, "supportsRecording", "()Z", &[])
407    }
408    fn start_recording(&self) -> CameraFuture<Rc<dyn NativeCameraRecording>> {
409        let Some(java) = call_object(
410            &self.java,
411            "startRecording",
412            "()Lorg/ferroui/media/camera/CameraRecordingRequest;",
413            &[],
414        )
415        .map(|request| request.to_global()) else {
416            return Box::pin(async {
417                Err(error(
418                    CameraErrorKind::NativeFailure,
419                    "No native recording request",
420                ))
421            });
422        };
423        let recording = Rc::new(AndroidRecording(java));
424        poll_native(move || {
425            if call_boolean(&recording.0, "startPending", "()Z", &[]) {
426                return Poll::Pending;
427            }
428            let failure = text(&recording.0, "error");
429            if !failure.is_empty() {
430                return Poll::Ready(Err(recording_error(failure)));
431            }
432            Poll::Ready(Ok(recording.clone() as Rc<dyn NativeCameraRecording>))
433        })
434    }
435    fn set_control(&self, control: CameraControl) -> Result<(), CameraError> {
436        self.check()?;
437        let (kind, value) = match control {
438            CameraControl::Torch(value) => (0, if value { 1. } else { 0. }),
439            CameraControl::Zoom(value) => (1, value as f32),
440            _ => {
441                return Err(error(
442                    CameraErrorKind::Unsupported,
443                    "Camera control not implemented",
444                ))
445            }
446        };
447        let message = call_object(
448            &self.java,
449            "control",
450            "(IF)Ljava/lang/String;",
451            &[JavaValue::Int(kind), JavaValue::Float(value)],
452        )
453        .map(|v| string_of(&v))
454        .unwrap_or_default();
455        if message.is_empty() {
456            Ok(())
457        } else {
458            Err(error(CameraErrorKind::Unsupported, message))
459        }
460    }
461    fn stop(&self) {
462        self.stream.stop();
463        call_void(&self.java, "close", "()V", &[]);
464    }
465}
466
467struct AndroidRecording(JavaObject);
468
469fn recording_error(message: String) -> CameraError {
470    let kind = if message == "Recording cancelled" {
471        CameraErrorKind::Cancelled
472    } else {
473        CameraErrorKind::NativeFailure
474    };
475    error(kind, message)
476}
477
478impl NativeCameraRecording for AndroidRecording {
479    fn finish(&self) -> CameraFuture<CameraVideo> {
480        const MAX_BYTES: usize = 256 * 1024 * 1024;
481        call_void(&self.0, "finish", "()V", &[]);
482        let java = self.0.clone();
483        let mut worker = None::<std::thread::JoinHandle<Result<CameraVideo, CameraError>>>;
484        poll_native(move || {
485            if worker.as_ref().is_some_and(|worker| worker.is_finished()) {
486                return Poll::Ready(worker.take().unwrap().join().unwrap_or_else(|_| {
487                    Err(error(
488                        CameraErrorKind::NativeFailure,
489                        "Recording copy worker failed",
490                    ))
491                }));
492            }
493            if worker.is_some() || call_boolean(&java, "pending", "()Z", &[]) {
494                return Poll::Pending;
495            }
496            let failure = text(&java, "error");
497            if !failure.is_empty() {
498                return Poll::Ready(Err(recording_error(failure)));
499            }
500            let java = java.clone();
501            match std::thread::Builder::new()
502                .name("FerroUI.Media.Camera.Recording".into())
503                .spawn(move || {
504                    let Some(bytes) = call_object(&java, "takeBytes", "()[B", &[]) else {
505                        let failure = text(&java, "error");
506                        return Err(recording_error(if failure.is_empty() {
507                            "Recording output unavailable".into()
508                        } else {
509                            failure
510                        }));
511                    };
512                    let encoded = bounded_byte_array_of(&bytes, MAX_BYTES).ok_or_else(|| {
513                        error(
514                            CameraErrorKind::NativeFailure,
515                            "Recording exceeds size limit",
516                        )
517                    })?;
518                    let info = text(&java, "info");
519                    let mut fields = info.split(',');
520                    let width = fields.next().and_then(|value| value.parse().ok());
521                    let height = fields.next().and_then(|value| value.parse().ok());
522                    let duration = fields.next().and_then(|value| value.parse().ok());
523                    let (Some(width), Some(height), Some(duration)) = (width, height, duration)
524                    else {
525                        return Err(error(
526                            CameraErrorKind::NativeFailure,
527                            "Invalid recording metadata",
528                        ));
529                    };
530                    Ok(CameraVideo {
531                        encoded: encoded.into(),
532                        content_type: "video/mp4".into(),
533                        width,
534                        height,
535                        duration: Duration::from_micros(duration),
536                    })
537                }) {
538                Ok(thread) => {
539                    worker = Some(thread);
540                    Poll::Pending
541                }
542                Err(failure) => Poll::Ready(Err(error(
543                    CameraErrorKind::NativeFailure,
544                    failure.to_string(),
545                ))),
546            }
547        })
548    }
549
550    fn cancel(&self) {
551        call_void(&self.0, "cancel", "()V", &[]);
552    }
553}
554
555impl Drop for AndroidRecording {
556    fn drop(&mut self) {
557        self.cancel();
558    }
559}