#![allow(non_snake_case)]
use crate::hardware::{CameraSettings, CameraError};
use crate::hardware::camera::{WhiteBalanceGains, ExposureMode, FocusMode, WhiteBalanceMode, SceneMode};
use image::RgbaImage;
use std::sync::{Arc, Mutex};
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2::rc::{Retained, autoreleasepool};
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2::runtime::{ProtocolObject};
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2::{DeclaredClass, msg_send, class};
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2_foundation::{ NSArray, NSDictionary, NSNumber, NSString};
#[cfg(any(target_os = "ios", target_os = "macos"))]
use dispatch2::{DispatchRetained, DispatchQueue, GlobalQueueIdentifier, DispatchQueueGlobalPriority};
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2::__framework_prelude::NSObject;
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2_core_foundation::CGPoint;
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2_core_media::CMTime;
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2::ffi::nil;
#[cfg(any(target_os = "ios", target_os = "macos"))]
use std::thread;
#[cfg(target_os = "ios")]
use objc2_foundation::NSError;
#[cfg(target_os = "ios")]
use objc2_core_media::{CMSampleBuffer};
#[cfg(target_os = "ios")]
use objc2::{define_class, AllocAnyThread};
#[cfg(target_os = "ios")]
use objc2::runtime::{NSObjectProtocol};
#[cfg(target_os = "ios")]
use std::{slice::from_raw_parts};
#[cfg(target_os = "ios")]
use objc2_av_foundation::{
AVCaptureConnection,
AVCaptureOutput,
AVCaptureVideoDataOutputSampleBufferDelegate,
};
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2_av_foundation::{
AVCaptureVideoDataOutput,
AVCaptureDeviceDiscoverySession,
AVCaptureSession,
AVCaptureSessionPresetMedium,
AVMediaTypeVideo,
AVCaptureDeviceInput,
AVCaptureDevicePosition,
AVCaptureSessionPresetHigh,
AVCaptureSessionPresetPhoto,
AVCaptureDeviceTypeBuiltInWideAngleCamera,
AVCaptureDevice,
AVCaptureVideoStabilizationMode,
AVCaptureExposureMode,
AVCaptureFocusMode,
AVCaptureWhiteBalanceMode,
AVCaptureTorchMode,
AVCaptureWhiteBalanceGains,
};
#[cfg(target_os = "ios")]
use objc2_core_video::{
CVPixelBufferGetHeight,
CVPixelBufferGetWidth,
CVPixelBufferGetBytesPerRow,
CVPixelBufferGetBaseAddress,
CVPixelBufferLockFlags
};
#[cfg(any(target_os = "ios", target_os = "macos"))]
use objc2_core_video::{
kCVPixelBufferPixelFormatTypeKey,
kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
kCVPixelFormatType_14Bayer_GRBG,
kCVPixelFormatType_14Bayer_BGGR,
kCVPixelFormatType_14Bayer_GBRG,
kCVPixelFormatType_14Bayer_RGGB,
kCVPixelFormatType_32BGRA,
};
mod apple_custom_processor;
use apple_custom_processor::CustomProcessor;
mod apple_standard_processor;
use apple_standard_processor::StandardProcessor;
#[derive(Clone, Debug)]
pub enum AppleCamera {
Standard(StandardAppleCamera),
Unprocessed(UnprocessedAppleCamera)
}
impl AppleCamera {
pub fn new_standard() -> Result<Self, CameraError> {
Ok(AppleCamera::Standard(StandardAppleCamera::new()))
}
pub fn new_unprocessed() -> Result<Self, CameraError> {
Ok(AppleCamera::Unprocessed(UnprocessedAppleCamera::new()))
}
pub fn frame(&self) -> Result<RgbaImage, CameraError> {
match self {
AppleCamera::Standard(c) => c.frame().ok_or(CameraError::FailedToGetFrame),
AppleCamera::Unprocessed(c) => c.frame().ok_or(CameraError::FailedToGetFrame)
}
}
pub fn start(&mut self) {
let _ = match self {
AppleCamera::Standard(c) => c.start(),
AppleCamera::Unprocessed(c) => c.start()
};
}
pub fn toggle_flashlight(&self) {
}
pub fn settings(&self) -> Option<Arc<Mutex<CameraSettings>>> {
match self {
AppleCamera::Standard(_) => None,
AppleCamera::Unprocessed(c) => Some(c.settings())
}
}
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[derive(Debug, Clone)]
pub struct StandardAppleCamera {
pub session: Retained<AVCaptureSession>,
processor: Retained<StandardProcessor>,
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
impl StandardAppleCamera {
pub fn new() -> Self {
unsafe {
StandardAppleCamera {
session: AVCaptureSession::new(),
processor: StandardProcessor::new(),
}
}
}
pub fn start(&self) -> Result<(), String> {
unsafe {
let device_types = NSArray::from_slice(&[objc2_av_foundation::AVCaptureDeviceTypeBuiltInWideAngleCamera]);
let discovery_session = AVCaptureDeviceDiscoverySession::discoverySessionWithDeviceTypes_mediaType_position(
&device_types,
AVMediaTypeVideo,
AVCaptureDevicePosition::Back,
);
let devices = discovery_session.devices();
let device = devices.into_iter().next().expect("No device at index 0");
let input = AVCaptureDeviceInput::deviceInputWithDevice_error(&device)
.expect("Failed to create AVCaptureDeviceInput");
self.session.beginConfiguration();
self.session.setSessionPreset(AVCaptureSessionPresetMedium);
if self.session.canAddInput(&input) {
self.session.addInput(&input);
}
let output = AVCaptureVideoDataOutput::new();
let pixel_format_value = NSNumber::new_u32(kCVPixelFormatType_32BGRA);
let pixel_format_key: &NSString = &*(kCVPixelBufferPixelFormatTypeKey as *const _ as *const NSString);
let video_settings = NSDictionary::from_slices(
&[pixel_format_key],
&[pixel_format_value.as_ref()],
);
output.setVideoSettings(Some(&video_settings));
let queue = DispatchQueue::new("CameraQueue", None);
output.setSampleBufferDelegate_queue(
Some(ProtocolObject::from_ref(&*self.processor)),
Some(&queue),
);
if self.session.canAddOutput(&output) {
self.session.addOutput(&output);
}
self.session.commitConfiguration();
self.session.startRunning();
}
Ok(())
}
pub fn frame(&self) -> Option<RgbaImage> {
None
}
pub fn stop(&self) {}
}
impl Default for StandardAppleCamera {
fn default() -> Self { Self::new() }
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
#[derive(Debug, Clone)]
pub struct UnprocessedAppleCamera {
pub session: Retained<AVCaptureSession>,
processor: Retained<CustomProcessor>,
video_output: Option<Retained<AVCaptureVideoDataOutput>>,
device: Option<Retained<AVCaptureDevice>>,
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
impl UnprocessedAppleCamera {
pub fn new() -> Self {
Self {
session: unsafe { AVCaptureSession::new() },
processor: CustomProcessor::new(),
video_output: None,
device: None,
}
}
pub fn start(&mut self) -> Result<(), String> {
unsafe {
let device_types = NSArray::from_slice(&[AVCaptureDeviceTypeBuiltInWideAngleCamera]);
let discovery = AVCaptureDeviceDiscoverySession::discoverySessionWithDeviceTypes_mediaType_position(
&device_types, AVMediaTypeVideo, AVCaptureDevicePosition::Back);
let device = discovery.devices().into_iter().next().ok_or("No camera device")?;
self.device = Some(device.clone());
let input = AVCaptureDeviceInput::deviceInputWithDevice_error(&device)
.map_err(|e| format!("Input error: {e:?}"))?;
self.session.beginConfiguration();
for preset in [AVCaptureSessionPresetPhoto, AVCaptureSessionPresetHigh, AVCaptureSessionPresetMedium] {
if self.session.canSetSessionPreset(preset) {
self.session.setSessionPreset(preset);
break;
}
}
self.session.addInput(&input);
if device.lockForConfiguration().is_ok() {
device.unlockForConfiguration();
}
let output = AVCaptureVideoDataOutput::new();
let key = &*(kCVPixelBufferPixelFormatTypeKey as *const _ as *const NSString);
let supported = output.availableVideoCVPixelFormatTypes().iter().map(|f| f.unsignedIntValue()).collect::<Vec<_>>();
for f in [kCVPixelFormatType_14Bayer_RGGB, kCVPixelFormatType_14Bayer_BGGR, kCVPixelFormatType_14Bayer_GRBG,
kCVPixelFormatType_14Bayer_GBRG, kCVPixelFormatType_32BGRA, kCVPixelFormatType_420YpCbCr8BiPlanarFullRange,
kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] {
if supported.contains(&f) {
let settings = NSDictionary::from_slices(&[key], &[NSNumber::new_u32(f).as_ref()]);
let queue = DispatchQueue::new("CameraQueue", None);
output.setVideoSettings(Some(&settings));
output.setAlwaysDiscardsLateVideoFrames(true);
output.setSampleBufferDelegate_queue(Some(ProtocolObject::from_ref(&*self.processor)), Some(&queue));
self.session.addOutput(&output);
self.video_output = Some(output);
break;
}
}
self.session.commitConfiguration();
self.session.startRunning();
}
Ok(())
}
pub fn stop(&self) {
unsafe {
self.session.stopRunning();
*self.processor.ivars().last_raw_frame.lock().unwrap() = None;
*self.processor.ivars().bayer_format_verified.lock().unwrap() = false;
*self.processor.ivars().ready.lock().unwrap() = false;
}
}
pub fn frame(&self) -> Option<RgbaImage> {
let settings = self.settings().lock().unwrap().clone();
if settings.is_updated {
self.apply_settings(&settings);
self.settings().lock().as_mut().unwrap().is_updated = false;
}
if !self.processor.is_ready() { return None; }
self.processor.ivars().last_raw_frame.lock().unwrap().clone()
}
pub fn settings(&self) -> Arc<Mutex<CameraSettings>> {
self.processor.settings()
}
pub fn apply_settings(&self, settings: &CameraSettings) -> Result<(), CameraError> {
unsafe {
if let Some(device) = &self.device {
if !device.isExposureModeSupported(objc2_av_foundation::AVCaptureExposureMode::Custom) {
return Err(CameraError::FailedToGetFrame);
}
device.lockForConfiguration().map_err(|_| CameraError::FailedToGetFrame)?;
match settings.exposure_mode {
ExposureMode::Auto => {
device.setExposureMode(AVCaptureExposureMode::AutoExpose);
}
ExposureMode::Continuous => {
device.setExposureMode(AVCaptureExposureMode::ContinuousAutoExposure);
}
ExposureMode::Custom => {
if let Some(custom) = settings.custom_exposure {
unsafe {
let format_retained = device.activeFormat();
let format: &objc2::runtime::Object = format_retained.as_ref(); let min_d: objc2_core_media::CMTime = msg_send![format, minExposureDuration];
let max_d: objc2_core_media::CMTime = msg_send![format, maxExposureDuration];
let dur = (min_d.value as f64 / min_d.timescale as f64)
+ ((max_d.value as f64 / max_d.timescale as f64)
- (min_d.value as f64 / min_d.timescale as f64))
* custom.duration.clamp(0.0, 1.0) as f64;
let duration = objc2_core_media::CMTime {
value: (dur * 1_000_000_000.0) as i64,
timescale: 1_000_000_000,
flags: objc2_core_media::CMTimeFlags(0),
epoch: 0,
};
let min_iso = device.activeFormat().minISO();
let max_iso = device.activeFormat().maxISO();
let iso = (min_iso + (max_iso - min_iso) * custom.iso.clamp(0.0, 1.0)).clamp(min_iso, max_iso);
device.setExposureMode(objc2_av_foundation::AVCaptureExposureMode::Custom);
let () = msg_send![device, setExposureModeCustomWithDuration: duration ISO: iso completionHandler: nil];
}
}
}
}
match settings.focus_mode {
FocusMode::Auto => device.setFocusMode(AVCaptureFocusMode::AutoFocus),
FocusMode::Continuous => device.setFocusMode(AVCaptureFocusMode::ContinuousAutoFocus),
FocusMode::Locked => device.setFocusMode(AVCaptureFocusMode::Locked),
FocusMode::Manual => {
device.setFocusMode(AVCaptureFocusMode::Locked);
if let Some(pos) = settings.focus_distance {
let _: () = msg_send![
device,
setFocusModeLockedWithLensPosition: pos
completionHandler: core::ptr::null::<objc2::runtime::Object>()
];
}
}
}
match settings.white_balance_mode {
WhiteBalanceMode::Auto if device.isWhiteBalanceModeSupported(AVCaptureWhiteBalanceMode::AutoWhiteBalance) => {
device.setWhiteBalanceMode(AVCaptureWhiteBalanceMode::AutoWhiteBalance);
}
WhiteBalanceMode::Locked if device.isWhiteBalanceModeSupported(AVCaptureWhiteBalanceMode::Locked) => {
device.setWhiteBalanceMode(AVCaptureWhiteBalanceMode::Locked);
}
WhiteBalanceMode::Custom if device.isWhiteBalanceModeSupported(AVCaptureWhiteBalanceMode::Locked) => {
device.setWhiteBalanceMode(objc2_av_foundation::AVCaptureWhiteBalanceMode::Locked);
if let Some(gains) = &settings.white_balance_gains {
let max_gain = device.maxWhiteBalanceGain();
let wb_gains = objc2_av_foundation::AVCaptureWhiteBalanceGains {
redGain: 1.0 + (max_gain - 1.0) * gains.red.clamp(0.0, 1.0),
greenGain: 1.0 + (max_gain - 1.0) * gains.green.clamp(0.0, 1.0),
blueGain: 1.0 + (max_gain - 1.0) * gains.blue.clamp(0.0, 1.0),
};
let block = block2::ConcreteBlock::new(|_: *mut objc2::runtime::Object| {});
let _: () = msg_send![
device,
setWhiteBalanceModeLockedWithDeviceWhiteBalanceGains: wb_gains
completionHandler: &*block
];
}
}
_ => {}
}
device.unlockForConfiguration();
}
}
Ok(())
}
}
impl Default for UnprocessedAppleCamera {
fn default() -> Self { Self::new() }
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
pub trait DeviceWhiteBalanceGainsExt {
fn setWhiteBalanceModeLockedWithDeviceWhiteBalanceGains(&self, gains: WhiteBalanceGains);
}
#[cfg(any(target_os = "ios", target_os = "macos"))]
impl DeviceWhiteBalanceGainsExt for Retained<AVCaptureDevice> {
fn setWhiteBalanceModeLockedWithDeviceWhiteBalanceGains(&self, gains: WhiteBalanceGains) {
unsafe {
let device_gains: *mut AVCaptureWhiteBalanceGains = msg_send![
class!(AVCaptureWhiteBalanceGains),
deviceWhiteBalanceGainsWithRed: gains.red,
green: gains.green,
blue: gains.blue,
];
let _: () = msg_send![self, setWhiteBalanceModeLockedWithDeviceWhiteBalanceGains: device_gains];
}
}
}