use anyhow::Result;
use objc2::rc::Retained;
use objc2::runtime::Bool;
use objc2::{class, msg_send};
use objc2_av_foundation::{
AVCaptureDevice, AVCaptureDeviceInput, AVCaptureSession, AVCaptureVideoDataOutput,
AVMediaTypeVideo,
};
use objc2_foundation::{NSNumber, NSObject, NSString};
use std::path::PathBuf;
use std::sync::mpsc;
use crate::frame::Frame;
#[derive(Debug, Clone)]
pub struct RawFrame {
pub width: u32,
pub height: u32,
pub data: Vec<u8>,
pub format: RawFormat,
pub timestamp_us: u64,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum RawFormat {
Bgra,
}
pub struct RetainedPixelBuffer {
ptr: *const std::ffi::c_void,
pub width: u32,
pub height: u32,
pub bytes_per_row: usize,
pub timestamp_us: u64,
}
unsafe impl Send for RetainedPixelBuffer {}
impl RetainedPixelBuffer {
pub fn as_ptr(&self) -> *const std::ffi::c_void {
self.ptr
}
}
impl Drop for RetainedPixelBuffer {
fn drop(&mut self) {
if !self.ptr.is_null() {
unsafe { CFRelease(self.ptr) };
}
}
}
#[derive(Debug, Clone)]
pub struct CameraInfo {
pub index: u32,
pub name: String,
pub path: PathBuf,
}
#[derive(Debug, Clone, Default)]
pub struct CameraOptions {
pub prefer_yuyv: bool,
}
pub struct Camera {
width: u32,
height: u32,
rx: mpsc::Receiver<RetainedPixelBuffer>,
start_time: std::time::Instant,
_session: Retained<AVCaptureSession>,
_delegate: Retained<NSObject>,
}
unsafe impl Send for Camera {}
#[link(name = "CoreFoundation", kind = "framework")]
extern "C" {
fn CFRetain(cf: *const std::ffi::c_void) -> *const std::ffi::c_void;
fn CFRelease(cf: *const std::ffi::c_void);
}
#[link(name = "CoreVideo", kind = "framework")]
extern "C" {
fn CVPixelBufferLockBaseAddress(pixel_buffer: *const std::ffi::c_void, flags: u64) -> i32;
fn CVPixelBufferUnlockBaseAddress(pixel_buffer: *const std::ffi::c_void, flags: u64) -> i32;
fn CVPixelBufferGetBaseAddress(pixel_buffer: *const std::ffi::c_void) -> *const u8;
fn CVPixelBufferGetBytesPerRow(pixel_buffer: *const std::ffi::c_void) -> usize;
fn CVPixelBufferGetWidth(pixel_buffer: *const std::ffi::c_void) -> usize;
fn CVPixelBufferGetHeight(pixel_buffer: *const std::ffi::c_void) -> usize;
}
#[link(name = "CoreMedia", kind = "framework")]
extern "C" {
fn CMSampleBufferGetImageBuffer(
sample_buffer: *const std::ffi::c_void,
) -> *const std::ffi::c_void;
}
#[link(name = "System")]
extern "C" {
fn dispatch_queue_create(
label: *const i8,
attr: *const std::ffi::c_void,
) -> *mut std::ffi::c_void;
}
#[link(name = "CoreFoundation", kind = "framework")]
extern "C" {
fn CFRunLoopGetCurrent() -> *mut std::ffi::c_void;
fn CFRunLoopRun();
fn CFRunLoopStop(rl: *mut std::ffi::c_void);
}
#[link(name = "objc", kind = "dylib")]
extern "C" {
fn class_addMethod(
cls: *const std::ffi::c_void,
name: objc2::runtime::Sel,
imp: *const std::ffi::c_void,
types: *const i8,
) -> Bool;
}
static FRAME_SENDER: std::sync::Mutex<Option<mpsc::SyncSender<RetainedPixelBuffer>>> =
std::sync::Mutex::new(None);
const K_CV_PIXEL_BUFFER_LOCK_READ_ONLY: u64 = 0x00000001;
extern "C" fn capture_callback(
_this: *mut std::ffi::c_void,
_cmd: objc2::runtime::Sel,
_output: *mut std::ffi::c_void,
sample_buffer: *mut std::ffi::c_void,
_connection: *mut std::ffi::c_void,
) {
if sample_buffer.is_null() {
return;
}
unsafe {
let pixel_buffer = CMSampleBufferGetImageBuffer(sample_buffer);
if pixel_buffer.is_null() {
return;
}
let width = CVPixelBufferGetWidth(pixel_buffer) as u32;
let height = CVPixelBufferGetHeight(pixel_buffer) as u32;
let bytes_per_row = CVPixelBufferGetBytesPerRow(pixel_buffer);
CFRetain(pixel_buffer);
let frame = RetainedPixelBuffer {
ptr: pixel_buffer,
width,
height,
bytes_per_row,
timestamp_us: 0, };
if let Ok(guard) = FRAME_SENDER.lock() {
if let Some(tx) = guard.as_ref() {
let _ = tx.try_send(frame);
}
}
}
}
impl Camera {
pub fn open(index: u32, width: u32, height: u32, fps: u32) -> Result<Self> {
Self::open_with_options(index, width, height, fps, CameraOptions::default())
}
pub fn open_with_options(
index: u32,
width: u32,
height: u32,
fps: u32,
options: CameraOptions,
) -> Result<Self> {
Self::open_on_thread(index, width, height, fps, options)
}
fn open_on_thread(
index: u32,
width: u32,
height: u32,
_fps: u32,
_options: CameraOptions,
) -> Result<Self> {
eprintln!("[camera_macos] open_on_thread: checking auth...");
unsafe {
let media_type = AVMediaTypeVideo.expect("AVMediaTypeVideo not available");
eprintln!("[camera_macos] open_on_thread: calling authorizationStatusForMediaType...");
let status: isize = msg_send![
class!(AVCaptureDevice),
authorizationStatusForMediaType: media_type
];
eprintln!("[camera_macos] auth status = {}", status);
match status {
3 => {} 0 => {
let (auth_tx, auth_rx) = std::sync::mpsc::sync_channel(1);
let block = block2::StackBlock::new(move |granted: Bool| {
let _ = auth_tx.send(granted.as_bool());
});
let () = msg_send![
class!(AVCaptureDevice),
requestAccessForMediaType: media_type,
completionHandler: &block
];
let granted = auth_rx
.recv_timeout(std::time::Duration::from_secs(30))
.map_err(|_| {
anyhow::anyhow!(
"Camera authorization timed out. \
Grant access in System Settings > Privacy & Security > Camera"
)
})?;
if !granted {
anyhow::bail!(
"Camera access denied. \
Grant access in System Settings > Privacy & Security > Camera"
);
}
}
1 => anyhow::bail!("Camera access restricted by system policy"),
2 => anyhow::bail!(
"Camera access denied. \
Grant access in System Settings > Privacy & Security > Camera"
),
_ => anyhow::bail!("Unknown camera authorization status: {}", status),
}
}
eprintln!("[camera_macos] auth passed, listing cameras...");
let cameras = list_cameras()?;
eprintln!("[camera_macos] found {} cameras", cameras.len());
let _cam_info = cameras
.iter()
.find(|c| c.index == index)
.ok_or_else(|| anyhow::anyhow!("Camera index {} not found", index))?;
unsafe {
let capture_session = AVCaptureSession::new();
capture_session.beginConfiguration();
let preset_str = match (width, height) {
(w, h) if w >= 1920 && h >= 1080 => "AVCaptureSessionPreset1920x1080",
(w, h) if w >= 1280 && h >= 720 => "AVCaptureSessionPreset1280x720",
_ => "AVCaptureSessionPreset640x480",
};
let preset = NSString::from_str(preset_str);
let can_set: Bool = msg_send![&capture_session, canSetSessionPreset: &*preset];
if can_set.as_bool() {
let _: () = msg_send![&capture_session, setSessionPreset: &*preset];
}
let media_type = AVMediaTypeVideo.expect("AVMediaTypeVideo not available");
let device = if index == 0 {
AVCaptureDevice::defaultDeviceWithMediaType(media_type)
.ok_or_else(|| anyhow::anyhow!("No default camera device found"))?
} else {
let devices = list_av_devices()?;
if (index as usize) >= devices.len() {
anyhow::bail!(
"Camera index {} out of range (found {})",
index,
devices.len()
);
}
devices.into_iter().nth(index as usize).unwrap()
};
let device_input = AVCaptureDeviceInput::deviceInputWithDevice_error(&device)
.map_err(|e| anyhow::anyhow!("Failed to create device input: {:?}", e))?;
if !capture_session.canAddInput(&device_input) {
anyhow::bail!("Cannot add camera input to session");
}
capture_session.addInput(&device_input);
let video_output = AVCaptureVideoDataOutput::new();
let format_key = NSString::from_str("PixelFormatType");
let format_value: Retained<NSNumber> =
msg_send![class!(NSNumber), numberWithUnsignedInt: 0x42475241u32];
let video_settings: Retained<NSObject> = msg_send![
class!(NSDictionary),
dictionaryWithObject: &*format_value,
forKey: &*format_key
];
let _: () = msg_send![&video_output, setVideoSettings: &*video_settings];
video_output.setAlwaysDiscardsLateVideoFrames(true);
let delegate = create_capture_delegate()?;
let queue_label = b"com.xoq.camera.queue\0";
let callback_queue =
dispatch_queue_create(queue_label.as_ptr() as *const i8, std::ptr::null());
set_sample_buffer_delegate(
&*video_output as *const _ as *const std::ffi::c_void,
&*delegate as *const _ as *const std::ffi::c_void,
callback_queue,
);
if !capture_session.canAddOutput(&video_output) {
anyhow::bail!("Cannot add video output to session");
}
capture_session.addOutput(&video_output);
capture_session.commitConfiguration();
let (tx, rx) = mpsc::sync_channel(1);
{
let mut guard = FRAME_SENDER.lock().unwrap();
*guard = Some(tx);
}
capture_session.startRunning();
let (actual_w, actual_h) = match preset_str {
"AVCaptureSessionPreset1920x1080" => (1920, 1080),
"AVCaptureSessionPreset1280x720" => (1280, 720),
_ => (640, 480),
};
Ok(Camera {
width: actual_w,
height: actual_h,
rx,
start_time: std::time::Instant::now(),
_session: capture_session,
_delegate: delegate,
})
}
}
pub fn width(&self) -> u32 {
self.width
}
pub fn height(&self) -> u32 {
self.height
}
pub fn format_name(&self) -> &'static str {
"BGRA"
}
pub fn capture(&mut self) -> Result<Frame> {
let retained = self
.rx
.recv()
.map_err(|_| anyhow::anyhow!("Camera capture channel closed"))?;
let timestamp_us = self.start_time.elapsed().as_micros() as u64;
let width = retained.width;
let height = retained.height;
let bytes_per_row = retained.bytes_per_row;
let rgb_data = unsafe {
CVPixelBufferLockBaseAddress(retained.ptr, K_CV_PIXEL_BUFFER_LOCK_READ_ONLY);
let base = CVPixelBufferGetBaseAddress(retained.ptr);
let data = if !base.is_null() {
let bgra = std::slice::from_raw_parts(base, bytes_per_row * height as usize);
bgra_to_rgb(bgra, width, height, bytes_per_row)
} else {
vec![]
};
CVPixelBufferUnlockBaseAddress(retained.ptr, K_CV_PIXEL_BUFFER_LOCK_READ_ONLY);
data
};
Ok(Frame {
width,
height,
data: rgb_data,
timestamp_us,
})
}
pub fn capture_raw(&mut self) -> Result<RawFrame> {
let retained = self
.rx
.recv()
.map_err(|_| anyhow::anyhow!("Camera capture channel closed"))?;
let timestamp_us = self.start_time.elapsed().as_micros() as u64;
let width = retained.width;
let height = retained.height;
let bytes_per_row = retained.bytes_per_row;
let data = unsafe {
CVPixelBufferLockBaseAddress(retained.ptr, K_CV_PIXEL_BUFFER_LOCK_READ_ONLY);
let base = CVPixelBufferGetBaseAddress(retained.ptr);
let d = if !base.is_null() {
let bgra = std::slice::from_raw_parts(base, bytes_per_row * height as usize);
extract_packed_bgra(bgra, width, height, bytes_per_row)
} else {
vec![]
};
CVPixelBufferUnlockBaseAddress(retained.ptr, K_CV_PIXEL_BUFFER_LOCK_READ_ONLY);
d
};
Ok(RawFrame {
width,
height,
data,
format: RawFormat::Bgra,
timestamp_us,
})
}
pub fn capture_pixel_buffer(&mut self) -> Result<RetainedPixelBuffer> {
let mut retained = self
.rx
.recv()
.map_err(|_| anyhow::anyhow!("Camera capture channel closed"))?;
retained.timestamp_us = self.start_time.elapsed().as_micros() as u64;
Ok(retained)
}
pub fn is_yuyv(&self) -> bool {
false
}
}
impl Drop for Camera {
fn drop(&mut self) {
unsafe { self._session.stopRunning() };
if let Ok(mut guard) = FRAME_SENDER.lock() {
*guard = None;
}
}
}
fn bgra_to_rgb(bgra: &[u8], width: u32, height: u32, bytes_per_row: usize) -> Vec<u8> {
let w = width as usize;
let h = height as usize;
let mut rgb = Vec::with_capacity(w * h * 3);
for y in 0..h {
let row_start = y * bytes_per_row;
for x in 0..w {
let idx = row_start + x * 4;
if idx + 2 < bgra.len() {
rgb.push(bgra[idx + 2]); rgb.push(bgra[idx + 1]); rgb.push(bgra[idx]); }
}
}
rgb
}
fn extract_packed_bgra(bgra: &[u8], width: u32, height: u32, bytes_per_row: usize) -> Vec<u8> {
let w = width as usize;
let h = height as usize;
let packed_row = w * 4;
if bytes_per_row == packed_row {
bgra[..packed_row * h].to_vec()
} else {
let mut packed = Vec::with_capacity(packed_row * h);
for y in 0..h {
let start = y * bytes_per_row;
let end = start + packed_row;
if end <= bgra.len() {
packed.extend_from_slice(&bgra[start..end]);
}
}
packed
}
}
unsafe fn create_capture_delegate() -> Result<Retained<NSObject>> {
use objc2::declare::ClassBuilder;
use objc2::runtime::AnyProtocol;
use objc2::ClassType;
use std::ffi::CStr;
let class_name = CStr::from_bytes_with_nul(b"XoqCameraMacosDelegate\0").unwrap();
let protocol_name =
CStr::from_bytes_with_nul(b"AVCaptureVideoDataOutputSampleBufferDelegate\0").unwrap();
let protocol =
AnyProtocol::get(protocol_name).ok_or_else(|| anyhow::anyhow!("Protocol not found"))?;
let mut builder = ClassBuilder::new(class_name, NSObject::class()).ok_or_else(|| {
anyhow::anyhow!("Failed to create class builder (class may already exist)")
})?;
builder.add_protocol(protocol);
let delegate_class = builder.register();
let method_sel = objc2::sel!(captureOutput:didOutputSampleBuffer:fromConnection:);
let method_types = b"v@:@@@\0";
let added = class_addMethod(
delegate_class as *const _ as *const std::ffi::c_void,
method_sel,
capture_callback as *const std::ffi::c_void,
method_types.as_ptr() as *const i8,
);
if !added.as_bool() {
anyhow::bail!("Failed to add method to delegate class");
}
let delegate: Retained<NSObject> = msg_send![delegate_class, new];
Ok(delegate)
}
unsafe fn set_sample_buffer_delegate(
output: *const std::ffi::c_void,
delegate: *const std::ffi::c_void,
queue: *const std::ffi::c_void,
) {
#[link(name = "objc", kind = "dylib")]
extern "C" {
#[link_name = "objc_msgSend"]
fn objc_msgSend_set_delegate(
receiver: *const std::ffi::c_void,
sel: objc2::runtime::Sel,
delegate: *const std::ffi::c_void,
queue: *const std::ffi::c_void,
);
}
let sel = objc2::sel!(setSampleBufferDelegate:queue:);
objc_msgSend_set_delegate(output, sel, delegate, queue);
}
fn list_av_devices() -> Result<Vec<Retained<AVCaptureDevice>>> {
let media_type = unsafe { AVMediaTypeVideo.expect("AVMediaTypeVideo not available") };
unsafe {
let built_in = NSString::from_str("AVCaptureDeviceTypeBuiltInWideAngleCamera");
let external = NSString::from_str("AVCaptureDeviceTypeExternal");
let objects: [*const NSString; 2] = [&*built_in, &*external];
let device_types: Retained<objc2_foundation::NSArray<NSString>> = msg_send![
class!(NSArray),
arrayWithObjects: objects.as_ptr(),
count: 2usize
];
let discovery_session: Retained<NSObject> = msg_send![
class!(AVCaptureDeviceDiscoverySession),
discoverySessionWithDeviceTypes: &*device_types,
mediaType: media_type,
position: 0isize ];
let devices: Retained<objc2_foundation::NSArray<AVCaptureDevice>> =
msg_send![&discovery_session, devices];
let mut result = Vec::new();
for i in 0..devices.len() {
result.push(devices.objectAtIndex(i).clone());
}
Ok(result)
}
}
pub fn list_cameras() -> Result<Vec<CameraInfo>> {
let devices = list_av_devices()?;
let cameras: Vec<CameraInfo> = devices
.iter()
.enumerate()
.map(|(i, dev)| {
let name = unsafe { dev.localizedName() }.to_string();
let unique_id = unsafe {
let uid: Retained<NSString> = msg_send![dev, uniqueID];
uid.to_string()
};
CameraInfo {
index: i as u32,
name,
path: PathBuf::from(unique_id),
}
})
.collect();
Ok(cameras)
}