#![allow(unsafe_code)]
use std::collections::VecDeque;
use std::ffi::c_char;
use std::sync::atomic::{AtomicI64, Ordering};
use std::sync::mpsc::sync_channel;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use crate::CaptureError;
use crate::audio::AudioCapture;
use crate::desktop::DesktopAudioCapture;
use crate::desktop::DesktopVideoCapture;
use block2::RcBlock;
use mediaway_common::{
AudioFrame, Bytes, CodecKind, PixelFormat, Rational, SampleFormat, StreamInfo, VideoFrame,
VideoFrameStorage, VideoGeometry,
};
use objc2::rc::Retained;
use objc2_core_audio_types::{kAudioFormatFlagIsFloat, kAudioFormatFlagIsNonInterleaved};
use objc2_core_media::{CMAudioFormatDescription, CMSampleBuffer};
use objc2_foundation::NSError;
use objc2_replay_kit::{RPSampleBufferType, RPScreenRecorder};
const QUEUE_CAP: usize = 4;
const COMPLETION_TIMEOUT: Duration = Duration::from_secs(10);
struct FrameQueues {
video: Mutex<VecDeque<VideoFrame>>,
app_audio: Mutex<VecDeque<AudioFrame>>,
mic_audio: Mutex<VecDeque<AudioFrame>>,
}
impl FrameQueues {
const fn new() -> Self {
Self {
video: Mutex::new(VecDeque::new()),
app_audio: Mutex::new(VecDeque::new()),
mic_audio: Mutex::new(VecDeque::new()),
}
}
}
struct StreamInfos {
video: std::sync::OnceLock<StreamInfo>,
app_audio: std::sync::OnceLock<StreamInfo>,
mic_audio: std::sync::OnceLock<StreamInfo>,
}
impl StreamInfos {
const fn new() -> Self {
Self {
video: std::sync::OnceLock::new(),
app_audio: std::sync::OnceLock::new(),
mic_audio: std::sync::OnceLock::new(),
}
}
}
fn unknown_video_info() -> &'static StreamInfo {
use std::sync::OnceLock;
static INFO: OnceLock<StreamInfo> = OnceLock::new();
INFO.get_or_init(|| StreamInfo::Video {
id: 0,
codec: CodecKind::RawVideo,
time_base: Rational::new(1, 30),
geometry: VideoGeometry {
width: 0,
height: 0,
},
extra_data: Bytes::new(),
})
}
fn unknown_audio_info() -> &'static StreamInfo {
use std::sync::OnceLock;
static INFO: OnceLock<StreamInfo> = OnceLock::new();
INFO.get_or_init(|| StreamInfo::Audio {
id: 0,
codec: CodecKind::RawAudio,
time_base: Rational::new(1, 48_000),
sample_rate: 0,
channels: 0,
extra_data: Bytes::new(),
})
}
unsafe fn classify_and_queue_sample_buffer(
queues: &FrameQueues,
infos: &StreamInfos,
next_pts: &AtomicI64,
sample_buffer: &CMSampleBuffer,
kind: RPSampleBufferType,
) {
match kind {
RPSampleBufferType::Video => {
let Some((data, width, height)) =
(unsafe { super::pixel::extract_nv12(sample_buffer) })
else {
return;
};
let pts = next_pts.fetch_add(1, Ordering::Relaxed);
let _ = infos.video.set(StreamInfo::Video {
id: 0,
codec: CodecKind::RawVideo,
time_base: Rational::new(1, 30),
geometry: VideoGeometry { width, height },
extra_data: Bytes::new(),
});
let frame = VideoFrame {
pts,
duration: 1,
width,
height,
format: PixelFormat::Nv12,
storage: VideoFrameStorage::Cpu { data },
};
push_bounded(&queues.video, frame);
}
RPSampleBufferType::AudioApp => {
if let Some(frame) = unsafe { extract_pcm(sample_buffer) } {
let _ = infos.app_audio.set(audio_stream_info(&frame));
push_bounded(&queues.app_audio, frame);
}
}
RPSampleBufferType::AudioMic => {
if let Some(frame) = unsafe { extract_pcm(sample_buffer) } {
let _ = infos.mic_audio.set(audio_stream_info(&frame));
push_bounded(&queues.mic_audio, frame);
}
}
_ => {}
}
}
fn audio_stream_info(frame: &AudioFrame) -> StreamInfo {
StreamInfo::Audio {
id: 0,
codec: CodecKind::RawAudio,
time_base: Rational::new(1, frame.sample_rate.max(1)),
sample_rate: frame.sample_rate,
channels: frame.channels,
extra_data: Bytes::new(),
}
}
fn push_bounded<T>(queue: &Mutex<VecDeque<T>>, item: T) {
if let Ok(mut q) = queue.lock() {
if q.len() >= QUEUE_CAP {
let _ = q.pop_front();
}
q.push_back(item);
}
}
unsafe fn extract_pcm(sample_buffer: &CMSampleBuffer) -> Option<AudioFrame> {
let format_description = unsafe { sample_buffer.format_description() }?;
let audio_description = format_description
.downcast::<CMAudioFormatDescription>()
.ok()?;
let asbd_ptr = unsafe {
objc2_core_media::CMAudioFormatDescriptionGetStreamBasicDescription(&audio_description)
};
if asbd_ptr.is_null() {
return None;
}
let asbd = unsafe { *asbd_ptr };
if asbd.mFormatFlags & kAudioFormatFlagIsFloat == 0 {
return None;
}
if asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0 {
return None;
}
let channels = asbd.mChannelsPerFrame;
if channels == 0 || asbd.mSampleRate <= 0.0 {
return None;
}
let block_buffer = unsafe { sample_buffer.data_buffer() }?;
let total_len = unsafe { block_buffer.data_length() };
let mut length_at_offset: usize = 0;
let mut data_ptr: *mut c_char = std::ptr::null_mut();
let status = unsafe {
block_buffer.data_pointer(
0,
&raw mut length_at_offset,
std::ptr::null_mut(),
&raw mut data_ptr,
)
};
if status != 0 || data_ptr.is_null() || length_at_offset != total_len {
return None;
}
let bytes = unsafe { std::slice::from_raw_parts(data_ptr.cast::<u8>(), total_len) };
let bytes_per_frame = 4usize.saturating_mul(channels as usize);
let num_frames = total_len.checked_div(bytes_per_frame).unwrap_or(0);
#[allow(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
reason = "mSampleRate > 0.0 checked above; real sample rates are small positive integers"
)]
let sample_rate = asbd.mSampleRate as u32;
Some(AudioFrame {
pts: 0,
duration: u64::try_from(num_frames).unwrap_or(0),
sample_rate,
channels: u16::try_from(channels).unwrap_or(0),
format: SampleFormat::F32,
data: Bytes::copy_from_slice(bytes),
})
}
struct InAppSession {
queues: Arc<FrameQueues>,
infos: Arc<StreamInfos>,
recorder: Retained<RPScreenRecorder>,
_capture_handler:
RcBlock<dyn Fn(std::ptr::NonNull<CMSampleBuffer>, RPSampleBufferType, *mut NSError)>,
}
pub struct AppleScreenCapture {
inner: Option<InAppSession>,
}
impl AppleScreenCapture {
pub fn open() -> Result<Self, CaptureError> {
let recorder = unsafe { RPScreenRecorder::sharedRecorder() };
unsafe { recorder.setMicrophoneEnabled(true) };
let queues = Arc::new(FrameQueues::new());
let infos = Arc::new(StreamInfos::new());
let next_pts = Arc::new(AtomicI64::new(0));
let queues_cb = Arc::clone(&queues);
let infos_cb = Arc::clone(&infos);
let next_pts_cb = Arc::clone(&next_pts);
let capture_handler: RcBlock<
dyn Fn(std::ptr::NonNull<CMSampleBuffer>, RPSampleBufferType, *mut NSError),
> = RcBlock::new(
move |sample_buffer: std::ptr::NonNull<CMSampleBuffer>,
kind: RPSampleBufferType,
_error: *mut NSError| {
let sample_buffer = unsafe { sample_buffer.as_ref() };
unsafe {
classify_and_queue_sample_buffer(
&queues_cb,
&infos_cb,
&next_pts_cb,
sample_buffer,
kind,
);
}
},
);
start_in_app_capture(&recorder, &capture_handler)?;
Ok(Self {
inner: Some(InAppSession {
queues,
infos,
recorder,
_capture_handler: capture_handler,
}),
})
}
fn close_inner(&mut self) -> Result<(), CaptureError> {
let Some(session) = self.inner.take() else {
return Ok(());
};
stop_in_app_capture(&session.recorder)
}
}
impl DesktopVideoCapture for AppleScreenCapture {
fn stream_info(&self) -> &StreamInfo {
#[allow(
clippy::option_if_let_else,
reason = "map_or_else forces 'static vs 'self lifetime clash"
)]
if let Some(session) = self.inner.as_ref() {
session
.infos
.video
.get()
.unwrap_or_else(|| unknown_video_info())
} else {
unknown_video_info()
}
}
fn poll_frame(&mut self) -> Result<Option<VideoFrame>, CaptureError> {
let Some(session) = self.inner.as_ref() else {
return Err(CaptureError::Closed);
};
let mut q = session
.queues
.video
.lock()
.map_err(|_| CaptureError::Backend)?;
Ok(q.pop_front())
}
fn release_frame(&mut self) -> Result<(), CaptureError> {
if self.inner.is_none() {
return Err(CaptureError::Closed);
}
Ok(())
}
fn close(&mut self) -> Result<(), CaptureError> {
self.close_inner()
}
}
impl DesktopAudioCapture for AppleScreenCapture {
fn stream_info(&self) -> &StreamInfo {
#[allow(
clippy::option_if_let_else,
reason = "map_or_else forces 'static vs 'self lifetime clash"
)]
if let Some(session) = self.inner.as_ref() {
session
.infos
.app_audio
.get()
.unwrap_or_else(|| unknown_audio_info())
} else {
unknown_audio_info()
}
}
fn poll_frame(&mut self) -> Result<Option<AudioFrame>, CaptureError> {
let Some(session) = self.inner.as_ref() else {
return Err(CaptureError::Closed);
};
let mut q = session
.queues
.app_audio
.lock()
.map_err(|_| CaptureError::Backend)?;
Ok(q.pop_front())
}
fn close(&mut self) -> Result<(), CaptureError> {
self.close_inner()
}
}
impl AudioCapture for AppleScreenCapture {
fn stream_info(&self) -> &StreamInfo {
#[allow(
clippy::option_if_let_else,
reason = "map_or_else forces 'static vs 'self lifetime clash"
)]
if let Some(session) = self.inner.as_ref() {
session
.infos
.mic_audio
.get()
.unwrap_or_else(|| unknown_audio_info())
} else {
unknown_audio_info()
}
}
fn poll_frame(&mut self) -> Result<Option<AudioFrame>, CaptureError> {
let Some(session) = self.inner.as_ref() else {
return Err(CaptureError::Closed);
};
let mut q = session
.queues
.mic_audio
.lock()
.map_err(|_| CaptureError::Backend)?;
Ok(q.pop_front())
}
fn close(&mut self) -> Result<(), CaptureError> {
self.close_inner()
}
}
impl Drop for AppleScreenCapture {
fn drop(&mut self) {
let _ = self.close_inner();
}
}
fn start_in_app_capture(
recorder: &RPScreenRecorder,
capture_handler: &RcBlock<
dyn Fn(std::ptr::NonNull<CMSampleBuffer>, RPSampleBufferType, *mut NSError),
>,
) -> Result<(), CaptureError> {
let (tx, rx) = sync_channel::<Result<(), CaptureError>>(1);
let tx = Arc::new(Mutex::new(Some(tx)));
let completion: RcBlock<dyn Fn(*mut NSError)> = RcBlock::new(move |error: *mut NSError| {
let result = if error.is_null() {
Ok(())
} else {
Err(CaptureError::Backend)
};
if let Ok(mut guard) = tx.lock()
&& let Some(tx) = guard.take()
{
let _ = tx.send(result);
}
});
unsafe {
recorder
.startCaptureWithHandler_completionHandler(Some(capture_handler), Some(&completion));
}
rx.recv_timeout(COMPLETION_TIMEOUT)
.map_err(|_| CaptureError::Backend)?
}
fn stop_in_app_capture(recorder: &RPScreenRecorder) -> Result<(), CaptureError> {
let (tx, rx) = sync_channel::<Result<(), CaptureError>>(1);
let tx = Arc::new(Mutex::new(Some(tx)));
let handler: RcBlock<dyn Fn(*mut NSError)> = RcBlock::new(move |error: *mut NSError| {
let result = if error.is_null() {
Ok(())
} else {
Err(CaptureError::Backend)
};
if let Ok(mut guard) = tx.lock()
&& let Some(tx) = guard.take()
{
let _ = tx.send(result);
}
});
unsafe { recorder.stopCaptureWithHandler(Some(&handler)) };
rx.recv_timeout(COMPLETION_TIMEOUT)
.map_err(|_| CaptureError::Backend)?
}
struct ExtensionSession {
queues: Arc<FrameQueues>,
infos: Arc<StreamInfos>,
next_pts: AtomicI64,
}
pub struct AppleBroadcastExtensionCapture {
inner: Option<ExtensionSession>,
}
impl Default for AppleBroadcastExtensionCapture {
fn default() -> Self {
Self::new()
}
}
impl AppleBroadcastExtensionCapture {
#[must_use]
pub fn new() -> Self {
Self {
inner: Some(ExtensionSession {
queues: Arc::new(FrameQueues::new()),
infos: Arc::new(StreamInfos::new()),
next_pts: AtomicI64::new(0),
}),
}
}
pub fn push_sample_buffer(
&self,
sample_buffer: &CMSampleBuffer,
kind: RPSampleBufferType,
) -> Result<(), CaptureError> {
let Some(session) = self.inner.as_ref() else {
return Err(CaptureError::Closed);
};
unsafe {
classify_and_queue_sample_buffer(
&session.queues,
&session.infos,
&session.next_pts,
sample_buffer,
kind,
);
}
Ok(())
}
fn close_inner(&mut self) {
self.inner = None;
}
}
impl DesktopVideoCapture for AppleBroadcastExtensionCapture {
fn stream_info(&self) -> &StreamInfo {
#[allow(
clippy::option_if_let_else,
reason = "map_or_else forces 'static vs 'self lifetime clash"
)]
if let Some(session) = self.inner.as_ref() {
session
.infos
.video
.get()
.unwrap_or_else(|| unknown_video_info())
} else {
unknown_video_info()
}
}
fn poll_frame(&mut self) -> Result<Option<VideoFrame>, CaptureError> {
let Some(session) = self.inner.as_ref() else {
return Err(CaptureError::Closed);
};
let mut q = session
.queues
.video
.lock()
.map_err(|_| CaptureError::Backend)?;
Ok(q.pop_front())
}
fn release_frame(&mut self) -> Result<(), CaptureError> {
if self.inner.is_none() {
return Err(CaptureError::Closed);
}
Ok(())
}
fn close(&mut self) -> Result<(), CaptureError> {
self.close_inner();
Ok(())
}
}
impl DesktopAudioCapture for AppleBroadcastExtensionCapture {
fn stream_info(&self) -> &StreamInfo {
#[allow(
clippy::option_if_let_else,
reason = "map_or_else forces 'static vs 'self lifetime clash"
)]
if let Some(session) = self.inner.as_ref() {
session
.infos
.app_audio
.get()
.unwrap_or_else(|| unknown_audio_info())
} else {
unknown_audio_info()
}
}
fn poll_frame(&mut self) -> Result<Option<AudioFrame>, CaptureError> {
let Some(session) = self.inner.as_ref() else {
return Err(CaptureError::Closed);
};
let mut q = session
.queues
.app_audio
.lock()
.map_err(|_| CaptureError::Backend)?;
Ok(q.pop_front())
}
fn close(&mut self) -> Result<(), CaptureError> {
self.close_inner();
Ok(())
}
}
impl AudioCapture for AppleBroadcastExtensionCapture {
fn stream_info(&self) -> &StreamInfo {
#[allow(
clippy::option_if_let_else,
reason = "map_or_else forces 'static vs 'self lifetime clash"
)]
if let Some(session) = self.inner.as_ref() {
session
.infos
.mic_audio
.get()
.unwrap_or_else(|| unknown_audio_info())
} else {
unknown_audio_info()
}
}
fn poll_frame(&mut self) -> Result<Option<AudioFrame>, CaptureError> {
let Some(session) = self.inner.as_ref() else {
return Err(CaptureError::Closed);
};
let mut q = session
.queues
.mic_audio
.lock()
.map_err(|_| CaptureError::Backend)?;
Ok(q.pop_front())
}
fn close(&mut self) -> Result<(), CaptureError> {
self.close_inner();
Ok(())
}
}
impl Drop for AppleBroadcastExtensionCapture {
fn drop(&mut self) {
self.close_inner();
}
}
#[cfg(test)]
#[path = "replaykit_tests.rs"]
mod tests;