use core::ffi::c_void;
use core::ptr;
use std::sync::{Arc, Mutex};
use apple_cf::cf::{AsCFType, CFDictionary, CFType};
use apple_cf::cm::CMTime;
#[cfg(feature = "async")]
use doom_fish_utils::completion::AsyncCompletion;
use doom_fish_utils::panic_safe::catch_user_panic;
use crate::error::VTError;
use crate::ffi;
use crate::session::{self, Codec};
use crate::tagged_buffer_group::TaggedBufferGroup;
pub struct DecodedFrame {
pub image_buffer: Option<apple_cf::cv::CVPixelBuffer>,
pub presentation_time: CMTime,
pub duration: CMTime,
pub info_flags: ffi::VTDecodeInfoFlags,
pub status: i32,
}
pub struct DecodedMultiImageFrame {
pub tagged_buffer_group: Option<TaggedBufferGroup>,
pub presentation_time: CMTime,
pub duration: CMTime,
pub info_flags: ffi::VTDecodeInfoFlags,
pub status: i32,
}
type DecodeCallback = Box<dyn FnMut(DecodedFrame) + Send + 'static>;
type MultiImageDecodeCallback = Box<dyn FnMut(DecodedMultiImageFrame) + Send + 'static>;
struct CallbackState {
callback: Mutex<DecodeCallback>,
multi_image_callback: Mutex<Option<MultiImageDecodeCallback>>,
}
pub struct DecompressionSession {
session: ffi::VTDecompressionSessionRef,
state: Arc<CallbackState>,
callback_ref_con: *const CallbackState,
}
unsafe impl Send for DecompressionSession {}
impl DecompressionSession {
#[must_use]
pub fn type_id() -> usize {
unsafe { ffi::VTDecompressionSessionGetTypeID() }
}
#[must_use]
pub fn is_hardware_decode_supported(codec: Codec) -> bool {
unsafe { ffi::VTIsHardwareDecodeSupported(codec.as_cm_codec_type()) != 0 }
}
#[must_use]
pub fn is_stereo_mvhevc_decode_supported() -> bool {
ffi::dynamic::VTIsStereoMVHEVCDecodeSupported()
.is_ok_and(|is_supported| unsafe { is_supported() } != 0)
}
pub fn new<F>(
format_description: &apple_cf::cm::CMFormatDescription,
callback: F,
) -> Result<Self, VTError>
where
F: FnMut(DecodedFrame) + Send + 'static,
{
Self::new_with_image_buffer_attributes(format_description, None, callback)
}
pub fn new_with_image_buffer_attributes<F>(
format_description: &apple_cf::cm::CMFormatDescription,
image_buffer_attributes: Option<&CFDictionary>,
callback: F,
) -> Result<Self, VTError>
where
F: FnMut(DecodedFrame) + Send + 'static,
{
let state = Arc::new(CallbackState {
callback: Mutex::new(Box::new(callback)),
multi_image_callback: Mutex::new(None),
});
let callback_ref_con = Arc::into_raw(Arc::clone(&state));
let ref_con = callback_ref_con.cast::<c_void>().cast_mut();
let record = ffi::VTDecompressionOutputCallbackRecord {
decompression_output_callback: decode_trampoline,
decompression_output_ref_con: ref_con,
};
let mut session: ffi::VTDecompressionSessionRef = ptr::null_mut();
let status = unsafe {
ffi::VTDecompressionSessionCreate(
ffi::kCFAllocatorDefault,
format_description.as_ptr().cast(),
ptr::null(),
image_buffer_attributes.map_or(ptr::null(), |d| AsCFType::as_ptr(d).cast()),
&raw const record,
&raw mut session,
)
};
if status != 0 || session.is_null() {
unsafe { drop(Arc::from_raw(callback_ref_con)) };
return Err(VTError::EncoderCallback(if status == 0 {
-1
} else {
status
}));
}
Ok(Self {
session,
state,
callback_ref_con,
})
}
pub fn decode(&self, sample_buffer: &apple_cf::cm::CMSampleBuffer) -> Result<(), VTError> {
self.decode_with_options(sample_buffer, 0, None).map(|_| ())
}
pub fn decode_with_options(
&self,
sample_buffer: &apple_cf::cm::CMSampleBuffer,
decode_flags: ffi::VTDecodeFrameFlags,
frame_options: Option<&CFDictionary>,
) -> Result<ffi::VTDecodeInfoFlags, VTError> {
let mut info_flags: ffi::VTDecodeInfoFlags = 0;
let status = match frame_options {
None => unsafe {
ffi::VTDecompressionSessionDecodeFrame(
self.session,
sample_buffer.as_ptr().cast(),
decode_flags,
ptr::null_mut(),
&raw mut info_flags,
)
},
Some(frame_options) => {
let decode = ffi::dynamic::VTDecompressionSessionDecodeFrameWithOptions()?;
unsafe {
decode(
self.session,
sample_buffer.as_ptr().cast(),
decode_flags,
frame_options.as_ptr().cast_const().cast(),
ptr::null_mut(),
&raw mut info_flags,
)
}
}
};
if status == 0 {
Ok(info_flags)
} else {
Err(VTError::EncoderCallback(status))
}
}
#[cfg(feature = "async")]
#[cfg_attr(docsrs, doc(cfg(feature = "async")))]
#[allow(clippy::future_not_send)]
pub async fn decode_frame_async(
&self,
sample_buffer: apple_cf::cm::CMSampleBuffer,
frame_flags: u32,
) -> Result<apple_cf::cv::CVImageBuffer, VTError> {
validate_async_sample_count(&sample_buffer)?;
let (future, completion) = AsyncCompletion::<apple_cf::cv::CVImageBuffer>::create();
let status = unsafe {
ffi::VTDecompressionSessionDecodeFrame(
self.session,
sample_buffer.as_ptr().cast(),
frame_flags,
completion,
ptr::null_mut(),
)
};
if status != 0 {
unsafe {
AsyncCompletion::<apple_cf::cv::CVImageBuffer>::complete_err(
completion,
status.to_string(),
);
};
return Err(VTError::EncoderCallback(status));
}
future
.await
.map_err(|error| VTError::EncoderCallback(parse_async_status(&error)))
}
pub fn set_multi_image_callback<F>(&self, callback: F) -> Result<(), VTError>
where
F: FnMut(DecodedMultiImageFrame) + Send + 'static,
{
let set_callback = ffi::dynamic::VTDecompressionSessionSetMultiImageCallback()?;
{
let mut slot = self
.state
.multi_image_callback
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*slot = Some(Box::new(callback));
}
let status = unsafe {
set_callback(
self.session,
decode_multi_image_trampoline,
Arc::as_ptr(&self.state).cast::<c_void>().cast_mut(),
)
};
if status != 0 {
let mut slot = self
.state
.multi_image_callback
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*slot = None;
drop(slot);
return Err(VTError::ApiFailed {
api: "VTDecompressionSessionSetMultiImageCallback",
status,
});
}
Ok(())
}
pub fn wait_for_async_frames(&self) -> Result<(), VTError> {
let status = unsafe { ffi::VTDecompressionSessionWaitForAsynchronousFrames(self.session) };
if status == 0 {
Ok(())
} else {
Err(VTError::EncoderCallback(status))
}
}
pub fn invalidate(mut self) -> Result<(), VTError> {
self.teardown()
}
pub fn copy_black_pixel_buffer(&self) -> Result<apple_cf::cv::CVPixelBuffer, VTError> {
let mut out = ptr::null_mut();
let status =
unsafe { ffi::VTDecompressionSessionCopyBlackPixelBuffer(self.session, &raw mut out) };
if status != 0 || out.is_null() {
return Err(VTError::ApiFailed {
api: "VTDecompressionSessionCopyBlackPixelBuffer",
status,
});
}
unsafe { apple_cf::cv::CVPixelBuffer::from_raw(out.cast()) }.ok_or(VTError::ApiFailed {
api: "VTDecompressionSessionCopyBlackPixelBuffer",
status,
})
}
pub unsafe fn copy_property(&self, key: ffi::CFStringRef) -> Result<Option<CFType>, VTError> {
session::copy_property(self.session.cast(), key)
}
pub fn supported_property_dictionary(&self) -> Result<CFDictionary, VTError> {
unsafe { session::copy_supported_property_dictionary(self.session.cast()) }
}
pub fn serializable_properties(&self) -> Result<CFDictionary, VTError> {
unsafe { session::copy_serializable_properties(self.session.cast()) }
}
pub fn set_properties(&self, properties: &CFDictionary) -> Result<(), VTError> {
unsafe { session::set_properties(self.session.cast(), properties) }
}
pub unsafe fn set_property(
&self,
key: ffi::CFStringRef,
value: ffi::CFTypeRef,
) -> Result<(), VTError> {
let status = ffi::VTSessionSetProperty(self.session.cast(), key, value);
if status != 0 {
return Err(VTError::SetPropertyFailed {
key: "<custom>".to_string(),
status,
});
}
Ok(())
}
pub fn set_real_time(&self, real_time: bool) -> Result<(), VTError> {
let v = unsafe {
if real_time {
ffi::kCFBooleanTrue
} else {
ffi::kCFBooleanFalse
}
};
unsafe { self.set_property(ffi::kVTDecompressionPropertyKey_RealTime, v.cast()) }
}
pub fn finish_delayed_frames(&self) -> Result<(), VTError> {
let status = unsafe { ffi::VTDecompressionSessionFinishDelayedFrames(self.session) };
if status == 0 {
Ok(())
} else {
Err(VTError::EncoderCallback(status))
}
}
#[must_use]
pub unsafe fn can_accept_format(&self, format: ffi::CMFormatDescriptionRef) -> bool {
ffi::VTDecompressionSessionCanAcceptFormatDescription(self.session, format)
}
fn teardown(&mut self) -> Result<(), VTError> {
if self.session.is_null() {
return Ok(());
}
let wait_status =
unsafe { ffi::VTDecompressionSessionWaitForAsynchronousFrames(self.session) };
unsafe {
ffi::VTDecompressionSessionInvalidate(self.session);
ffi::CFRelease(self.session.cast());
}
self.session = ptr::null_mut();
if wait_status == 0 {
self.release_callback_ref_con();
Ok(())
} else {
self.callback_ref_con = ptr::null();
Err(VTError::ApiFailed {
api: "VTDecompressionSessionWaitForAsynchronousFrames",
status: wait_status,
})
}
}
fn release_callback_ref_con(&mut self) {
let callback_ref_con = core::mem::replace(&mut self.callback_ref_con, ptr::null());
if !callback_ref_con.is_null() {
unsafe { drop(Arc::from_raw(callback_ref_con)) };
}
}
}
impl Drop for DecompressionSession {
fn drop(&mut self) {
let _ = self.teardown();
}
}
#[cfg(feature = "async")]
fn parse_async_status(error: &str) -> ffi::OSStatus {
error.parse().unwrap_or(-1)
}
#[cfg(feature = "async")]
fn validate_async_sample_count(
sample_buffer: &apple_cf::cm::CMSampleBuffer,
) -> Result<(), VTError> {
let actual = sample_buffer.num_samples();
if actual == 1 {
Ok(())
} else {
Err(VTError::UnexpectedSampleCount {
operation: "DecompressionSession::decode_frame_async",
expected: 1,
actual,
})
}
}
#[cfg(feature = "async")]
fn complete_async_decode(
source_frame_ref_con: *mut c_void,
status: ffi::OSStatus,
image_buffer: *mut c_void,
) {
catch_user_panic("videotoolbox::decompression::decode_frame_async", || {
if status != 0 {
unsafe {
AsyncCompletion::<apple_cf::cv::CVImageBuffer>::complete_err(
source_frame_ref_con,
status.to_string(),
);
};
return;
}
if image_buffer.is_null() {
unsafe {
AsyncCompletion::<apple_cf::cv::CVImageBuffer>::complete_err(
source_frame_ref_con,
"-1".into(),
);
};
return;
}
let Some(image_buffer) =
(unsafe { apple_cf::cv::CVImageBuffer::from_raw_borrowed(image_buffer) })
else {
unsafe {
AsyncCompletion::<apple_cf::cv::CVImageBuffer>::complete_err(
source_frame_ref_con,
"-1".into(),
);
};
return;
};
unsafe {
AsyncCompletion::<apple_cf::cv::CVImageBuffer>::complete_ok(
source_frame_ref_con,
image_buffer,
);
};
});
}
unsafe extern "C" fn decode_trampoline(
output_ref_con: *mut c_void,
source_frame_ref_con: *mut c_void,
status: ffi::OSStatus,
info_flags: ffi::VTDecodeInfoFlags,
image_buffer: *mut c_void,
pts: ffi::CMTime,
duration: ffi::CMTime,
) {
#[cfg(not(feature = "async"))]
let _ = source_frame_ref_con;
#[cfg(feature = "async")]
if !source_frame_ref_con.is_null() {
complete_async_decode(source_frame_ref_con, status, image_buffer);
return;
}
if output_ref_con.is_null() {
return;
}
let state = unsafe { Arc::from_raw(output_ref_con.cast::<CallbackState>()) };
let state_clone = state.clone();
core::mem::forget(state);
let image = if image_buffer.is_null() {
None
} else {
unsafe { apple_cf::cv::CVPixelBuffer::from_raw_borrowed(image_buffer) }
};
let frame = DecodedFrame {
image_buffer: image,
presentation_time: pts,
duration,
info_flags,
status,
};
let Ok(mut guard) = state_clone.callback.lock() else {
return;
};
catch_user_panic("videotoolbox::decompression::decode_callback", || {
guard(frame);
});
}
unsafe extern "C" fn decode_multi_image_trampoline(
output_ref_con: *mut c_void,
source_frame_ref_con: *mut c_void,
status: ffi::OSStatus,
info_flags: ffi::VTDecodeInfoFlags,
tagged_buffer_group: ffi::CMTaggedBufferGroupRef,
pts: ffi::CMTime,
duration: ffi::CMTime,
) {
#[cfg(not(feature = "async"))]
let _ = source_frame_ref_con;
#[cfg(feature = "async")]
if !source_frame_ref_con.is_null() {
complete_async_decode(source_frame_ref_con, status, ptr::null_mut());
return;
}
if output_ref_con.is_null() {
return;
}
let state = unsafe { Arc::from_raw(output_ref_con.cast::<CallbackState>()) };
let state_clone = state.clone();
core::mem::forget(state);
let frame = DecodedMultiImageFrame {
tagged_buffer_group: unsafe { TaggedBufferGroup::from_raw_retained(tagged_buffer_group) },
presentation_time: pts,
duration,
info_flags,
status,
};
let Ok(mut guard) = state_clone.multi_image_callback.lock() else {
return;
};
let Some(callback) = guard.as_mut() else {
return;
};
catch_user_panic("videotoolbox::decompression::multi_image_callback", || {
callback(frame);
});
}
#[cfg(test)]
mod tests {
use core::ptr;
use std::sync::{Arc, Mutex};
use apple_cf::cm::CMTime;
use super::{decode_multi_image_trampoline, CallbackState};
fn state() -> Arc<CallbackState> {
Arc::new(CallbackState {
callback: Mutex::new(Box::new(|_| {})),
multi_image_callback: Mutex::new(None),
})
}
#[cfg(feature = "async")]
fn deliver_to_pending_decode(status: i32) -> Result<apple_cf::cv::CVImageBuffer, String> {
use doom_fish_utils::completion::AsyncCompletion;
let state = state();
let (future, context) = AsyncCompletion::<apple_cf::cv::CVImageBuffer>::create();
unsafe {
decode_multi_image_trampoline(
Arc::as_ptr(&state).cast_mut().cast(),
context,
status,
0,
ptr::null_mut(),
CMTime::new(1, 30),
CMTime::new(1, 30),
);
}
pollster::block_on(future)
}
#[cfg(feature = "async")]
#[test]
fn multi_image_failure_resolves_the_pending_async_decode() {
assert_eq!(
deliver_to_pending_decode(-12909).err().as_deref(),
Some("-12909")
);
}
#[cfg(feature = "async")]
#[test]
fn multi_image_output_without_an_image_buffer_resolves_the_async_decode() {
assert_eq!(deliver_to_pending_decode(0).err().as_deref(), Some("-1"));
}
#[test]
fn multi_image_output_without_a_frame_context_reaches_the_callback() {
let received = Arc::new(Mutex::new(Vec::new()));
let sink = Arc::clone(&received);
let state = state();
*state.multi_image_callback.lock().expect("unpoisoned") =
Some(Box::new(move |frame: super::DecodedMultiImageFrame| {
sink.lock().expect("unpoisoned").push((
frame.status,
frame.presentation_time,
frame.duration,
));
}));
unsafe {
decode_multi_image_trampoline(
Arc::as_ptr(&state).cast_mut().cast(),
ptr::null_mut(),
-12909,
0,
ptr::null_mut(),
CMTime::new(3, 30),
CMTime::new(1, 30),
);
}
assert_eq!(
*received.lock().expect("unpoisoned"),
[(-12909, CMTime::new(3, 30), CMTime::new(1, 30))]
);
}
#[test]
fn panicking_multi_image_callback_is_contained() {
let state = state();
*state.multi_image_callback.lock().expect("unpoisoned") =
Some(Box::new(|_| panic!("callback panic")));
unsafe {
decode_multi_image_trampoline(
Arc::as_ptr(&state).cast_mut().cast(),
ptr::null_mut(),
0,
0,
ptr::null_mut(),
CMTime::new(0, 30),
CMTime::new(1, 30),
);
}
}
}