#![cfg(all(
feature = "videotoolbox",
target_os = "macos"
))]
use std::ffi::c_void;
use std::ptr;
use std::sync::Mutex;
use crate::mp4_demux::{TrackCodec, VideoTrack};
type OSStatus = i32;
type CFAllocatorRef = *const c_void;
type CFDictionaryRef = *const c_void;
type CFStringRef = *const c_void;
type CFNumberRef = *const c_void;
type CVPixelBufferRef = *mut c_void;
type CMSampleBufferRef = *mut c_void;
type CMBlockBufferRef = *mut c_void;
type CMFormatDescriptionRef = *const c_void;
type VTDecompressionSessionRef = *mut c_void;
const K_CF_NUMBER_SINT32: i32 = 3;
const PIXEL_FORMAT_BGRA: i32 = 0x4247_5241;
#[repr(C)]
#[derive(Clone, Copy)]
struct CMTime {
value: i64,
timescale: i32,
flags: u32,
epoch: i64,
}
impl CMTime {
fn new(value: i64, timescale: u32) -> Self {
CMTime {
value,
timescale: timescale as i32,
flags: 1, epoch: 0,
}
}
}
const DECODE_ENABLE_TEMPORAL: u32 = 1 << 3;
#[repr(C)]
#[derive(Clone, Copy)]
struct CMSampleTimingInfo {
duration: CMTime,
presentation: CMTime,
decode: CMTime,
}
#[repr(C)]
struct VTDecompressionOutputCallbackRecord {
callback: Option<
extern "C" fn(*mut c_void, *mut c_void, OSStatus, u32, CVPixelBufferRef, CMTime, CMTime),
>,
ref_con: *mut c_void,
}
#[link(name = "CoreFoundation", kind = "framework")]
unsafe extern "C" {
fn CFRelease(cf: *const c_void);
fn CFNumberCreate(alloc: CFAllocatorRef, ty: i32, value: *const c_void) -> CFNumberRef;
fn CFDictionaryCreate(
alloc: CFAllocatorRef,
keys: *const *const c_void,
values: *const *const c_void,
count: isize,
key_cb: *const c_void,
value_cb: *const c_void,
) -> CFDictionaryRef;
static kCFAllocatorNull: CFAllocatorRef;
static kCFTypeDictionaryKeyCallBacks: c_void;
static kCFTypeDictionaryValueCallBacks: c_void;
}
#[link(name = "CoreMedia", kind = "framework")]
unsafe extern "C" {
fn CMVideoFormatDescriptionCreateFromH264ParameterSets(
alloc: CFAllocatorRef,
count: usize,
pointers: *const *const u8,
sizes: *const usize,
nal_header_len: i32,
out: *mut CMFormatDescriptionRef,
) -> OSStatus;
fn CMVideoFormatDescriptionCreateFromHEVCParameterSets(
alloc: CFAllocatorRef,
count: usize,
pointers: *const *const u8,
sizes: *const usize,
nal_header_len: i32,
extensions: CFDictionaryRef,
out: *mut CMFormatDescriptionRef,
) -> OSStatus;
fn CMBlockBufferCreateWithMemoryBlock(
alloc: CFAllocatorRef,
memory: *mut c_void,
length: usize,
block_alloc: CFAllocatorRef,
custom: *const c_void,
offset: usize,
data_len: usize,
flags: u32,
out: *mut CMBlockBufferRef,
) -> OSStatus;
fn CMSampleBufferCreateReady(
alloc: CFAllocatorRef,
block: CMBlockBufferRef,
format: CMFormatDescriptionRef,
num_samples: isize,
num_timings: isize,
timings: *const CMSampleTimingInfo,
num_sizes: isize,
sizes: *const usize,
out: *mut CMSampleBufferRef,
) -> OSStatus;
}
#[link(name = "VideoToolbox", kind = "framework")]
unsafe extern "C" {
fn VTDecompressionSessionCreate(
alloc: CFAllocatorRef,
format: CMFormatDescriptionRef,
spec: CFDictionaryRef,
dest_attrs: CFDictionaryRef,
output: *const VTDecompressionOutputCallbackRecord,
out: *mut VTDecompressionSessionRef,
) -> OSStatus;
fn VTDecompressionSessionDecodeFrame(
session: VTDecompressionSessionRef,
sample: CMSampleBufferRef,
flags: u32,
source_ref_con: *mut c_void,
info_out: *mut u32,
) -> OSStatus;
fn VTDecompressionSessionWaitForAsynchronousFrames(
session: VTDecompressionSessionRef,
) -> OSStatus;
fn VTDecompressionSessionInvalidate(session: VTDecompressionSessionRef);
}
#[link(name = "CoreVideo", kind = "framework")]
unsafe extern "C" {
static kCVPixelBufferPixelFormatTypeKey: CFStringRef;
fn CVPixelBufferLockBaseAddress(pb: CVPixelBufferRef, flags: u64) -> i32;
fn CVPixelBufferUnlockBaseAddress(pb: CVPixelBufferRef, flags: u64) -> i32;
fn CVPixelBufferGetBaseAddress(pb: CVPixelBufferRef) -> *mut c_void;
fn CVPixelBufferGetBytesPerRow(pb: CVPixelBufferRef) -> usize;
fn CVPixelBufferGetWidth(pb: CVPixelBufferRef) -> usize;
fn CVPixelBufferGetHeight(pb: CVPixelBufferRef) -> usize;
}
fn cfnum(v: i32) -> CFNumberRef {
unsafe { CFNumberCreate(ptr::null(), K_CF_NUMBER_SINT32, &v as *const i32 as *const c_void) }
}
#[derive(Default)]
struct Sink {
ready: Vec<(i64, Vec<u8>)>,
err: Option<String>,
}
extern "C" fn on_decoded(
ref_con: *mut c_void,
_source: *mut c_void,
status: OSStatus,
_flags: u32,
image: CVPixelBufferRef,
pts: CMTime,
_dur: CMTime,
) {
if ref_con.is_null() {
return;
}
let shared = unsafe { &*(ref_con as *const Mutex<Sink>) };
let Ok(mut sink) = shared.lock() else { return };
if status != 0 {
sink.err = Some(format!("decode callback status {status}"));
return;
}
if image.is_null() {
return;
}
unsafe {
if CVPixelBufferLockBaseAddress(image, 1) != 0 {
sink.err = Some("could not lock the decoded pixel buffer".into());
return;
}
let w = CVPixelBufferGetWidth(image);
let h = CVPixelBufferGetHeight(image);
let stride = CVPixelBufferGetBytesPerRow(image);
let base = CVPixelBufferGetBaseAddress(image) as *const u8;
if !base.is_null() && w > 0 && h > 0 {
let mut frame = vec![0u8; w * h * 4];
for y in 0..h {
let src = base.add(y * stride);
let dst = frame.as_mut_ptr().add(y * w * 4);
ptr::copy_nonoverlapping(src, dst, w * 4);
}
sink.ready.push((pts.value, frame));
}
CVPixelBufferUnlockBaseAddress(image, 1);
}
}
fn split_avcc(cfg: &[u8]) -> Result<(Vec<(usize, usize)>, i32), String> {
if cfg.len() < 7 {
return Err("avcC too short".into());
}
let nal_len = (cfg[4] & 0x03) as i32 + 1;
let mut sets = Vec::new();
let mut at = 5usize;
for round in 0..2 {
let count = if round == 0 {
let n = (cfg[at] & 0x1F) as usize;
at += 1;
n
} else {
if at >= cfg.len() {
break;
}
let n = cfg[at] as usize;
at += 1;
n
};
for _ in 0..count {
if at + 2 > cfg.len() {
return Err("avcC ended inside a parameter set".into());
}
let len = u16::from_be_bytes([cfg[at], cfg[at + 1]]) as usize;
at += 2;
if at + len > cfg.len() {
return Err("avcC parameter set runs past the box".into());
}
sets.push((at, len));
at += len;
}
}
if sets.is_empty() {
return Err("avcC carried no parameter sets".into());
}
Ok((sets, nal_len))
}
fn split_hvcc(cfg: &[u8]) -> Result<(Vec<(usize, usize)>, i32), String> {
if cfg.len() < 23 {
return Err("hvcC too short".into());
}
let nal_len = (cfg[21] & 0x03) as i32 + 1;
let arrays = cfg[22] as usize;
let mut sets = Vec::new();
let mut at = 23usize;
for _ in 0..arrays {
if at + 3 > cfg.len() {
return Err("hvcC ended inside an array header".into());
}
let count = u16::from_be_bytes([cfg[at + 1], cfg[at + 2]]) as usize;
at += 3;
for _ in 0..count {
if at + 2 > cfg.len() {
return Err("hvcC ended inside a parameter set".into());
}
let len = u16::from_be_bytes([cfg[at], cfg[at + 1]]) as usize;
at += 2;
if at + len > cfg.len() {
return Err("hvcC parameter set runs past the box".into());
}
sets.push((at, len));
at += len;
}
}
if sets.is_empty() {
return Err("hvcC carried no parameter sets".into());
}
Ok((sets, nal_len))
}
pub struct VideoToolboxDecoder {
session: VTDecompressionSessionRef,
format: CMFormatDescriptionRef,
sink: Box<Mutex<Sink>>,
width: u32,
height: u32,
scratch: Vec<u8>,
timescale: u32,
}
impl VideoToolboxDecoder {
pub fn new(track: &VideoTrack) -> Result<Self, String> {
let (sets, nal_len) = match track.codec {
TrackCodec::H264 => split_avcc(&track.config)?,
TrackCodec::Hevc => split_hvcc(&track.config)?,
TrackCodec::Vp9 | TrackCodec::Av1 => {
return Err(format!("VideoToolbox decoding of {:?} is not wired up", track.codec))
}
};
let pointers: Vec<*const u8> =
sets.iter().map(|(o, _)| track.config[*o..].as_ptr()).collect();
let sizes: Vec<usize> = sets.iter().map(|(_, l)| *l).collect();
let mut format: CMFormatDescriptionRef = ptr::null();
let st = unsafe {
match track.codec {
TrackCodec::H264 => CMVideoFormatDescriptionCreateFromH264ParameterSets(
ptr::null(),
pointers.len(),
pointers.as_ptr(),
sizes.as_ptr(),
nal_len,
&mut format,
),
TrackCodec::Vp9 | TrackCodec::Av1 => unreachable!("refused above"),
TrackCodec::Hevc => CMVideoFormatDescriptionCreateFromHEVCParameterSets(
ptr::null(),
pointers.len(),
pointers.as_ptr(),
sizes.as_ptr(),
nal_len,
ptr::null(),
&mut format,
),
}
};
if st != 0 || format.is_null() {
return Err(format!("could not build a format description: status {st}"));
}
let sink = Box::new(Mutex::new(Sink::default()));
let mut session: VTDecompressionSessionRef = ptr::null_mut();
let st = unsafe {
let key = kCVPixelBufferPixelFormatTypeKey;
let val = cfnum(PIXEL_FORMAT_BGRA);
let attrs = CFDictionaryCreate(
ptr::null(),
&key as *const CFStringRef,
&val as *const CFNumberRef,
1,
&kCFTypeDictionaryKeyCallBacks as *const c_void,
&kCFTypeDictionaryValueCallBacks as *const c_void,
);
let record = VTDecompressionOutputCallbackRecord {
callback: Some(on_decoded),
ref_con: &*sink as *const Mutex<Sink> as *mut c_void,
};
let st = VTDecompressionSessionCreate(
ptr::null(),
format,
ptr::null(),
attrs,
&record,
&mut session,
);
CFRelease(val);
CFRelease(attrs);
st
};
if st != 0 || session.is_null() {
unsafe { CFRelease(format) };
return Err(format!("VTDecompressionSessionCreate failed: status {st}"));
}
Ok(Self {
session,
format,
sink,
width: track.width,
height: track.height,
scratch: Vec::new(),
timescale: track.timescale,
})
}
pub fn size(&self) -> (u32, u32) {
(self.width, self.height)
}
pub fn push(&mut self, sample: &[u8], pts: i64, dts: i64, duration: i64) -> Result<(), String> {
self.scratch.clear();
self.scratch.extend_from_slice(sample);
{
let mut s = self.sink.lock().map_err(|_| "decoder sink poisoned")?;
s.err = None;
}
let mut block: CMBlockBufferRef = ptr::null_mut();
let st = unsafe {
CMBlockBufferCreateWithMemoryBlock(
ptr::null(),
self.scratch.as_mut_ptr() as *mut c_void,
self.scratch.len(),
kCFAllocatorNull,
ptr::null(),
0,
self.scratch.len(),
0,
&mut block,
)
};
if st != 0 || block.is_null() {
return Err(format!("CMBlockBufferCreateWithMemoryBlock: status {st}"));
}
let timing = CMSampleTimingInfo {
duration: CMTime::new(duration, self.timescale),
presentation: CMTime::new(pts, self.timescale),
decode: CMTime::new(dts, self.timescale),
};
let mut sbuf: CMSampleBufferRef = ptr::null_mut();
let sizes = [self.scratch.len()];
let st = unsafe {
CMSampleBufferCreateReady(
ptr::null(),
block,
self.format,
1,
1,
&timing,
1,
sizes.as_ptr(),
&mut sbuf,
)
};
unsafe { CFRelease(block as *const c_void) };
if st != 0 || sbuf.is_null() {
return Err(format!("CMSampleBufferCreateReady: status {st}"));
}
let st = unsafe {
let mut info = 0u32;
let st = VTDecompressionSessionDecodeFrame(
self.session,
sbuf,
DECODE_ENABLE_TEMPORAL,
ptr::null_mut(),
&mut info,
);
if st == 0 {
VTDecompressionSessionWaitForAsynchronousFrames(self.session);
}
CFRelease(sbuf as *const c_void);
st
};
if st != 0 {
return Err(format!("VTDecompressionSessionDecodeFrame: status {st}"));
}
let s = self.sink.lock().map_err(|_| "decoder sink poisoned")?;
if let Some(e) = &s.err {
return Err(e.clone());
}
Ok(())
}
pub fn take(&mut self, pts: i64) -> Option<Vec<u8>> {
let mut sink = self.sink.lock().ok()?;
let hit = sink.ready.iter().position(|(p, _)| *p == pts)?;
let (_, frame) = sink.ready.remove(hit);
sink.ready.retain(|(p, _)| *p > pts);
Some(frame)
}
pub fn queued(&self) -> usize {
self.sink.lock().map(|s| s.ready.len()).unwrap_or(0)
}
}
impl Drop for VideoToolboxDecoder {
fn drop(&mut self) {
unsafe {
if !self.session.is_null() {
VTDecompressionSessionInvalidate(self.session);
CFRelease(self.session as *const c_void);
}
if !self.format.is_null() {
CFRelease(self.format);
}
}
}
}
unsafe impl Send for VideoToolboxDecoder {}