#![cfg(all(feature = "videotoolbox", target_os = "macos"))]
use std::ffi::c_void;
use std::ptr;
use std::sync::Mutex;
use crate::decode::{ColorInfo, DecodedFrame, PackedFrame};
use crate::mp4_demux::{TrackCodec, VideoTrack};
type OSStatus = i32;
type CFAllocatorRef = *const c_void;
type CFDictionaryRef = *const c_void;
type CFStringRef = *const c_void;
type CFNumberRef = *const c_void;
type CVPixelBufferRef = *mut c_void;
type CMSampleBufferRef = *mut c_void;
type CMBlockBufferRef = *mut c_void;
type CMFormatDescriptionRef = *const c_void;
type VTDecompressionSessionRef = *mut c_void;
const K_CF_NUMBER_SINT32: i32 = 3;
const PIXEL_FORMAT_BGRA: i32 = 0x4247_5241;
const PIXEL_FORMAT_NV12: i32 = 0x3432_3076;
#[repr(C)]
#[derive(Clone, Copy)]
struct CMTime {
value: i64,
timescale: i32,
flags: u32,
epoch: i64,
}
impl CMTime {
fn new(value: i64, timescale: u32) -> Self {
CMTime {
value,
timescale: timescale as i32,
flags: 1, epoch: 0,
}
}
}
const DECODE_ENABLE_TEMPORAL: u32 = 1 << 3;
#[repr(C)]
#[derive(Clone, Copy)]
struct CMSampleTimingInfo {
duration: CMTime,
presentation: CMTime,
decode: CMTime,
}
#[repr(C)]
struct VTDecompressionOutputCallbackRecord {
callback: Option<
extern "C" fn(*mut c_void, *mut c_void, OSStatus, u32, CVPixelBufferRef, CMTime, CMTime),
>,
ref_con: *mut c_void,
}
#[link(name = "CoreFoundation", kind = "framework")]
unsafe extern "C" {
fn CFRelease(cf: *const c_void);
fn CFNumberCreate(alloc: CFAllocatorRef, ty: i32, value: *const c_void) -> CFNumberRef;
fn CFDictionaryCreate(
alloc: CFAllocatorRef,
keys: *const *const c_void,
values: *const *const c_void,
count: isize,
key_cb: *const c_void,
value_cb: *const c_void,
) -> CFDictionaryRef;
fn CFDataCreate(alloc: CFAllocatorRef, bytes: *const u8, len: isize) -> *const c_void;
fn CFStringCreateWithCString(
alloc: CFAllocatorRef,
s: *const std::ffi::c_char,
encoding: u32,
) -> CFStringRef;
static kCFAllocatorNull: CFAllocatorRef;
static kCFTypeDictionaryKeyCallBacks: c_void;
static kCFTypeDictionaryValueCallBacks: c_void;
}
#[link(name = "CoreMedia", kind = "framework")]
unsafe extern "C" {
fn CMVideoFormatDescriptionCreateFromH264ParameterSets(
alloc: CFAllocatorRef,
count: usize,
pointers: *const *const u8,
sizes: *const usize,
nal_header_len: i32,
out: *mut CMFormatDescriptionRef,
) -> OSStatus;
fn CMVideoFormatDescriptionCreateFromHEVCParameterSets(
alloc: CFAllocatorRef,
count: usize,
pointers: *const *const u8,
sizes: *const usize,
nal_header_len: i32,
extensions: CFDictionaryRef,
out: *mut CMFormatDescriptionRef,
) -> OSStatus;
fn CMVideoFormatDescriptionCreate(
alloc: CFAllocatorRef,
codec: u32,
width: i32,
height: i32,
extensions: CFDictionaryRef,
out: *mut CMFormatDescriptionRef,
) -> OSStatus;
static kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms: CFStringRef;
fn CMBlockBufferCreateWithMemoryBlock(
alloc: CFAllocatorRef,
memory: *mut c_void,
length: usize,
block_alloc: CFAllocatorRef,
custom: *const c_void,
offset: usize,
data_len: usize,
flags: u32,
out: *mut CMBlockBufferRef,
) -> OSStatus;
fn CMSampleBufferCreateReady(
alloc: CFAllocatorRef,
block: CMBlockBufferRef,
format: CMFormatDescriptionRef,
num_samples: isize,
num_timings: isize,
timings: *const CMSampleTimingInfo,
num_sizes: isize,
sizes: *const usize,
out: *mut CMSampleBufferRef,
) -> OSStatus;
}
#[link(name = "VideoToolbox", kind = "framework")]
unsafe extern "C" {
fn VTRegisterSupplementalVideoDecoderIfAvailable(codec: u32);
fn VTDecompressionSessionCreate(
alloc: CFAllocatorRef,
format: CMFormatDescriptionRef,
spec: CFDictionaryRef,
dest_attrs: CFDictionaryRef,
output: *const VTDecompressionOutputCallbackRecord,
out: *mut VTDecompressionSessionRef,
) -> OSStatus;
fn VTDecompressionSessionDecodeFrame(
session: VTDecompressionSessionRef,
sample: CMSampleBufferRef,
flags: u32,
source_ref_con: *mut c_void,
info_out: *mut u32,
) -> OSStatus;
fn VTDecompressionSessionWaitForAsynchronousFrames(
session: VTDecompressionSessionRef,
) -> OSStatus;
fn VTDecompressionSessionInvalidate(session: VTDecompressionSessionRef);
}
#[link(name = "CoreVideo", kind = "framework")]
unsafe extern "C" {
static kCVPixelBufferPixelFormatTypeKey: CFStringRef;
fn CVPixelBufferLockBaseAddress(pb: CVPixelBufferRef, flags: u64) -> i32;
fn CVPixelBufferUnlockBaseAddress(pb: CVPixelBufferRef, flags: u64) -> i32;
fn CVPixelBufferGetBaseAddress(pb: CVPixelBufferRef) -> *mut c_void;
fn CVPixelBufferGetBytesPerRow(pb: CVPixelBufferRef) -> usize;
fn CVPixelBufferGetWidth(pb: CVPixelBufferRef) -> usize;
fn CVPixelBufferGetHeight(pb: CVPixelBufferRef) -> usize;
fn CVPixelBufferGetPlaneCount(pb: CVPixelBufferRef) -> usize;
fn CVPixelBufferGetBaseAddressOfPlane(pb: CVPixelBufferRef, plane: usize) -> *mut c_void;
fn CVPixelBufferGetBytesPerRowOfPlane(pb: CVPixelBufferRef, plane: usize) -> usize;
fn CVPixelBufferGetWidthOfPlane(pb: CVPixelBufferRef, plane: usize) -> usize;
fn CVPixelBufferGetHeightOfPlane(pb: CVPixelBufferRef, plane: usize) -> usize;
}
fn cfnum(v: i32) -> CFNumberRef {
unsafe {
CFNumberCreate(
ptr::null(),
K_CF_NUMBER_SINT32,
&v as *const i32 as *const c_void,
)
}
}
#[derive(Default)]
struct Sink {
ready: Vec<(i64, Picture)>,
err: Option<String>,
planar: bool,
pool: crate::bytes::FramePool,
}
enum Picture {
Bgra(Vec<u8>),
Planes(PackedFrame),
}
extern "C" fn on_decoded(
ref_con: *mut c_void,
_source: *mut c_void,
status: OSStatus,
_flags: u32,
image: CVPixelBufferRef,
pts: CMTime,
_dur: CMTime,
) {
if ref_con.is_null() {
return;
}
let shared = unsafe { &*(ref_con as *const Mutex<Sink>) };
let Ok(mut sink) = shared.lock() else { return };
if status != 0 {
sink.err = Some(format!("decode callback status {status}"));
return;
}
if image.is_null() {
return;
}
unsafe {
if CVPixelBufferLockBaseAddress(image, 1) != 0 {
sink.err = Some("could not lock the decoded pixel buffer".into());
return;
}
let w = CVPixelBufferGetWidth(image);
let h = CVPixelBufferGetHeight(image);
if sink.planar {
if let Some(frame) = planes(image, w, h, &sink.pool) {
sink.ready.push((pts.value, Picture::Planes(frame)));
}
CVPixelBufferUnlockBaseAddress(image, 1);
return;
}
let stride = CVPixelBufferGetBytesPerRow(image);
let base = CVPixelBufferGetBaseAddress(image) as *const u8;
if !base.is_null() && w > 0 && h > 0 {
let mut frame = vec![0u8; w * h * 4];
for y in 0..h {
let src = base.add(y * stride);
let dst = frame.as_mut_ptr().add(y * w * 4);
ptr::copy_nonoverlapping(src, dst, w * 4);
}
sink.ready.push((pts.value, Picture::Bgra(frame)));
}
CVPixelBufferUnlockBaseAddress(image, 1);
}
}
unsafe fn planes(
image: CVPixelBufferRef,
w: usize,
h: usize,
pool: &crate::bytes::FramePool,
) -> Option<PackedFrame> {
if w == 0 || h == 0 || unsafe { CVPixelBufferGetPlaneCount(image) } != 2 {
return None;
}
let mut out: [Vec<u8>; 2] = [
pool.take(w * h),
pool.take(w.div_ceil(2) * 2 * h.div_ceil(2)),
];
for (i, plane) in out.iter_mut().enumerate() {
let (row, rows) = if i == 0 {
(w, h)
} else {
(w.div_ceil(2) * 2, h.div_ceil(2))
};
let base = unsafe { CVPixelBufferGetBaseAddressOfPlane(image, i) } as *const u8;
let stride = unsafe { CVPixelBufferGetBytesPerRowOfPlane(image, i) };
let samples_across = if i == 0 { w } else { w.div_ceil(2) };
if base.is_null()
|| unsafe { CVPixelBufferGetWidthOfPlane(image, i) } < samples_across
|| unsafe { CVPixelBufferGetHeightOfPlane(image, i) } < rows
|| stride < row
{
return None;
}
plane.reserve(row * rows);
for y in 0..rows {
plane.extend_from_slice(unsafe {
std::slice::from_raw_parts(base.add(y * stride), row)
});
}
}
let [y, uv] = out;
Some(PackedFrame {
width: w as u32,
height: h as u32,
bit_depth: 8,
y,
uv,
color: ColorInfo::from_vui(None, false, h as u32),
})
}
fn split_avcc(cfg: &[u8]) -> Result<(Vec<(usize, usize)>, i32), String> {
if cfg.len() < 7 {
return Err("avcC too short".into());
}
let nal_len = (cfg[4] & 0x03) as i32 + 1;
let mut sets = Vec::new();
let mut at = 5usize;
for round in 0..2 {
let count = if round == 0 {
let n = (cfg[at] & 0x1F) as usize;
at += 1;
n
} else {
if at >= cfg.len() {
break;
}
let n = cfg[at] as usize;
at += 1;
n
};
for _ in 0..count {
if at + 2 > cfg.len() {
return Err("avcC ended inside a parameter set".into());
}
let len = u16::from_be_bytes([cfg[at], cfg[at + 1]]) as usize;
at += 2;
if at + len > cfg.len() {
return Err("avcC parameter set runs past the box".into());
}
sets.push((at, len));
at += len;
}
}
if sets.is_empty() {
return Err("avcC carried no parameter sets".into());
}
Ok((sets, nal_len))
}
fn split_hvcc(cfg: &[u8]) -> Result<(Vec<(usize, usize)>, i32), String> {
if cfg.len() < 23 {
return Err("hvcC too short".into());
}
let nal_len = (cfg[21] & 0x03) as i32 + 1;
let arrays = cfg[22] as usize;
let mut sets = Vec::new();
let mut at = 23usize;
for _ in 0..arrays {
if at + 3 > cfg.len() {
return Err("hvcC ended inside an array header".into());
}
let count = u16::from_be_bytes([cfg[at + 1], cfg[at + 2]]) as usize;
at += 3;
for _ in 0..count {
if at + 2 > cfg.len() {
return Err("hvcC ended inside a parameter set".into());
}
let len = u16::from_be_bytes([cfg[at], cfg[at + 1]]) as usize;
at += 2;
if at + len > cfg.len() {
return Err("hvcC parameter set runs past the box".into());
}
sets.push((at, len));
at += len;
}
}
if sets.is_empty() {
return Err("hvcC carried no parameter sets".into());
}
Ok((sets, nal_len))
}
unsafe fn atom_format(track: &VideoTrack, out: &mut CMFormatDescriptionRef) -> OSStatus {
const UTF8: u32 = 0x0800_0100;
let (codec, atom): (u32, &[u8]) = match track.codec {
TrackCodec::Av1 => (u32::from_be_bytes(*b"av01"), b"av1C\0"),
_ => {
unsafe { VTRegisterSupplementalVideoDecoderIfAvailable(u32::from_be_bytes(*b"vp09")) };
(u32::from_be_bytes(*b"vp09"), b"vpcC\0")
}
};
if track.config.is_empty() {
return -12710;
}
let key = unsafe {
CFStringCreateWithCString(ptr::null(), atom.as_ptr() as *const std::ffi::c_char, UTF8)
};
let data = unsafe {
CFDataCreate(
ptr::null(),
track.config.as_ptr(),
track.config.len() as isize,
)
};
let atoms = unsafe {
CFDictionaryCreate(
ptr::null(),
&key as *const CFStringRef,
&data as *const *const c_void,
1,
&kCFTypeDictionaryKeyCallBacks as *const c_void,
&kCFTypeDictionaryValueCallBacks as *const c_void,
)
};
let outer_key = unsafe { kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms };
let extensions = unsafe {
CFDictionaryCreate(
ptr::null(),
&outer_key as *const CFStringRef,
&atoms as *const CFDictionaryRef,
1,
&kCFTypeDictionaryKeyCallBacks as *const c_void,
&kCFTypeDictionaryValueCallBacks as *const c_void,
)
};
let st = unsafe {
CMVideoFormatDescriptionCreate(
ptr::null(),
codec,
track.width as i32,
track.height as i32,
extensions,
out,
)
};
unsafe { CFRelease(extensions) };
unsafe { CFRelease(atoms) };
unsafe { CFRelease(data) };
unsafe { CFRelease(key) };
st
}
pub struct VideoToolboxDecoder {
session: VTDecompressionSessionRef,
format: CMFormatDescriptionRef,
sink: Box<Mutex<Sink>>,
width: u32,
height: u32,
scratch: Vec<u8>,
timescale: u32,
}
impl VideoToolboxDecoder {
pub fn new(track: &VideoTrack) -> Result<Self, String> {
Self::open(track, false)
}
pub fn new_planar(track: &VideoTrack) -> Result<Self, String> {
Self::open(track, true)
}
fn open(track: &VideoTrack, planar: bool) -> Result<Self, String> {
let (sets, nal_len) = match track.codec {
TrackCodec::H264 => split_avcc(&track.config)?,
TrackCodec::Hevc => split_hvcc(&track.config)?,
TrackCodec::Vp9 | TrackCodec::Av1 => (Vec::new(), 0),
};
let pointers: Vec<*const u8> = sets
.iter()
.map(|(o, _)| track.config[*o..].as_ptr())
.collect();
let sizes: Vec<usize> = sets.iter().map(|(_, l)| *l).collect();
let mut format: CMFormatDescriptionRef = ptr::null();
let st = unsafe {
match track.codec {
TrackCodec::H264 => CMVideoFormatDescriptionCreateFromH264ParameterSets(
ptr::null(),
pointers.len(),
pointers.as_ptr(),
sizes.as_ptr(),
nal_len,
&mut format,
),
TrackCodec::Vp9 | TrackCodec::Av1 => atom_format(track, &mut format),
TrackCodec::Hevc => CMVideoFormatDescriptionCreateFromHEVCParameterSets(
ptr::null(),
pointers.len(),
pointers.as_ptr(),
sizes.as_ptr(),
nal_len,
ptr::null(),
&mut format,
),
}
};
if st != 0 || format.is_null() {
return Err(format!("could not build a format description: status {st}"));
}
let sink = Box::new(Mutex::new(Sink {
planar,
pool: Default::default(),
..Sink::default()
}));
let mut session: VTDecompressionSessionRef = ptr::null_mut();
let st = unsafe {
let key = kCVPixelBufferPixelFormatTypeKey;
let val = cfnum(if planar {
PIXEL_FORMAT_NV12
} else {
PIXEL_FORMAT_BGRA
});
let attrs = CFDictionaryCreate(
ptr::null(),
&key as *const CFStringRef,
&val as *const CFNumberRef,
1,
&kCFTypeDictionaryKeyCallBacks as *const c_void,
&kCFTypeDictionaryValueCallBacks as *const c_void,
);
let record = VTDecompressionOutputCallbackRecord {
callback: Some(on_decoded),
ref_con: &*sink as *const Mutex<Sink> as *mut c_void,
};
let st = VTDecompressionSessionCreate(
ptr::null(),
format,
ptr::null(),
attrs,
&record,
&mut session,
);
CFRelease(val);
CFRelease(attrs);
st
};
if st != 0 || session.is_null() {
unsafe { CFRelease(format) };
return Err(format!("VTDecompressionSessionCreate failed: status {st}"));
}
Ok(Self {
session,
format,
sink,
width: track.width,
height: track.height,
scratch: Vec::new(),
timescale: track.timescale,
})
}
pub fn size(&self) -> (u32, u32) {
(self.width, self.height)
}
pub fn push(&mut self, sample: &[u8], pts: i64, dts: i64, duration: i64) -> Result<(), String> {
self.scratch.clear();
self.scratch.extend_from_slice(sample);
{
let mut s = self.sink.lock().map_err(|_| "decoder sink poisoned")?;
s.err = None;
}
let mut block: CMBlockBufferRef = ptr::null_mut();
let st = unsafe {
CMBlockBufferCreateWithMemoryBlock(
ptr::null(),
self.scratch.as_mut_ptr() as *mut c_void,
self.scratch.len(),
kCFAllocatorNull,
ptr::null(),
0,
self.scratch.len(),
0,
&mut block,
)
};
if st != 0 || block.is_null() {
return Err(format!("CMBlockBufferCreateWithMemoryBlock: status {st}"));
}
let timing = CMSampleTimingInfo {
duration: CMTime::new(duration, self.timescale),
presentation: CMTime::new(pts, self.timescale),
decode: CMTime::new(dts, self.timescale),
};
let mut sbuf: CMSampleBufferRef = ptr::null_mut();
let sizes = [self.scratch.len()];
let st = unsafe {
CMSampleBufferCreateReady(
ptr::null(),
block,
self.format,
1,
1,
&timing,
1,
sizes.as_ptr(),
&mut sbuf,
)
};
unsafe { CFRelease(block as *const c_void) };
if st != 0 || sbuf.is_null() {
return Err(format!("CMSampleBufferCreateReady: status {st}"));
}
let st = unsafe {
let mut info = 0u32;
let st = VTDecompressionSessionDecodeFrame(
self.session,
sbuf,
DECODE_ENABLE_TEMPORAL,
ptr::null_mut(),
&mut info,
);
if st == 0 {
VTDecompressionSessionWaitForAsynchronousFrames(self.session);
}
CFRelease(sbuf as *const c_void);
st
};
if st != 0 {
return Err(format!("VTDecompressionSessionDecodeFrame: status {st}"));
}
let s = self.sink.lock().map_err(|_| "decoder sink poisoned")?;
if let Some(e) = &s.err {
return Err(e.clone());
}
Ok(())
}
pub fn take(&mut self, pts: i64) -> Option<Vec<u8>> {
match self.take_picture(pts)? {
Picture::Bgra(bgra) => Some(bgra),
Picture::Planes(frame) => Some(frame.to_decoded().to_bgra()),
}
}
pub fn take_frame(&mut self, pts: i64) -> Option<DecodedFrame> {
self.take_packed(pts).map(|f| f.to_decoded())
}
pub fn set_pool(&mut self, pool: crate::bytes::FramePool) {
if let Ok(mut sink) = self.sink.lock() {
sink.pool = pool;
}
}
pub fn take_packed(&mut self, pts: i64) -> Option<PackedFrame> {
match self.take_picture(pts)? {
Picture::Planes(frame) => Some(frame),
Picture::Bgra(_) => None,
}
}
fn take_picture(&mut self, pts: i64) -> Option<Picture> {
let mut sink = self.sink.lock().ok()?;
let hit = sink.ready.iter().position(|(p, _)| *p == pts)?;
let (_, frame) = sink.ready.remove(hit);
sink.ready.retain(|(p, _)| *p > pts);
Some(frame)
}
pub fn queued(&self) -> usize {
self.sink.lock().map(|s| s.ready.len()).unwrap_or(0)
}
}
impl Drop for VideoToolboxDecoder {
fn drop(&mut self) {
unsafe {
if !self.session.is_null() {
VTDecompressionSessionInvalidate(self.session);
CFRelease(self.session as *const c_void);
}
if !self.format.is_null() {
CFRelease(self.format);
}
}
}
}
unsafe impl Send for VideoToolboxDecoder {}