#![cfg(feature = "ffmpeg")]
use anyhow::{Result, anyhow};
use bytes::Bytes;
use std::collections::VecDeque;
use ffmpeg::codec::{self, Context as CodecContext, decoder, packet};
use ffmpeg::ffi as sys;
use ffmpeg::format::Pixel;
use ffmpeg::software::scaling::{context::Context as Scaler, flag::Flags as ScalerFlags};
use ffmpeg::util::frame::video::Video as VideoFrameFfmpeg;
use ffmpeg_next as ffmpeg;
use super::Decoder;
use crate::frame::{ColorSpace, PixelFormat, StreamInfo, VideoFrame};
#[cfg(target_os = "macos")]
const HWACCEL_PREFERENCE: &[&str] = &["videotoolbox", "vulkan"];
#[cfg(all(not(target_os = "macos"), target_os = "windows"))]
const HWACCEL_PREFERENCE: &[&str] = &["vulkan", "cuda", "d3d11va", "dxva2"];
#[cfg(all(not(target_os = "macos"), not(target_os = "windows")))]
const HWACCEL_PREFERENCE: &[&str] = &["vulkan", "cuda", "vaapi"];
struct HwDeviceCtx {
ptr: *mut sys::AVBufferRef,
#[allow(dead_code)]
device_type: sys::AVHWDeviceType,
}
unsafe impl Send for HwDeviceCtx {}
impl Drop for HwDeviceCtx {
fn drop(&mut self) {
unsafe {
if !self.ptr.is_null() {
sys::av_buffer_unref(&mut self.ptr);
}
}
}
}
fn hwdevice_type_from_name(name: &str) -> Option<sys::AVHWDeviceType> {
use sys::AVHWDeviceType::*;
Some(match name {
"vulkan" => AV_HWDEVICE_TYPE_VULKAN,
"cuda" => AV_HWDEVICE_TYPE_CUDA,
"d3d11va" => AV_HWDEVICE_TYPE_D3D11VA,
"dxva2" => AV_HWDEVICE_TYPE_DXVA2,
"vaapi" => AV_HWDEVICE_TYPE_VAAPI,
"videotoolbox" => AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
"qsv" => AV_HWDEVICE_TYPE_QSV,
_ => return None,
})
}
unsafe fn codec_advertises_hwaccel(
codec: *const sys::AVCodec,
device_type: sys::AVHWDeviceType,
) -> bool {
let mut i: i32 = 0;
loop {
let cfg = sys::avcodec_get_hw_config(codec, i);
if cfg.is_null() {
return false;
}
let cfg_ref = &*cfg;
let methods_ok =
(cfg_ref.methods & sys::AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX as i32) != 0;
if methods_ok && cfg_ref.device_type == device_type {
return true;
}
i += 1;
}
}
fn try_open_hwaccel(codec: *const sys::AVCodec) -> Option<HwDeviceCtx> {
let override_name = std::env::var("FFMPEG_HWACCEL").ok();
if override_name.as_deref() == Some("none") {
return None;
}
let preference: Vec<&str> = match override_name.as_deref() {
Some(name) => vec![name],
None => HWACCEL_PREFERENCE.iter().copied().collect(),
};
for name in preference {
let Some(device_type) = hwdevice_type_from_name(name) else {
continue;
};
unsafe {
if !codec_advertises_hwaccel(codec, device_type) {
tracing::debug!(hwaccel = name, "codec does not advertise this hwaccel");
continue;
}
let mut ctx: *mut sys::AVBufferRef = std::ptr::null_mut();
let rc = sys::av_hwdevice_ctx_create(
&mut ctx,
device_type,
std::ptr::null(), std::ptr::null_mut(),
0,
);
if rc == 0 && !ctx.is_null() {
tracing::info!(hwaccel = name, "FFmpeg HW device created");
return Some(HwDeviceCtx {
ptr: ctx,
device_type,
});
} else {
tracing::debug!(
hwaccel = name,
rc = rc,
"av_hwdevice_ctx_create failed; trying next"
);
}
}
}
tracing::info!("no FFmpeg hwaccel available; falling back to software decode");
None
}
fn init_ffmpeg() {
static INIT: std::sync::OnceLock<()> = std::sync::OnceLock::new();
INIT.get_or_init(|| {
let _ = ffmpeg::init();
ffmpeg::util::log::set_level(ffmpeg::util::log::Level::Warning);
});
}
fn codec_id_from_label(codec_lower: &str) -> Option<codec::Id> {
use codec::Id::*;
Some(match codec_lower {
"h264" | "avc1" | "avc" => H264,
"h265" | "hevc" | "hvc1" | "hev1" | "hvc2" | "hev2" => HEVC,
"vp8" => VP8,
"vp9" | "vp09" => VP9,
"av1" | "av01" => AV1,
"mpeg2" | "mpeg2video" => MPEG2VIDEO,
"mpeg4" | "mp4v" => MPEG4,
"prores" => PRORES,
_ => return None,
})
}
pub struct FfmpegDecoder {
info: StreamInfo,
decoder: decoder::Video,
decoded: VideoFrameFfmpeg,
hw_transfer: VideoFrameFfmpeg,
scaler: Option<Scaler>,
target_pix_fmt: Pixel,
pending_frames: VecDeque<VideoFrame>,
frame_counter: u64,
done: bool,
#[allow(dead_code)]
hw_device: Option<HwDeviceCtx>,
hwaccel_name: &'static str,
}
impl FfmpegDecoder {
pub fn new(info: StreamInfo) -> Result<Self> {
init_ffmpeg();
let codec_lower = info.codec.to_ascii_lowercase();
let codec_id = codec_id_from_label(&codec_lower)
.ok_or_else(|| anyhow!("FFmpeg: no codec_id mapped for '{codec_lower}'"))?;
let ff_codec = decoder::find(codec_id).ok_or_else(|| {
anyhow!(
"FFmpeg: decoder for {codec_id:?} not present in this libavcodec build — \
rebuild FFmpeg with the relevant --enable-decoder flag"
)
})?;
let mut ctx = CodecContext::new_with_codec(ff_codec);
let (hw_device, hwaccel_name) = unsafe {
let codec_ptr = ff_codec.as_ptr() as *const sys::AVCodec;
match try_open_hwaccel(codec_ptr) {
Some(hw) => {
let raw_ctx = ctx.as_mut_ptr();
(*raw_ctx).hw_device_ctx = sys::av_buffer_ref(hw.ptr);
let name = match hw.device_type {
sys::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN => "vulkan",
sys::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA => "cuda",
sys::AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA => "d3d11va",
sys::AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2 => "dxva2",
sys::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI => "vaapi",
sys::AVHWDeviceType::AV_HWDEVICE_TYPE_QSV => "qsv",
sys::AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX => "videotoolbox",
_ => "other",
};
(Some(hw), name)
}
None => (None, "none"),
}
};
let mut dec = ctx
.decoder()
.video()
.map_err(|e| anyhow!("FFmpeg: decoder().video() failed: {e}"))?;
dec.set_flags(codec::Flags::empty());
dec.set_threading(codec::threading::Config {
kind: codec::threading::Type::Frame,
count: 0, #[cfg(not(feature = "ffmpeg_6_0"))]
safe: true,
});
let target_pix_fmt = match info.pixel_format {
PixelFormat::Yuv420p10le => Pixel::YUV420P10LE,
_ => Pixel::YUV420P,
};
tracing::info!(
codec = %codec_lower,
hwaccel = hwaccel_name,
width = info.width,
height = info.height,
"FFmpeg decoder opened"
);
Ok(Self {
info,
decoder: dec,
decoded: VideoFrameFfmpeg::empty(),
hw_transfer: VideoFrameFfmpeg::empty(),
scaler: None,
target_pix_fmt,
pending_frames: VecDeque::new(),
frame_counter: 0,
done: false,
hw_device,
hwaccel_name,
})
}
pub fn hwaccel_engaged(&self) -> &'static str {
self.hwaccel_name
}
fn ensure_scaler(&mut self, src_w: u32, src_h: u32, src_fmt: Pixel) -> Result<()> {
let needs_rebuild = match self.scaler.as_ref() {
None => true,
Some(s) => {
s.input().width != src_w || s.input().height != src_h || s.input().format != src_fmt
}
};
if needs_rebuild {
self.scaler = Some(
Scaler::get(
src_fmt,
src_w,
src_h,
self.target_pix_fmt,
self.info.width,
self.info.height,
ScalerFlags::BILINEAR,
)
.map_err(|e| anyhow!("FFmpeg: sws_scale ctx: {e}"))?,
);
}
Ok(())
}
fn drain_decoded(&mut self) -> Result<()> {
loop {
match self.decoder.receive_frame(&mut self.decoded) {
Ok(()) => {
let decoded_has_hw_ctx = unsafe {
let raw = self.decoded.as_ptr();
!raw.is_null() && !(*raw).hw_frames_ctx.is_null()
};
let src_frame: &VideoFrameFfmpeg = if decoded_has_hw_ctx {
unsafe {
let rc = sys::av_hwframe_transfer_data(
self.hw_transfer.as_mut_ptr(),
self.decoded.as_ptr(),
0,
);
if rc < 0 {
return Err(anyhow!(
"FFmpeg: av_hwframe_transfer_data failed (rc={rc})"
));
}
}
&self.hw_transfer
} else {
&self.decoded
};
let src_w = src_frame.width();
let src_h = src_frame.height();
let src_fmt = src_frame.format();
if src_w == 0 || src_h == 0 {
continue;
}
self.ensure_scaler(src_w, src_h, src_fmt)?;
let mut scaled = VideoFrameFfmpeg::empty();
self.scaler
.as_mut()
.unwrap()
.run(src_frame, &mut scaled)
.map_err(|e| anyhow!("FFmpeg: sws_scale run: {e}"))?;
let frame = ffmpeg_frame_to_video_frame(
&scaled,
&self.info,
self.target_pix_fmt,
self.frame_counter,
)?;
self.frame_counter += 1;
self.pending_frames.push_back(frame);
}
Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::util::error::EAGAIN => {
return Ok(());
}
Err(ffmpeg::Error::Eof) => {
self.done = true;
return Ok(());
}
Err(e) => {
return Err(anyhow!("FFmpeg: receive_frame: {e}"));
}
}
}
}
}
fn ffmpeg_frame_to_video_frame(
src: &VideoFrameFfmpeg,
info: &StreamInfo,
target_fmt: Pixel,
pts: u64,
) -> Result<VideoFrame> {
let w = src.width() as usize;
let h = src.height() as usize;
let (bytes_per_sample, out_fmt) = match target_fmt {
Pixel::YUV420P10LE => (2, PixelFormat::Yuv420p10le),
_ => (1, PixelFormat::Yuv420p),
};
let y_len = w * h * bytes_per_sample;
let uv_w = w / 2;
let uv_h = h / 2;
let uv_len = uv_w * uv_h * bytes_per_sample;
let mut data = Vec::with_capacity(y_len + 2 * uv_len);
for plane in 0..3 {
let pw = if plane == 0 { w } else { uv_w };
let ph = if plane == 0 { h } else { uv_h };
let stride = src.stride(plane) as usize;
let plane_bytes = src.data(plane);
for row in 0..ph {
let row_start = row * stride;
let row_end = row_start + pw * bytes_per_sample;
if row_end > plane_bytes.len() {
return Err(anyhow!(
"FFmpeg frame plane {plane} row {row} exceeds buffer"
));
}
data.extend_from_slice(&plane_bytes[row_start..row_end]);
}
}
Ok(VideoFrame::new(
Bytes::from(data),
info.width,
info.height,
out_fmt,
info.color_space,
pts,
))
}
impl Decoder for FfmpegDecoder {
fn stream_info(&self) -> &StreamInfo {
&self.info
}
fn push_sample(&mut self, data: &[u8]) -> Result<()> {
if self.done {
return Err(anyhow!("FFmpeg: push_sample after finish"));
}
let mut pkt = packet::Packet::copy(data);
pkt.set_pts(None);
pkt.set_dts(None);
self.decoder
.send_packet(&pkt)
.map_err(|e| anyhow!("FFmpeg: send_packet: {e}"))?;
self.drain_decoded()?;
Ok(())
}
fn finish(&mut self) -> Result<()> {
if self.done {
return Ok(());
}
self.decoder
.send_eof()
.map_err(|e| anyhow!("FFmpeg: send_eof: {e}"))?;
self.drain_decoded()?;
self.done = true;
Ok(())
}
fn decode_next(&mut self) -> Result<Option<VideoFrame>> {
if self.pending_frames.is_empty() && self.done {
self.drain_decoded()?;
}
Ok(self.pending_frames.pop_front())
}
}
#[allow(unused_imports)]
use ffmpeg_next::util::error as _ff_error;
#[cfg(test)]
mod tests {
use super::*;
use crate::frame::{ColorMetadata, ColorSpace, PixelFormat};
fn test_info() -> StreamInfo {
StreamInfo {
codec: "h264".to_string(),
width: 320,
height: 176,
frame_rate: 24.0,
duration: 0.0,
pixel_format: PixelFormat::Yuv420p,
color_space: ColorSpace::Bt709,
total_frames: 0,
bitrate: 0,
color_metadata: ColorMetadata::default(),
}
}
#[test]
fn codec_id_mapping_covers_mainstream() {
assert!(codec_id_from_label("h264").is_some());
assert!(codec_id_from_label("hevc").is_some());
assert!(codec_id_from_label("av1").is_some());
assert!(codec_id_from_label("vp9").is_some());
assert!(codec_id_from_label("vp8").is_some());
assert!(codec_id_from_label("mpeg2").is_some());
assert!(codec_id_from_label("mpeg4").is_some());
assert!(codec_id_from_label("prores").is_some());
assert!(codec_id_from_label("unknown").is_none());
}
#[test]
fn construct_h264_decoder() {
match FfmpegDecoder::new(test_info()) {
Ok(dec) => {
assert_eq!(dec.stream_info().codec, "h264");
let name = dec.hwaccel_engaged();
eprintln!("hwaccel engaged: {name}");
assert!(matches!(
name,
"vulkan"
| "cuda"
| "d3d11va"
| "dxva2"
| "vaapi"
| "qsv"
| "videotoolbox"
| "other"
| "none"
));
}
Err(e) => {
eprintln!("skip: FFmpeg H.264 decoder construct failed: {e}");
}
}
}
#[test]
fn hwaccel_override_none_forces_software() {
unsafe {
std::env::set_var("FFMPEG_HWACCEL", "none");
}
match FfmpegDecoder::new(test_info()) {
Ok(dec) => assert_eq!(dec.hwaccel_engaged(), "none"),
Err(e) => eprintln!("skip: construct failed: {e}"),
}
unsafe {
std::env::remove_var("FFMPEG_HWACCEL");
}
}
}