use super::frame::VideoFrame;
use super::options::{PixelLayout, Sampling};
use super::sampler::EMIT_COUNT_KEY;
use crate::core::filter::frame_filter::{FrameFilter, FrameFilterError, RequestFrameMode};
use crate::core::filter::frame_filter_context::FrameFilterContext;
use crate::util::ffmpeg_utils::frame_is_eof_marker;
use crate::util::frame_utils::ensure_software_format;
use crossbeam_channel::{Receiver, Sender};
use ffmpeg_next::Frame;
use ffmpeg_sys_next::{
av_dict_get, av_dict_set, av_rescale_q, AVFrame, AVMediaType, AVMediaType::AVMEDIA_TYPE_VIDEO,
AVRational, AV_FRAME_FLAG_KEY, AV_NOPTS_VALUE,
};
use std::ffi::CStr;
#[cfg(test)]
use std::sync::atomic::{AtomicU64, Ordering};
const US_PER_SEC: AVRational = AVRational {
num: 1,
den: 1_000_000,
};
struct BufferPool {
tx: Sender<Vec<u8>>,
rx: Receiver<Vec<u8>>,
#[cfg(test)]
reuses: AtomicU64,
#[cfg(test)]
allocs: AtomicU64,
}
impl BufferPool {
fn new(slots: usize) -> Self {
let (tx, rx) = crossbeam_channel::bounded(slots.max(1));
Self {
tx,
rx,
#[cfg(test)]
reuses: AtomicU64::new(0),
#[cfg(test)]
allocs: AtomicU64::new(0),
}
}
fn recycler(&self) -> Sender<Vec<u8>> {
self.tx.clone()
}
fn take(&self, total: usize) -> Vec<u8> {
let max_keep = total.saturating_mul(4);
while let Ok(mut buf) = self.rx.try_recv() {
if buf.capacity() >= total && buf.capacity() <= max_keep {
buf.clear();
#[cfg(test)]
self.reuses.fetch_add(1, Ordering::Relaxed);
return buf;
}
}
#[cfg(test)]
self.allocs.fetch_add(1, Ordering::Relaxed);
Vec::with_capacity(total)
}
#[cfg(test)]
fn reuse_count(&self) -> u64 {
self.reuses.load(Ordering::Relaxed)
}
#[cfg(test)]
fn alloc_count(&self) -> u64 {
self.allocs.load(Ordering::Relaxed)
}
}
pub(crate) struct ExportSink {
tx: Sender<VideoFrame>,
sampling: Sampling,
layout: PixelLayout,
max_frames: Option<u64>,
uniform: bool,
emitted: u64,
done: bool,
pool: BufferPool,
}
impl ExportSink {
pub(crate) fn new(
tx: Sender<VideoFrame>,
sampling: Sampling,
layout: PixelLayout,
max_frames: Option<u64>,
) -> Self {
let uniform = matches!(sampling, Sampling::UniformN(_));
let pool = BufferPool::new(tx.capacity().unwrap_or(1).saturating_add(2));
Self {
tx,
sampling,
layout,
max_frames,
uniform,
emitted: 0,
done: false,
pool,
}
}
unsafe fn frame_pts_us(p: *const AVFrame) -> Option<i64> {
let tb = (*p).time_base;
if tb.den == 0 {
return None;
}
let pts = (*p).pts;
if pts == AV_NOPTS_VALUE {
return None;
}
Some(av_rescale_q(pts, tb, US_PER_SEC))
}
unsafe fn select(&self, p: *const AVFrame) -> bool {
match self.sampling {
Sampling::KeyframesOnly => (*p).flags & AV_FRAME_FLAG_KEY != 0,
_ => true,
}
}
fn deliver(&mut self, w: u32, h: u32, pts_us: Option<i64>, data: Vec<u8>) -> bool {
if let Some(max) = self.max_frames {
if self.emitted >= max {
self.done = true;
return false;
}
}
let vf = VideoFrame::new(
w,
h,
self.layout,
pts_us,
self.emitted,
data,
Some(self.pool.recycler()),
);
match self.tx.send(vf) {
Ok(()) => {
self.emitted += 1;
true
}
Err(_) => {
self.done = true;
false
}
}
}
}
impl FrameFilter for ExportSink {
fn media_type(&self) -> AVMediaType {
AVMEDIA_TYPE_VIDEO
}
fn request_frame_mode(&self) -> RequestFrameMode {
RequestFrameMode::Never
}
fn filter_frame(
&mut self,
mut frame: Frame,
_ctx: &mut FrameFilterContext,
) -> Result<Option<Frame>, FrameFilterError> {
if frame_is_eof_marker(&frame) {
return Ok(Some(frame));
}
let p = unsafe { frame.as_ptr() };
if p.is_null() {
return Ok(Some(frame));
}
if self.done {
if self.uniform {
let _ = unsafe { read_and_strip_emit_count(&mut frame) };
}
return Ok(Some(frame));
}
let pts_us = unsafe { Self::frame_pts_us(p) };
if self.uniform {
let count = unsafe { read_and_strip_emit_count(&mut frame) };
let count = match self.sampling {
Sampling::UniformN(n) => count.min((n as u64).saturating_sub(self.emitted)),
_ => count,
};
if count == 0 {
return Ok(Some(frame));
}
let (w, h, mut bytes) = unsafe { pack_bytes(p, self.layout, &self.pool)? };
let mut remaining = count;
while remaining > 0 {
remaining -= 1;
let data = if remaining == 0 {
std::mem::take(&mut bytes)
} else {
let mut dup = self.pool.take(bytes.len());
dup.extend_from_slice(&bytes);
dup
};
if !self.deliver(w, h, pts_us, data) {
break;
}
}
return Ok(Some(frame));
}
if !unsafe { self.select(p) } {
return Ok(None);
}
let (w, h, bytes) = unsafe { pack_bytes(p, self.layout, &self.pool)? };
if self.deliver(w, h, pts_us, bytes) {
Ok(Some(frame))
} else {
Ok(Some(frame))
}
}
}
unsafe fn read_and_strip_emit_count(frame: &mut Frame) -> u64 {
let key = EMIT_COUNT_KEY;
let p = frame.as_mut_ptr();
let entry = av_dict_get((*p).metadata, key.as_ptr(), std::ptr::null(), 0);
let count = if entry.is_null() {
1
} else {
CStr::from_ptr((*entry).value)
.to_str()
.ok()
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(1)
.max(1)
};
while !av_dict_get((*p).metadata, key.as_ptr(), std::ptr::null(), 0).is_null() {
if av_dict_set(&mut (*p).metadata, key.as_ptr(), std::ptr::null(), 0) < 0 {
break;
}
}
count
}
unsafe fn pack_bytes(
p: *const AVFrame,
layout: PixelLayout,
pool: &BufferPool,
) -> Result<(u32, u32, Vec<u8>), FrameFilterError> {
ensure_software_format((*p).format)
.map_err(|e| -> FrameFilterError { format!("frame export: {e}").into() })?;
let expected = layout.av_pixel_format() as i32;
if (*p).format != expected {
return Err(format!(
"frame export: expected pixel format {expected}, got {}",
(*p).format
)
.into());
}
let width = (*p).width;
let height = (*p).height;
if width <= 0 || height <= 0 {
return Err(format!("frame export: non-positive frame dimensions {width}x{height}").into());
}
let bpp = layout.bytes_per_pixel();
let w = width as usize;
let h = height as usize;
let row_bytes = w
.checked_mul(bpp)
.ok_or_else(|| -> FrameFilterError { "frame export: row size overflow".into() })?;
let total = row_bytes
.checked_mul(h)
.ok_or_else(|| -> FrameFilterError { "frame export: frame size overflow".into() })?;
let base = (*p).data[0];
if base.is_null() {
return Err("frame export: packed plane pointer is null".into());
}
let linesize = (*p).linesize[0] as isize;
let linesize_abs = linesize.unsigned_abs();
if linesize_abs < row_bytes {
return Err(format!(
"frame export: linesize {linesize} smaller than {row_bytes} packed row bytes"
)
.into());
}
let src_footprint = h
.saturating_sub(1)
.checked_mul(linesize_abs)
.and_then(|x| x.checked_add(row_bytes))
.ok_or_else(|| -> FrameFilterError { "frame export: source footprint overflow".into() })?;
if src_footprint > isize::MAX as usize {
return Err("frame export: source footprint exceeds isize::MAX".into());
}
let mut out = pool.take(total);
debug_assert!(out.is_empty() && out.capacity() >= total);
for row in 0..h {
let src_row = base.offset(row as isize * linesize);
let src = std::slice::from_raw_parts(src_row, row_bytes);
out.extend_from_slice(src);
}
debug_assert_eq!(out.len(), total);
Ok((width as u32, height as u32, out))
}
#[cfg(test)]
mod tests {
use super::*;
use ffmpeg_sys_next::av_frame_alloc;
use ffmpeg_sys_next::AVPixelFormat::AV_PIX_FMT_RGB24;
fn flag_frame(key: bool) -> Frame {
unsafe {
let p = av_frame_alloc();
assert!(!p.is_null());
if key {
(*p).flags |= AV_FRAME_FLAG_KEY;
}
Frame::wrap(p)
}
}
#[test]
fn dropped_frame_recycles_its_buffer() {
let pool = BufferPool::new(2);
let mut first = pool.take(12);
first.extend_from_slice(&[7u8; 12]);
assert_eq!(
(pool.alloc_count(), pool.reuse_count()),
(1, 0),
"the first take has nothing to reuse"
);
let vf = VideoFrame::new(
2,
2,
PixelLayout::Rgb24,
None,
0,
first,
Some(pool.recycler()),
);
assert_eq!(pool.rx.len(), 0, "buffer still owned by the live frame");
drop(vf);
assert_eq!(pool.rx.len(), 1, "drop must park the buffer in the pool");
let mut second = pool.take(12);
assert_eq!(pool.rx.len(), 0, "take must consume the parked buffer");
assert!(second.is_empty(), "recycled buffers come back empty");
assert!(second.capacity() >= 12);
assert_eq!(
(pool.alloc_count(), pool.reuse_count()),
(1, 1),
"an equal-size take after a drop must be a reuse, not an allocation"
);
let parked_capacity = second.capacity();
second.extend_from_slice(&[7u8; 12]);
drop(VideoFrame::new(
2,
2,
PixelLayout::Rgb24,
None,
1,
second,
Some(pool.recycler()),
));
assert_eq!(
pool.rx.len(),
1,
"the grown take starts with a buffer parked"
);
let grown = pool.take(parked_capacity + 1);
assert!(grown.capacity() > parked_capacity);
assert_eq!(
(pool.alloc_count(), pool.reuse_count()),
(2, 1),
"a take grown past the parked capacity must allocate fresh"
);
}
#[test]
fn into_vec_detaches_from_the_pool() {
let pool = BufferPool::new(2);
let mut buf = pool.take(3);
buf.extend_from_slice(&[1, 2, 3]);
let vf = VideoFrame::new(1, 1, PixelLayout::Rgb24, None, 0, buf, Some(pool.recycler()));
let owned = vf.into_vec();
assert_eq!(owned, vec![1, 2, 3]);
assert!(
pool.rx.try_recv().is_err(),
"a detached buffer must never reach the pool"
);
}
#[test]
fn undersized_parked_buffers_are_skipped() {
let pool = BufferPool::new(2);
pool.recycler().try_send(Vec::with_capacity(4)).unwrap();
let buf = pool.take(1024);
assert!(buf.capacity() >= 1024);
assert!(
pool.rx.try_recv().is_err(),
"the undersized buffer must have been drained and dropped"
);
}
#[test]
fn oversized_parked_buffers_are_freed_on_shrink() {
let pool = BufferPool::new(2);
pool.recycler()
.try_send(Vec::with_capacity(1 << 20))
.unwrap();
let buf = pool.take(1024);
assert!(buf.capacity() >= 1024);
assert!(
buf.capacity() < (1 << 20),
"the jumbo buffer must have been dropped, not returned"
);
assert!(
pool.rx.try_recv().is_err(),
"the jumbo buffer must not remain parked either"
);
}
#[test]
fn modestly_larger_parked_buffers_are_reused() {
let pool = BufferPool::new(2);
let big = Vec::with_capacity(8192);
let ptr = big.as_ptr();
pool.recycler().try_send(big).unwrap();
let buf = pool.take(4096); assert_eq!(buf.as_ptr(), ptr, "a within-bound buffer must be reused");
assert_eq!(
(pool.alloc_count(), pool.reuse_count()),
(0, 1),
"the take must be served by the parked buffer, not an allocation"
);
}
#[test]
fn full_pool_drop_degrades_to_free() {
let pool = BufferPool::new(1);
let pre_parked = Vec::with_capacity(8);
let pre_parked_ptr = pre_parked.as_ptr();
pool.recycler().try_send(pre_parked).unwrap();
let vf = VideoFrame::new(
1,
1,
PixelLayout::Rgb24,
None,
0,
vec![0u8; 3],
Some(pool.recycler()),
);
drop(vf);
let parked = pool.take(8);
assert_eq!(
parked.as_ptr(),
pre_parked_ptr,
"take must return the pre-parked buffer itself"
);
assert!(
pool.rx.try_recv().is_err(),
"the overflow buffer must have been freed, not parked"
);
}
#[test]
fn cross_thread_drop_recycles() {
let pool = BufferPool::new(2);
let mut buf = pool.take(12);
buf.extend_from_slice(&[9u8; 12]);
let ptr = buf.as_ptr() as usize;
let vf = VideoFrame::new(2, 2, PixelLayout::Rgb24, None, 0, buf, Some(pool.recycler()));
std::thread::spawn(move || drop(vf)).join().expect("drop thread");
let reused = pool.take(12);
assert_eq!(
reused.as_ptr() as usize,
ptr,
"a cross-thread drop must land back in the pool"
);
assert_eq!(
(pool.alloc_count(), pool.reuse_count()),
(1, 1),
"the post-drop take must be a reuse"
);
}
unsafe fn synthetic_rgb24(backing: *mut u8, w: i32, h: i32, linesize: i32) -> Frame {
let p = av_frame_alloc();
assert!(!p.is_null());
(*p).format = AV_PIX_FMT_RGB24 as i32;
(*p).width = w;
(*p).height = h;
(*p).data[0] = backing;
(*p).linesize[0] = linesize;
Frame::wrap(p)
}
unsafe fn stamp_count(frame: &mut Frame, count: u64) {
let val = std::ffi::CString::new(count.to_string()).expect("digits have no NUL");
let p = frame.as_mut_ptr();
assert!(av_dict_set(&mut (*p).metadata, EMIT_COUNT_KEY.as_ptr(), val.as_ptr(), 0) >= 0);
}
#[test]
fn uniform_duplicates_draw_from_the_pool() {
let (tx, rx) = crossbeam_channel::bounded(8);
let mut sink = ExportSink::new(tx, Sampling::UniformN(4), PixelLayout::Rgb24, None);
for _ in 0..4 {
sink.pool
.recycler()
.try_send(Vec::with_capacity(12))
.expect("park a warm buffer");
}
let mut backing: Vec<u8> = (0u8..12).collect();
let mut frame = unsafe { synthetic_rgb24(backing.as_mut_ptr(), 2, 2, 6) };
unsafe { stamp_count(&mut frame, 4) };
let mut attrs: std::collections::HashMap<String, Box<dyn std::any::Any + Send>> =
std::collections::HashMap::new();
let mut ctx = FrameFilterContext::new("export_sink", &mut attrs);
let forwarded = sink.filter_frame(frame, &mut ctx).expect("filter_frame");
assert!(forwarded.is_some(), "uniform mode forwards the source frame");
let frames: Vec<VideoFrame> = rx.try_iter().collect();
assert_eq!(frames.len(), 4, "a dup-count of 4 must deliver 4 frames");
for (i, f) in frames.iter().enumerate() {
assert_eq!(f.index(), i as u64, "indices count every delivery");
assert_eq!(
f.as_bytes(),
&backing[..],
"every duplicate carries the packed payload"
);
}
assert!(
sink.pool.rx.try_recv().is_err(),
"all four parked buffers must be consumed: duplicates draw from \
the pool rather than allocating fresh"
);
}
#[test]
fn pack_skips_positive_stride_padding() {
let mut backing = vec![0xEEu8; 24];
for row in 0..2usize {
for i in 0..9usize {
backing[row * 12 + i] = (row * 9 + i) as u8;
}
}
let pool = BufferPool::new(1);
let (w, h, bytes) = unsafe {
let f = synthetic_rgb24(backing.as_mut_ptr(), 3, 2, 12);
pack_bytes(f.as_ptr(), PixelLayout::Rgb24, &pool).expect("pack")
};
assert_eq!((w, h), (3, 2));
let expected: Vec<u8> = (0u8..18).collect();
assert_eq!(bytes, expected, "payload rows only, no stride padding");
}
#[test]
fn pack_walks_negative_stride_top_down() {
let mut backing = vec![0xEEu8; 24];
for i in 0..9usize {
backing[i] = 100 + i as u8; backing[12 + i] = i as u8; }
let pool = BufferPool::new(1);
let (_, _, bytes) = unsafe {
let f = synthetic_rgb24(backing.as_mut_ptr().add(12), 3, 2, -12);
pack_bytes(f.as_ptr(), PixelLayout::Rgb24, &pool).expect("pack")
};
let mut expected: Vec<u8> = (0u8..9).collect();
expected.extend(100u8..109);
assert_eq!(bytes, expected, "display order: top row then bottom row");
}
#[test]
fn keyframes_only_selects_by_key_flag() {
let (tx, _rx) = crossbeam_channel::bounded(1);
let sink = ExportSink::new(tx, Sampling::KeyframesOnly, PixelLayout::Rgb24, None);
let key = flag_frame(true);
let delta = flag_frame(false);
unsafe {
assert!(
sink.select(key.as_ptr()),
"a KEY-flagged frame must be selected"
);
assert!(
!sink.select(delta.as_ptr()),
"a delta frame must not be selected"
);
}
}
}