#[derive(Debug, Clone, Copy, PartialEq)]
pub struct FrameStat {
pub luma: f32,
pub diff: f32,
}
#[derive(Debug, Default)]
pub struct Decoded {
pub frames: Vec<FrameStat>,
pub samples: usize,
pub failures: usize,
pub first_failure: Option<usize>,
}
pub const RUN: usize = 30;
const BLACK_LUMA: f32 = 8.0;
const STRIDE: usize = 4;
pub fn sample_luma(plane: &[u8], stride: usize, w: usize, h: usize) -> Vec<u8> {
(0..h)
.step_by(STRIDE)
.flat_map(|r| {
plane
.get(r * stride..r * stride + w)
.unwrap_or_default()
.iter()
.step_by(STRIDE)
.copied()
})
.collect()
}
pub fn frame_stat(cur: &[u8], prev: Option<&[u8]>) -> FrameStat {
let n = cur.len().max(1) as f32;
let luma = cur.iter().map(|&v| v as f32).sum::<f32>() / n;
let diff = match prev {
Some(p) if p.len() == cur.len() => {
cur.iter()
.zip(p)
.map(|(&a, &b)| a.abs_diff(b) as f32)
.sum::<f32>()
/ n
}
_ => 0.0,
};
FrameStat { luma, diff }
}
fn runs(frames: &[FrameStat], pred: impl Fn(&FrameStat) -> bool) -> Vec<(usize, usize)> {
let mut out = Vec::new();
let mut start = None;
let end = FrameStat {
luma: 255.0,
diff: 1.0,
};
for (i, f) in frames.iter().chain(std::iter::once(&end)).enumerate() {
match (pred(f), start) {
(true, None) => start = Some(i),
(false, Some(s)) => {
if i - s >= RUN {
out.push((s, i - s));
}
start = None;
}
_ => {}
}
}
out
}
pub fn video_findings(d: &Decoded) -> Vec<(&'static str, String)> {
let mut out = Vec::new();
let missing = d.samples.saturating_sub(d.frames.len());
if d.failures > 0 || missing > 0 {
let first = d
.first_failure
.map_or(String::new(), |f| format!(", the first at frame {f}"));
out.push((
"video_decode",
format!(
"{} of {} frames could not be decoded{first}",
d.failures.max(missing),
d.samples
),
));
}
for (code, what, found) in [
(
"video_frozen",
"identical to the one before",
runs(&d.frames, |f| f.diff == 0.0),
),
(
"video_black",
"black",
runs(&d.frames, |f| f.luma < BLACK_LUMA),
),
] {
if let Some(&(start, len)) = found.iter().max_by_key(|r| r.1) {
let more = if found.len() > 1 {
format!(" (longest of {} runs)", found.len())
} else {
String::new()
};
out.push((
code,
format!("{len} frames in a row are {what}, from frame {start}{more}"),
));
}
}
out
}
#[cfg(feature = "decode")]
const SEGMENT: usize = 250;
#[cfg(any(feature = "decode", test))]
fn segments(sync: &[usize], total: usize, target: usize) -> Vec<std::ops::Range<usize>> {
let mut out = Vec::new();
let mut start = 0;
for &s in sync {
if s > start && s - start >= target && s < total {
out.push(start..s);
start = s;
}
}
out.push(start..total);
out
}
pub const AV1_ASM: bool = cfg!(any(
target_arch = "aarch64",
all(
target_arch = "x86_64",
any(
target_os = "linux",
target_os = "windows",
target_os = "macos"
)
),
feature = "asm"
));
pub fn asm_note(decoding: bool, asm: bool) -> Option<&'static str> {
(decoding && !asm).then_some(
"note: this build decodes AV1 without assembly (about 3x slower); the release binaries, PyPI wheels and \
cargo binstall have it, or rebuild with --features asm and nasm installed",
)
}
#[cfg(feature = "decode")]
pub fn decode_segmented(path: &std::path::Path, target: usize) -> anyhow::Result<Decoded> {
backend::decode_with(path, target)
}
#[cfg(feature = "decode")]
pub fn decode_checked(path: &std::path::Path) -> anyhow::Result<Decoded> {
backend::decode_video(path)
}
#[cfg(not(feature = "decode"))]
pub fn decode_checked(_: &std::path::Path) -> anyhow::Result<Decoded> {
anyhow::bail!("this build cannot decode; rebuild with `--features decode`")
}
#[cfg(feature = "decode")]
mod backend {
use super::{Decoded, SEGMENT, frame_stat, sample_luma, segments};
use crate::mp4::{Codec, Track, read_track};
use anyhow::{Context, Result};
use rayon::prelude::*;
use std::{ops::Range, path::Path};
type Push<'a> = dyn FnMut(&mut Decoded, &[u8], usize, usize, usize) + 'a;
struct Part {
d: Decoded,
first: Option<Vec<u8>>,
last: Option<Vec<u8>>,
}
pub fn decode_video(path: &Path) -> Result<Decoded> {
decode_with(path, SEGMENT)
}
pub(super) fn decode_with(path: &Path, target: usize) -> Result<Decoded> {
let track = read_track(path)?;
let parts = segments(&track.sync, track.samples.len(), target)
.into_par_iter()
.map(|r| decode_range(&track, r))
.collect::<Result<Vec<_>>>()?;
let mut out = Decoded::default();
let mut prev: Option<Vec<u8>> = None;
for mut p in parts {
if let (Some(first), Some(prev)) = (&p.first, &prev)
&& let Some(f) = p.d.frames.first_mut()
{
f.diff = frame_stat(first, Some(prev)).diff;
}
if p.last.is_some() {
prev = p.last;
}
out.samples += p.d.samples;
out.failures += p.d.failures;
out.first_failure = out.first_failure.or(p.d.first_failure);
out.frames.append(&mut p.d.frames);
}
Ok(out)
}
fn decode_range(track: &Track, range: Range<usize>) -> Result<Part> {
let base = range.start;
let samples = &track.samples[range];
let mut d = Decoded {
samples: samples.len(),
..Decoded::default()
};
let (mut first, mut prev): (Option<Vec<u8>>, Option<Vec<u8>>) = (None, None);
let mut push = |d: &mut Decoded, plane: &[u8], stride: usize, w: usize, h: usize| {
let cur = sample_luma(plane, stride, w, h);
d.frames.push(frame_stat(&cur, prev.as_deref()));
if first.is_none() {
first = Some(cur.clone());
}
prev = Some(cur);
};
match &track.codec {
Codec::H264 {
sps,
pps,
length_size,
} => h264(samples, base, sps, pps, *length_size, &mut d, &mut push)?,
Codec::Av1 { config } => av1(samples, base, config, &mut d, &mut push)?,
}
Ok(Part {
d,
first,
last: prev,
})
}
fn h264(
samples: &[Vec<u8>],
base: usize,
sps: &[Vec<u8>],
pps: &[Vec<u8>],
length_size: usize,
d: &mut Decoded,
push: &mut Push,
) -> Result<()> {
use openh264::{
OpenH264API,
decoder::{Decoder, DecoderConfig, Flush},
formats::YUVSource,
};
const START: [u8; 4] = [0, 0, 0, 1];
let config = DecoderConfig::new().flush_after_decode(Flush::NoFlush);
let mut dec = Decoder::with_api_config(OpenH264API::from_source(), config)
.context("creating the H.264 decoder")?;
let mut headers = Vec::new();
for set in sps.iter().chain(pps) {
headers.extend_from_slice(&START);
headers.extend_from_slice(set);
}
for (i, sample) in samples.iter().enumerate() {
let mut annex_b = if i == 0 { headers.clone() } else { Vec::new() };
let (mut at, mut ok) = (0, true);
while at + length_size <= sample.len() {
let len = sample[at..at + length_size]
.iter()
.fold(0usize, |a, &b| a << 8 | b as usize);
at += length_size;
match sample.get(at..at + len) {
Some(nal) => {
annex_b.extend_from_slice(&START);
annex_b.extend_from_slice(nal);
}
None => ok = false,
}
at += len;
}
match ok.then(|| dec.decode(&annex_b)) {
Some(Ok(Some(yuv))) => {
let (w, h) = yuv.dimensions();
push(d, yuv.y(), yuv.strides().0, w, h);
}
Some(Ok(None)) => {}
_ => {
d.failures += 1;
d.first_failure.get_or_insert(base + i);
}
}
}
if let Ok(rest) = dec.flush_remaining() {
for yuv in rest {
let (w, h) = yuv.dimensions();
push(d, yuv.y(), yuv.strides().0, w, h);
}
}
Ok(())
}
fn av1(
samples: &[Vec<u8>],
base: usize,
config: &[u8],
d: &mut Decoded,
push: &mut Push,
) -> Result<()> {
let mut dec = Av1::open()?;
dec.send(config, &mut |_, _, _, _| {});
for (i, sample) in samples.iter().enumerate() {
if !dec.send(sample, &mut |p, s, w, h| push(d, p, s, w, h)) {
d.failures += 1;
d.first_failure.get_or_insert(base + i);
}
}
dec.drain(&mut |p, s, w, h| push(d, p, s, w, h));
Ok(())
}
struct Av1 {
ctx: Option<rav1d::include::dav1d::dav1d::Dav1dContext>,
}
type Plane<'a> = dyn FnMut(&[u8], usize, usize, usize) + 'a;
impl Av1 {
fn open() -> Result<Self> {
use rav1d::{
include::dav1d::dav1d::Dav1dSettings,
{dav1d_default_settings, dav1d_open},
};
use std::{mem::MaybeUninit, ptr::NonNull};
let mut settings = MaybeUninit::<Dav1dSettings>::uninit();
let mut ctx = None;
let r = unsafe {
dav1d_default_settings(NonNull::from(&mut settings).cast());
let s = &mut *settings.as_mut_ptr();
s.n_threads = 1;
s.max_frame_delay = 1;
dav1d_open(
Some(NonNull::from(&mut ctx)),
Some(NonNull::from(&mut settings).cast()),
)
};
anyhow::ensure!(r.0 == 0, "creating the AV1 decoder failed ({})", r.0);
Ok(Self { ctx })
}
fn send(&mut self, bytes: &[u8], f: &mut Plane) -> bool {
use rav1d::{
include::dav1d::data::Dav1dData,
{dav1d_data_create, dav1d_data_unref, dav1d_send_data},
};
use std::{mem::MaybeUninit, ptr::NonNull};
let mut data = MaybeUninit::<Dav1dData>::zeroed();
unsafe {
let buf = dav1d_data_create(Some(NonNull::from(&mut data).cast()), bytes.len());
if buf.is_null() {
return false;
}
std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf, bytes.len());
let mut ok = true;
while (*data.as_ptr()).sz > 0 {
let r = dav1d_send_data(self.ctx, Some(NonNull::from(&mut data).cast()));
if r.0 < 0 && !would_block(r.0) {
ok = false;
break;
}
self.pictures(f);
}
dav1d_data_unref(Some(NonNull::from(&mut data).cast()));
self.pictures(f);
ok
}
}
fn drain(&mut self, f: &mut Plane) {
self.pictures(f);
}
fn pictures(&mut self, f: &mut Plane) {
use rav1d::{
include::dav1d::picture::Dav1dPicture,
{dav1d_get_picture, dav1d_picture_unref},
};
use std::{mem::MaybeUninit, ptr::NonNull};
loop {
let mut pic = MaybeUninit::<Dav1dPicture>::zeroed();
unsafe {
let r = dav1d_get_picture(self.ctx, Some(NonNull::from(&mut pic).cast()));
if r.0 != 0 {
return;
}
let p = &*pic.as_ptr();
let (w, h) = (p.p.w as usize, p.p.h as usize);
if let (Some(plane), true, 8) = (p.data[0], p.stride[0] > 0, p.p.bpc) {
let stride = p.stride[0] as usize;
f(
std::slice::from_raw_parts(plane.as_ptr().cast::<u8>(), stride * h),
stride,
w,
h,
);
}
dav1d_picture_unref(Some(NonNull::from(&mut pic).cast()));
}
}
}
}
fn would_block(code: i32) -> bool {
std::io::Error::from_raw_os_error(-code).kind() == std::io::ErrorKind::WouldBlock
}
impl Drop for Av1 {
fn drop(&mut self) {
unsafe { rav1d::dav1d_close(Some(std::ptr::NonNull::from(&mut self.ctx))) };
}
}
}
#[cfg(test)]
mod tests {
use super::{asm_note, segments};
#[test]
fn asm_note_only_when_decoding_without_assembly() {
assert!(asm_note(true, false).is_some_and(|n| n.contains("--features asm")));
assert_eq!(asm_note(true, true), None);
assert_eq!(asm_note(false, false), None);
}
fn spans(sync: &[usize], total: usize, target: usize) -> Vec<(usize, usize)> {
segments(sync, total, target)
.into_iter()
.map(|r| (r.start, r.end))
.collect()
}
#[test]
fn segments_cut_at_sync_samples_once_long_enough() {
let sync: Vec<usize> = (0..14).step_by(2).collect();
assert_eq!(spans(&sync, 14, 5), [(0, 6), (6, 12), (12, 14)]);
}
#[test]
fn segments_cover_every_sample_exactly_once() {
let sync: Vec<usize> = (0..100).step_by(3).collect();
for target in [1, 4, 10, 1000] {
let s = segments(&sync, 100, target);
assert_eq!(s.first().unwrap().start, 0);
assert_eq!(s.last().unwrap().end, 100);
assert!(s.windows(2).all(|w| w[0].end == w[1].start), "{s:?}");
}
}
#[test]
fn a_track_synced_only_at_its_start_is_one_segment() {
assert_eq!(spans(&[0], 500, 10), [(0, 500)]);
}
#[test]
fn an_empty_track_is_one_empty_segment() {
assert_eq!(spans(&[0], 0, 10), [(0, 0)]);
}
}