use super::backend::{MediaDecodeError, MediaMetadata, StreamingVideoDecoder, VideoFrame};
use super::mp4::{Mp4Error, Mp4VideoCodec, Mp4VideoIndex};
use super::mp4_avc::Mp4AvcStream;
use super::vp8_decoder::Vp8Decoder;
use super::vp9::split_superframe;
use super::vp9_decoder::Vp9Decoder;
use std::sync::Arc;
const MAX_BUFFER_BYTES: usize = 64 * 1024 * 1024;
const MAX_FRAMES_PER_PUSH: usize = 4;
#[derive(Default)]
pub struct Mp4VideoDecoder {
probe: Vec<u8>,
decoder: Option<Box<dyn StreamingVideoDecoder>>,
}
impl Mp4VideoDecoder {
pub fn new() -> Self {
Self::default()
}
}
impl StreamingVideoDecoder for Mp4VideoDecoder {
fn push(&mut self, bytes: &[u8]) -> Result<Vec<VideoFrame>, MediaDecodeError> {
if let Some(decoder) = &mut self.decoder {
return decoder.push(bytes);
}
if bytes.len() > MAX_BUFFER_BYTES.saturating_sub(self.probe.len()) {
return Err(MediaDecodeError::Unsupported);
}
self.probe.extend_from_slice(bytes);
let index = match Mp4VideoIndex::parse_prefix(&self.probe) {
Ok(index) => index,
Err(Mp4Error::Incomplete) => return Ok(Vec::new()),
Err(error) => return Err(mp4_error(error)),
};
let mut decoder: Box<dyn StreamingVideoDecoder> = match index.codec {
Mp4VideoCodec::Avc(_) => Box::new(Mp4AvcStream::new()),
Mp4VideoCodec::Vp8 | Mp4VideoCodec::Vp9 => Box::new(Mp4VpXStream::new(index)),
};
let frames = decoder.push(&self.probe)?;
self.probe.clear();
self.decoder = Some(decoder);
Ok(frames)
}
fn metadata(&self) -> Option<MediaMetadata> {
self.decoder.as_ref()?.metadata()
}
fn finish(&self) -> Result<(), MediaDecodeError> {
self.decoder.as_ref().ok_or(MediaDecodeError::Unsupported)?.finish()
}
fn has_buffered_samples(&self) -> bool {
self.decoder.as_ref().is_some_and(|decoder| decoder.has_buffered_samples())
}
}
enum VpXDecoder {
Vp8(Vp8Decoder),
Vp9(Vp9Decoder),
}
struct Mp4VpXStream {
bytes: Vec<u8>,
base_offset: u64,
index: Mp4VideoIndex,
next_sample: usize,
decoder: VpXDecoder,
}
impl Mp4VpXStream {
fn new(index: Mp4VideoIndex) -> Self {
let decoder = match &index.codec {
Mp4VideoCodec::Vp8 => VpXDecoder::Vp8(Vp8Decoder::new()),
Mp4VideoCodec::Vp9 => VpXDecoder::Vp9(Vp9Decoder::new()),
Mp4VideoCodec::Avc(_) => unreachable!("AVC uses its own sample stream"),
};
Self {
bytes: Vec::new(), base_offset: 0, index, next_sample: 0,
decoder,
}
}
fn sample_available(&self) -> bool {
self.index.samples.get(self.next_sample)
.and_then(|sample| sample.offset.checked_add(u64::from(sample.size)))
.is_some_and(|end| end <= self.base_offset + self.bytes.len() as u64)
}
}
impl StreamingVideoDecoder for Mp4VpXStream {
fn push(&mut self, bytes: &[u8]) -> Result<Vec<VideoFrame>, MediaDecodeError> {
if bytes.len() > MAX_BUFFER_BYTES.saturating_sub(self.bytes.len()) {
return Err(MediaDecodeError::Unsupported);
}
self.bytes.extend_from_slice(bytes);
let mut frames = Vec::new();
while frames.len() < MAX_FRAMES_PER_PUSH && self.sample_available() {
let sample = &self.index.samples[self.next_sample];
let start = usize::try_from(sample.offset.checked_sub(self.base_offset)
.ok_or(MediaDecodeError::Unsupported)?)
.map_err(|_| MediaDecodeError::Unsupported)?;
let end = start.checked_add(sample.size as usize)
.ok_or(MediaDecodeError::Unsupported)?;
let data = self.bytes.get(start..end).ok_or(MediaDecodeError::Unsupported)?;
let timestamp = sample.presentation_time as f32 / self.index.timescale as f32;
match &mut self.decoder {
VpXDecoder::Vp8(decoder) => {
let frame = decoder.decode(data)?;
frames.push(VideoFrame {
width: frame.width as u32,
height: frame.height as u32,
rgba: Arc::new(frame.rgba()),
timestamp,
});
}
VpXDecoder::Vp9(decoder) => {
for frame_data in split_superframe(data)? {
if let Some(frame) = decoder.decode(frame_data)? {
frames.push(VideoFrame {
width: frame.width as u32,
height: frame.height as u32,
rgba: Arc::new(frame.rgba()),
timestamp,
});
}
}
}
}
self.next_sample += 1;
}
let keep_from = self.index.samples.get(self.next_sample)
.map(|sample| sample.offset)
.unwrap_or(self.base_offset + self.bytes.len() as u64);
let discard = keep_from.saturating_sub(self.base_offset)
.min(self.bytes.len() as u64) as usize;
self.bytes.drain(..discard);
self.base_offset += discard as u64;
Ok(frames)
}
fn metadata(&self) -> Option<MediaMetadata> {
Some(MediaMetadata {
duration: Some(self.index.duration_ticks as f32 / self.index.timescale as f32),
width: Some(self.index.width),
height: Some(self.index.height),
sample_rate: None,
channels: None,
})
}
fn finish(&self) -> Result<(), MediaDecodeError> {
if self.next_sample == self.index.samples.len() { Ok(()) }
else { Err(MediaDecodeError::InvalidData("truncated MP4 video stream".into())) }
}
fn has_buffered_samples(&self) -> bool {
self.sample_available()
}
}
fn mp4_error(error: Mp4Error) -> MediaDecodeError {
match error {
Mp4Error::Unsupported(_) => MediaDecodeError::Unsupported,
other => MediaDecodeError::InvalidData(format!("MP4: {other:?}")),
}
}
#[cfg(test)]
mod tests {
use super::*;
use super::super::mp4::Sample;
#[test]
fn routes_vp8_mp4_samples_through_shared_decoder() {
let ivf = include_bytes!("../../tests/fixtures/vp8-keyframe.ivf");
assert_eq!(&ivf[..4], b"DKIF");
let size = u32::from_le_bytes(ivf[32..36].try_into().unwrap()) as usize;
let packet = &ivf[44..44 + size];
let index = Mp4VideoIndex {
timescale: 1, duration_ticks: 1, codec: Mp4VideoCodec::Vp8,
width: 32, height: 32,
samples: vec![Sample {
offset: 0, size: size as u32, decode_time: 0,
presentation_time: 0, keyframe: true,
}],
};
let mut stream = Mp4VpXStream::new(index);
assert!(stream.push(&packet[..size / 2]).unwrap().is_empty());
let frames = stream.push(&packet[size / 2..]).unwrap();
assert_eq!(frames.len(), 1);
assert_eq!((frames[0].width, frames[0].height), (32, 32));
assert_eq!(frames[0].rgba.len(), 32 * 32 * 4);
stream.finish().unwrap();
}
#[test]
fn routes_vp9_mp4_samples_through_shared_decoder() {
let bytes = std::env::var("WEBMEDIA_VP9_MP4_SAMPLE")
.map(|path| std::fs::read(path).unwrap())
.unwrap_or_else(|_| include_bytes!("../../tests/fixtures/vp9-keyframe.mp4").to_vec());
let index = Mp4VideoIndex::parse_prefix(&bytes).unwrap();
assert_eq!(index.codec, Mp4VideoCodec::Vp9);
let expected = index.samples.len();
let mut decoder = Mp4VideoDecoder::new();
let mut frames = Vec::new();
for chunk in bytes.chunks(4096) {
frames.extend(decoder.push(chunk).unwrap());
while decoder.has_buffered_samples() {
frames.extend(decoder.push(&[]).unwrap());
}
}
decoder.finish().unwrap();
assert_eq!(frames.len(), expected);
assert_eq!(decoder.metadata().unwrap().width, Some(frames[0].width));
assert_eq!(decoder.metadata().unwrap().height, Some(frames[0].height));
assert!(frames.windows(2).all(|pair| pair[0].timestamp <= pair[1].timestamp));
assert!(frames.iter().all(|frame| frame.rgba.len()
== frame.width as usize * frame.height as usize * 4));
}
#[test]
fn recognizes_vp08_mp4_sample_entry() {
let mut bytes = include_bytes!("../../tests/fixtures/vp9-keyframe.mp4").to_vec();
let entry = bytes.windows(4).position(|bytes| bytes == b"vp09").unwrap();
bytes[entry..entry + 4].copy_from_slice(b"vp08");
let index = Mp4VideoIndex::parse_prefix(&bytes).unwrap();
assert_eq!(index.codec, Mp4VideoCodec::Vp8);
}
#[test]
fn routes_avc_mp4_to_existing_decoder() {
let Ok(path) = std::env::var("WEBMEDIA_AVC_MP4_SAMPLE") else { return };
let bytes = std::fs::read(path).unwrap();
let index = Mp4VideoIndex::parse_prefix(&bytes).unwrap();
assert!(matches!(index.codec, Mp4VideoCodec::Avc(_)));
let mut decoder = Mp4VideoDecoder::new();
let mut frames = Vec::new();
for chunk in bytes.chunks(16 * 1024) {
frames.extend(decoder.push(chunk).unwrap());
while decoder.has_buffered_samples() {
frames.extend(decoder.push(&[]).unwrap());
}
if !frames.is_empty() { break; }
}
assert!(!frames.is_empty());
assert_eq!(decoder.metadata().unwrap().width, Some(frames[0].width));
}
}