mod sync_decoder;
#[cfg_attr(target_arch = "wasm32", path = "sync_decoder_wrapper_wasm.rs")]
#[cfg_attr(not(target_arch = "wasm32"), path = "sync_decoder_wrapper_native.rs")]
mod sync_decoder_wrapper;
mod image_decoder;
#[cfg(with_dav1d)]
mod av1;
#[cfg(with_ffmpeg)]
mod ffmpeg_cli;
#[cfg(with_ffmpeg)]
pub use ffmpeg_cli::FFmpegCliDecoder;
#[cfg(with_ffmpeg)]
pub use ffmpeg_cli::{
Error as FFmpegError, FFmpegVersion, FFmpegVersionParseError, ffmpeg_download_url,
};
#[cfg(target_arch = "wasm32")]
mod web_image_decoder;
#[cfg(target_arch = "wasm32")]
mod webcodecs;
#[cfg(target_arch = "wasm32")]
pub use webcodecs::WebVideoFrame;
mod rvl_decoder;
use crate::{SampleIndex, Time, VideoDataDescription, player::VideoPlaybackIssueSeverity};
#[derive(thiserror::Error, Debug, Clone, re_byte_size::SizeBytes)]
pub enum DecodeError {
#[error("Waiting for encoding details")]
WaitingForCodecDetails,
#[error("Unsupported codec: {0}")]
UnsupportedCodec(#[size_bytes(ignore)] String),
#[cfg(with_dav1d)]
#[error("dav1d: {0}")]
Dav1d(
#[from]
#[size_bytes(ignore)]
dav1d::Error,
),
#[error("To enabled native AV1 decoding, compile Rerun with the `nasm` feature enabled.")]
Dav1dWithoutNasm,
#[error(
"Rerun does not yet support native AV1 decoding on Linux ARM64. See https://github.com/rerun-io/rerun/issues/7755"
)]
NoDav1dOnLinuxArm64,
#[error("Image decode error: {0}")]
ImageDecoder(#[size_bytes(ignore)] String),
#[error(transparent)]
RvlDecoder(#[size_bytes(ignore)] re_rvl::RvlDecodeError),
#[cfg(target_arch = "wasm32")]
#[error(transparent)]
WebDecoder(
#[from]
#[size_bytes(ignore)]
webcodecs::WebError,
),
#[cfg(with_ffmpeg)]
#[error(transparent)]
Ffmpeg(#[size_bytes(ignore)] std::sync::Arc<FFmpegError>),
#[error("Unsupported bits per component: {0}")]
BadBitsPerComponent(#[size_bytes(ignore)] usize),
}
impl DecodeError {
pub fn should_request_more_frames(&self) -> bool {
match self {
Self::WaitingForCodecDetails
| Self::UnsupportedCodec(_)
| Self::Dav1dWithoutNasm
| Self::NoDav1dOnLinuxArm64
| Self::RvlDecoder(_) => false,
#[cfg(with_dav1d)]
Self::Dav1d(_) => true,
Self::ImageDecoder(_) => false,
#[cfg(target_arch = "wasm32")]
Self::WebDecoder(_) => true,
#[cfg(with_ffmpeg)]
Self::Ffmpeg(err) => err.should_request_more_frames(),
Self::BadBitsPerComponent(_) => false,
}
}
pub fn severity(&self) -> VideoPlaybackIssueSeverity {
match self {
Self::WaitingForCodecDetails => VideoPlaybackIssueSeverity::Informational,
#[cfg(with_dav1d)]
Self::Dav1d(err) => match err {
dav1d::Error::Again => VideoPlaybackIssueSeverity::Loading,
_ => VideoPlaybackIssueSeverity::Error,
},
Self::ImageDecoder(_) => VideoPlaybackIssueSeverity::Error,
#[cfg(target_arch = "wasm32")]
Self::WebDecoder(err) => err.severity(),
#[cfg(with_ffmpeg)]
Self::Ffmpeg(_) => VideoPlaybackIssueSeverity::Error,
Self::UnsupportedCodec(_)
| Self::Dav1dWithoutNasm
| Self::NoDav1dOnLinuxArm64
| Self::BadBitsPerComponent(_)
| Self::RvlDecoder(_) => VideoPlaybackIssueSeverity::Error,
}
}
}
pub type Result<T = (), E = DecodeError> = std::result::Result<T, E>;
pub type FrameResult = Result<Frame>;
pub trait AsyncDecoder: Send + Sync {
fn submit_chunk(&mut self, chunk: Chunk) -> Result<()>;
fn end_of_video(&mut self) -> Result<()> {
Ok(())
}
fn reset(&mut self, video_descr: &VideoDataDescription) -> Result<()>;
fn min_num_samples_to_enqueue_ahead(&self) -> usize {
0
}
}
pub fn new_decoder(
debug_name: &str,
video: &crate::VideoDataDescription,
decode_settings: &DecodeSettings,
output_sender: crate::Sender<FrameResult>,
) -> Result<Box<dyn AsyncDecoder>> {
#![allow(clippy::allow_attributes, unused_variables, clippy::needless_return)]
re_tracing::profile_function!();
re_log::trace!(
"Looking for decoder for {}",
video.human_readable_codec_string()
);
#[cfg(target_arch = "wasm32")]
{
return match &video.codec {
crate::VideoCodec::ImageSequence(codec) => {
if codec.as_deref() == Some("application/rvl") {
Ok(Box::new(sync_decoder_wrapper::SyncDecoderWrapper::new(
"rvl decoder".to_owned(),
Box::new(rvl_decoder::RvlDecoder),
output_sender,
)))
} else if let Some(decoder) =
web_image_decoder::WebImageDecoder::try_new(video, output_sender.clone())
{
Ok(Box::new(decoder))
} else {
Err(DecodeError::WaitingForCodecDetails)
}
}
_ => Ok(Box::new(webcodecs::WebVideoDecoder::new(
video,
decode_settings.hw_acceleration,
output_sender,
)?)),
};
}
#[cfg(not(target_arch = "wasm32"))]
match &video.codec {
#[cfg(feature = "av1")]
crate::VideoCodec::AV1 => {
#[cfg(linux_arm64)]
{
return Err(DecodeError::NoDav1dOnLinuxArm64);
}
#[cfg(with_dav1d)]
{
re_log::trace!("Decoding AV1…");
return Ok(Box::new(sync_decoder_wrapper::SyncDecoderWrapper::new(
debug_name.to_owned(),
Box::new(av1::SyncDav1dDecoder::new(debug_name.to_owned())?),
output_sender,
)));
}
}
#[cfg(with_ffmpeg)]
crate::VideoCodec::H264
| crate::VideoCodec::H265
| crate::VideoCodec::VP8
| crate::VideoCodec::VP9 => Ok(Box::new(FFmpegCliDecoder::new(
debug_name.to_owned(),
video.encoding_details.as_ref(),
output_sender,
decode_settings.ffmpeg_path.clone(),
&video.codec,
)?)),
crate::VideoCodec::ImageSequence(codec) => {
if codec.as_deref() == Some("application/rvl") {
Ok(Box::new(sync_decoder_wrapper::SyncDecoderWrapper::new(
"rvl decoder".to_owned(),
Box::new(rvl_decoder::RvlDecoder),
output_sender,
)))
} else if let Some(decoder) = image_decoder::SyncImageDecoder::try_new(video) {
Ok(Box::new(sync_decoder_wrapper::SyncDecoderWrapper::new(
format!("image decoder ({})", decoder.mime_type()),
Box::new(decoder),
output_sender,
)))
} else {
Err(DecodeError::WaitingForCodecDetails)
}
}
#[cfg(not(all(feature = "av1", with_ffmpeg)))]
_ => Err(DecodeError::UnsupportedCodec(
video.human_readable_codec_string(),
)),
}
}
#[derive(re_byte_size::SizeBytes)]
pub struct Chunk {
pub is_sync: bool,
pub data: Vec<u8>,
pub sample_idx: usize,
pub frame_nr: u32,
pub decode_timestamp: Time,
pub presentation_timestamp: Time,
pub duration: Option<Time>,
}
#[derive(re_byte_size::SizeBytes)]
pub struct DecodedFrameContent {
pub data: Vec<u8>,
pub width: u32,
pub height: u32,
#[size_bytes(ignore)]
pub format: PixelFormat,
}
impl DecodedFrameContent {
pub fn width(&self) -> u32 {
self.width
}
pub fn height(&self) -> u32 {
self.height
}
}
#[cfg(not(target_arch = "wasm32"))]
pub type FrameContent = DecodedFrameContent;
#[cfg(target_arch = "wasm32")]
#[derive(re_byte_size::SizeBytes)]
pub enum FrameContent {
WebVideoFrame(webcodecs::WebVideoFrame),
Decoded(DecodedFrameContent),
}
#[cfg(target_arch = "wasm32")]
impl FrameContent {
pub fn width(&self) -> u32 {
match self {
Self::WebVideoFrame(frame) => frame.display_width(),
Self::Decoded(frame) => frame.width(),
}
}
pub fn height(&self) -> u32 {
match self {
Self::WebVideoFrame(frame) => frame.display_height(),
Self::Decoded(frame) => frame.height(),
}
}
}
#[derive(Debug, Clone, re_byte_size::SizeBytes)]
pub struct FrameInfo {
pub is_sync: Option<bool>,
pub sample_idx: Option<SampleIndex>,
pub frame_nr: Option<u32>,
pub presentation_timestamp: Time,
pub duration: Option<Time>,
pub latest_decode_timestamp: Option<Time>,
}
impl FrameInfo {
pub fn presentation_time_range(&self) -> std::ops::Range<Time> {
if let Some(duration) = self.duration {
self.presentation_timestamp..self.presentation_timestamp + duration
} else {
self.presentation_timestamp..Time::MAX
}
}
}
#[derive(re_byte_size::SizeBytes)]
pub struct Frame {
pub content: FrameContent,
#[size_bytes(ignore)]
pub info: FrameInfo,
}
#[derive(Debug, Clone)]
pub enum PixelFormat {
L8,
L16,
R32Float,
Rgb8Unorm,
Rgba8Unorm,
Yuv {
layout: YuvPixelLayout,
range: YuvRange,
coefficients: YuvMatrixCoefficients,
},
}
impl PixelFormat {
pub fn bits_per_pixel(&self) -> u32 {
match self {
Self::L8 => 8,
Self::L16 => 16,
Self::Rgb8Unorm { .. } => 24,
Self::R32Float | Self::Rgba8Unorm { .. } => 32,
Self::Yuv { layout, .. } => match layout {
YuvPixelLayout::Y_U_V444 => 24,
YuvPixelLayout::Y_U_V422 => 16,
YuvPixelLayout::Y_U_V420 => 12,
YuvPixelLayout::Y400 => 8,
},
}
}
}
#[expect(non_camel_case_types)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum YuvPixelLayout {
Y_U_V444,
Y_U_V422,
Y_U_V420,
Y400,
}
#[derive(Debug, Clone, Copy)]
pub enum YuvRange {
Limited,
Full,
}
#[derive(Debug, Clone, Copy)]
pub enum YuvMatrixCoefficients {
Identity,
Bt601,
Bt709,
}
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Default, Hash, serde::Deserialize, serde::Serialize,
)]
pub enum DecodeHardwareAcceleration {
#[default]
Auto,
PreferSoftware,
PreferHardware,
}
#[derive(Debug, Clone, PartialEq, Eq, Default, Hash, serde::Deserialize, serde::Serialize)]
pub struct DecodeSettings {
pub hw_acceleration: DecodeHardwareAcceleration,
#[cfg(not(target_arch = "wasm32"))]
pub ffmpeg_path: Option<std::path::PathBuf>,
}
impl std::fmt::Display for DecodeHardwareAcceleration {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Auto => write!(f, "Auto"),
Self::PreferSoftware => write!(f, "Prefer software"),
Self::PreferHardware => write!(f, "Prefer hardware"),
}
}
}
impl std::str::FromStr for DecodeHardwareAcceleration {
type Err = ();
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s.trim().to_lowercase().replace('-', "_").as_str() {
"auto" => Ok(Self::Auto),
"prefer_software" | "software" => Ok(Self::PreferSoftware),
"prefer_hardware" | "hardware" => Ok(Self::PreferHardware),
_ => Err(()),
}
}
}