use alloc::vec::Vec;
use crate::annexb::{iter_length_prefixed_nals_with, normalise_nal_length_size};
use crate::bitreader::BitReader;
use crate::ir::{CodecConfig, Sample};
use crate::nal::{NalCodec, access_unit_is_rap, nal_unit_type, sei_messages};
const AVC_NAL_IDR: u8 = 5;
const AVC_NAL_SEI: u8 = 6;
const AVC_SEI_RECOVERY_POINT: u32 = 6;
const AVC_NAL_HEADER_LEN: usize = 1;
const HEVC_RASL_N: u8 = 8;
const HEVC_RASL_R: u8 = 9;
const HEVC_BLA_FIRST: u8 = 16;
const HEVC_BLA_LAST: u8 = 18;
const HEVC_IDR_W_RADL: u8 = 19;
const HEVC_IDR_N_LP: u8 = 20;
const HEVC_CRA: u8 = 21;
const HEVC_FIRST_NON_VCL: u8 = 32;
const VVC_IDR_W_RADL: u8 = 7;
const VVC_IDR_N_LP: u8 = 8;
const VVC_CRA: u8 = 9;
const CANONICAL_LENGTH_SIZE: usize = 4;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize))]
#[non_exhaustive]
pub enum RapKind {
Idr,
Bla,
Cra,
RecoveryPoint {
recovery_frame_cnt: u32,
exact_match: bool,
broken_link: bool,
},
SpsIntra,
}
impl RapKind {
pub fn is_self_contained(&self) -> bool {
matches!(self, RapKind::Idr | RapKind::Bla | RapKind::Cra)
}
pub fn name(&self) -> &'static str {
match self {
RapKind::Idr => "IDR",
RapKind::Bla => "BLA",
RapKind::Cra => "CRA",
RapKind::RecoveryPoint { .. } => "recovery_point",
RapKind::SpsIntra => "SPS+I",
}
}
}
broadcast_common::impl_spec_display!(RapKind);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum JoinPolicy {
#[default]
AnyRap,
SelfContainedOnly,
}
impl JoinPolicy {
pub fn name(&self) -> &'static str {
match self {
JoinPolicy::AnyRap => "any RAP",
JoinPolicy::SelfContainedOnly => "self-contained only",
}
}
}
broadcast_common::impl_spec_display!(JoinPolicy);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum DropReason {
BeforeRandomAccessPoint,
NotSelfContained,
LeadingPicture,
}
impl DropReason {
pub fn name(&self) -> &'static str {
match self {
DropReason::BeforeRandomAccessPoint => "before random-access point",
DropReason::NotSelfContained => "not self-contained",
DropReason::LeadingPicture => "leading picture",
}
}
}
broadcast_common::impl_spec_display!(DropReason);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum JoinDecision {
Drop(DropReason),
JoinPoint(RapKind),
Keep,
}
impl JoinDecision {
pub fn is_keep(&self) -> bool {
!matches!(self, JoinDecision::Drop(_))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum JoinState {
Waiting,
Joined {
kind: RapKind,
pts: Option<i64>,
in_leading_run: bool,
},
}
#[derive(Debug, Clone)]
pub struct LiveJoin {
codec: NalCodec,
length_size: usize,
policy: JoinPolicy,
state: JoinState,
}
impl LiveJoin {
pub fn new(codec: NalCodec, length_size: usize) -> Self {
Self {
codec,
length_size,
policy: JoinPolicy::default(),
state: JoinState::Waiting,
}
}
pub fn for_config(config: &CodecConfig) -> Option<Self> {
let (codec, minus_one) = match config {
CodecConfig::Avc { config, .. } => (NalCodec::Avc, config.config.length_size_minus_one),
CodecConfig::Hevc { config, .. } => {
(NalCodec::Hevc, config.config.length_size_minus_one)
}
CodecConfig::Vvc { config, .. } => (NalCodec::Vvc, config.config.length_size_minus_one),
_ => return None,
};
Some(Self::new(codec, usize::from(minus_one) + 1))
}
pub fn with_policy(mut self, policy: JoinPolicy) -> Self {
self.policy = policy;
self
}
pub fn set_policy(&mut self, policy: JoinPolicy) {
self.policy = policy;
}
pub fn policy(&self) -> JoinPolicy {
self.policy
}
pub fn state(&self) -> JoinState {
self.state
}
pub fn rejoin(&mut self) {
self.state = JoinState::Waiting;
}
pub fn push(&mut self, sample: &Sample) -> JoinDecision {
let Some(au) = self.canonical(&sample.data) else {
return match self.state {
JoinState::Waiting => JoinDecision::Drop(DropReason::BeforeRandomAccessPoint),
JoinState::Joined { .. } => JoinDecision::Keep,
};
};
match self.state {
JoinState::Waiting => match rap_kind(self.codec, &au) {
None => JoinDecision::Drop(DropReason::BeforeRandomAccessPoint),
Some(kind)
if self.policy == JoinPolicy::SelfContainedOnly
&& !kind.is_self_contained() =>
{
JoinDecision::Drop(DropReason::NotSelfContained)
}
Some(kind) => {
self.state = JoinState::Joined {
kind,
pts: sample.pts,
in_leading_run: true,
};
JoinDecision::JoinPoint(kind)
}
},
JoinState::Joined {
kind,
pts,
in_leading_run: true,
} => {
if is_leading(self.codec, &au, sample.pts, pts) {
JoinDecision::Drop(DropReason::LeadingPicture)
} else {
self.state = JoinState::Joined {
kind,
pts,
in_leading_run: false,
};
JoinDecision::Keep
}
}
JoinState::Joined { .. } => JoinDecision::Keep,
}
}
fn canonical<'a>(&self, data: &'a [u8]) -> Option<alloc::borrow::Cow<'a, [u8]>> {
if self.length_size == CANONICAL_LENGTH_SIZE {
iter_length_prefixed_nals_with(data, CANONICAL_LENGTH_SIZE).ok()?;
Some(alloc::borrow::Cow::Borrowed(data))
} else {
normalise_nal_length_size(data, self.length_size)
.ok()
.map(alloc::borrow::Cow::Owned)
}
}
}
fn rap_kind(codec: NalCodec, au: &[u8]) -> Option<RapKind> {
let nals = iter_length_prefixed_nals_with(au, CANONICAL_LENGTH_SIZE).ok()?;
match codec {
NalCodec::Avc => {
if nals
.iter()
.any(|n| nal_unit_type(NalCodec::Avc, n) == Some(AVC_NAL_IDR))
{
return Some(RapKind::Idr);
}
if let Some(kind) = nals.iter().find_map(|n| avc_recovery_point(n)) {
return Some(kind);
}
access_unit_is_rap(NalCodec::Avc, au, true).then_some(RapKind::SpsIntra)
}
NalCodec::Hevc => nals
.iter()
.filter_map(|n| nal_unit_type(NalCodec::Hevc, n))
.find_map(|t| match t {
HEVC_IDR_W_RADL | HEVC_IDR_N_LP => Some(RapKind::Idr),
HEVC_CRA => Some(RapKind::Cra),
HEVC_BLA_FIRST..=HEVC_BLA_LAST => Some(RapKind::Bla),
_ => None,
}),
NalCodec::Vvc => nals
.iter()
.filter_map(|n| nal_unit_type(NalCodec::Vvc, n))
.find_map(|t| match t {
VVC_IDR_W_RADL | VVC_IDR_N_LP => Some(RapKind::Idr),
VVC_CRA => Some(RapKind::Cra),
_ => None,
}),
}
}
fn avc_recovery_point(nal: &[u8]) -> Option<RapKind> {
if nal_unit_type(NalCodec::Avc, nal) != Some(AVC_NAL_SEI) {
return None;
}
let (_, payload) = sei_messages(&nal[AVC_NAL_HEADER_LEN..])
.into_iter()
.find(|(t, _)| *t == AVC_SEI_RECOVERY_POINT)?;
let mut r = BitReader::from_rbsp(&payload, "recovery_point").ok()?;
let recovery_frame_cnt = u32::try_from(r.read_ue("recovery_frame_cnt").ok()?).ok()?;
let exact_match = r.read_flag("exact_match_flag").ok()?;
let broken_link = r.read_flag("broken_link_flag").ok()?;
Some(RapKind::RecoveryPoint {
recovery_frame_cnt,
exact_match,
broken_link,
})
}
fn is_leading(codec: NalCodec, au: &[u8], pts: Option<i64>, rap_pts: Option<i64>) -> bool {
match codec {
NalCodec::Hevc => {
let Ok(nals) = iter_length_prefixed_nals_with(au, CANONICAL_LENGTH_SIZE) else {
return false;
};
let vcl: Vec<u8> = nals
.iter()
.filter_map(|n| nal_unit_type(NalCodec::Hevc, n))
.filter(|&t| t < HEVC_FIRST_NON_VCL)
.collect();
!vcl.is_empty() && vcl.iter().all(|&t| t == HEVC_RASL_N || t == HEVC_RASL_R)
}
NalCodec::Avc | NalCodec::Vvc => match (pts, rap_pts) {
(Some(p), Some(r)) => p < r,
_ => false,
},
}
}
#[cfg(test)]
mod tests {
use super::*;
use alloc::vec;
fn au(size: usize, nals: &[Vec<u8>]) -> Vec<u8> {
let mut out = Vec::new();
for n in nals {
let len = (n.len() as u32).to_be_bytes();
out.extend_from_slice(&len[4 - size..]);
out.extend_from_slice(n);
}
out
}
fn sample(data: Vec<u8>, pts: i64) -> Sample {
Sample::new(data, Some(pts), Some(pts), None, false)
}
const SPS: [u8; 4] = [0x67, 66, 0, 30];
const I_SLICE: [u8; 3] = [0x61, 0b1000_1000, 0x80];
const P_SLICE: [u8; 3] = [0x41, 0b1001_1000, 0x80];
const IDR_SLICE: [u8; 3] = [0x65, 0b1000_1000, 0x80];
fn recovery_sei(cnt_ue_bits: &[bool], exact: bool, broken: bool) -> Vec<u8> {
let mut bits: Vec<bool> = cnt_ue_bits.to_vec();
bits.push(exact);
bits.push(broken);
bits.extend([false, false]); let mut payload = vec![0u8; bits.len().div_ceil(8)];
for (i, b) in bits.iter().enumerate() {
if *b {
payload[i / 8] |= 0x80 >> (i % 8);
}
}
let mut nal = vec![0x06, 6, payload.len() as u8];
nal.extend(payload);
nal.push(0x80);
nal
}
#[test]
fn avc_drops_until_idr_and_keeps_everything_after() {
let mut j = LiveJoin::new(NalCodec::Avc, 4);
let p = sample(au(4, &[P_SLICE.to_vec()]), 10);
assert_eq!(
j.push(&p),
JoinDecision::Drop(DropReason::BeforeRandomAccessPoint)
);
let sps_p = sample(au(4, &[SPS.to_vec(), P_SLICE.to_vec()]), 11);
assert_eq!(
j.push(&sps_p),
JoinDecision::Drop(DropReason::BeforeRandomAccessPoint)
);
let idr = sample(au(4, &[SPS.to_vec(), IDR_SLICE.to_vec()]), 12);
assert_eq!(j.push(&idr), JoinDecision::JoinPoint(RapKind::Idr));
assert_eq!(j.push(&p), JoinDecision::Drop(DropReason::LeadingPicture));
let later = sample(au(4, &[P_SLICE.to_vec()]), 13);
assert_eq!(j.push(&later), JoinDecision::Keep);
assert_eq!(
j.state(),
JoinState::Joined {
kind: RapKind::Idr,
pts: Some(12),
in_leading_run: false
}
);
}
#[test]
fn avc_recovery_point_fields_and_leading_pictures() {
let sei = recovery_sei(&[false, false, true, false, false], true, false);
let rp = sample(au(4, &[sei, I_SLICE.to_vec()]), 100);
let mut j = LiveJoin::new(NalCodec::Avc, 4);
let kind = RapKind::RecoveryPoint {
recovery_frame_cnt: 3,
exact_match: true,
broken_link: false,
};
assert_eq!(j.push(&rp), JoinDecision::JoinPoint(kind));
assert!(!kind.is_self_contained());
for pts in [96, 98] {
assert_eq!(
j.push(&sample(au(4, &[P_SLICE.to_vec()]), pts)),
JoinDecision::Drop(DropReason::LeadingPicture)
);
}
assert_eq!(
j.push(&sample(au(4, &[P_SLICE.to_vec()]), 104)),
JoinDecision::Keep
);
assert_eq!(
j.push(&sample(au(4, &[P_SLICE.to_vec()]), 50)),
JoinDecision::Keep
);
}
#[test]
fn sps_plus_intra_without_sei_is_the_fallback_kind() {
let mut j = LiveJoin::new(NalCodec::Avc, 4);
let s = sample(au(4, &[SPS.to_vec(), I_SLICE.to_vec()]), 5);
assert_eq!(j.push(&s), JoinDecision::JoinPoint(RapKind::SpsIntra));
}
#[test]
fn self_contained_policy_waits_for_idr_and_rejoin_asks_for_the_next_rap() {
let sei = recovery_sei(&[true], true, false); let rp = |pts| sample(au(4, &[sei.clone(), I_SLICE.to_vec()]), pts);
let idr = |pts| sample(au(4, &[IDR_SLICE.to_vec()]), pts);
let p = |pts| sample(au(4, &[P_SLICE.to_vec()]), pts);
let mut j = LiveJoin::new(NalCodec::Avc, 4).with_policy(JoinPolicy::SelfContainedOnly);
assert_eq!(
j.push(&rp(1)),
JoinDecision::Drop(DropReason::NotSelfContained)
);
assert_eq!(
j.push(&p(2)),
JoinDecision::Drop(DropReason::BeforeRandomAccessPoint)
);
assert_eq!(j.push(&idr(3)), JoinDecision::JoinPoint(RapKind::Idr));
j.rejoin();
assert_eq!(j.state(), JoinState::Waiting);
assert_eq!(
j.push(&p(4)),
JoinDecision::Drop(DropReason::BeforeRandomAccessPoint)
);
j.set_policy(JoinPolicy::AnyRap);
assert!(matches!(
j.push(&rp(5)),
JoinDecision::JoinPoint(RapKind::RecoveryPoint { .. })
));
}
#[test]
fn two_byte_length_prefix_and_malformed_samples() {
let mut j = LiveJoin::new(NalCodec::Avc, 2);
let idr = sample(au(2, &[SPS.to_vec(), IDR_SLICE.to_vec()]), 1);
assert_eq!(j.push(&idr), JoinDecision::JoinPoint(RapKind::Idr));
let mut j = LiveJoin::new(NalCodec::Avc, 4);
let bad = sample(vec![0, 0, 0xFF, 0xFF, 0x65, 1], 1);
assert_eq!(
j.push(&bad),
JoinDecision::Drop(DropReason::BeforeRandomAccessPoint)
);
assert_eq!(j.state(), JoinState::Waiting);
}
fn hevc(nal_type: u8) -> Vec<u8> {
vec![nal_type << 1, 0x01, 0xAF]
}
#[test]
fn hevc_cra_join_drops_rasl_keeps_radl_and_trailing() {
let mut j = LiveJoin::new(NalCodec::Hevc, 4);
assert_eq!(
j.push(&sample(au(4, &[hevc(9)]), 90)),
JoinDecision::Drop(DropReason::BeforeRandomAccessPoint)
);
assert_eq!(
j.push(&sample(au(4, &[hevc(21)]), 100)),
JoinDecision::JoinPoint(RapKind::Cra)
);
assert_eq!(
j.push(&sample(au(4, &[hevc(8)]), 96)),
JoinDecision::Drop(DropReason::LeadingPicture)
);
assert_eq!(
j.push(&sample(au(4, &[hevc(9)]), 140)),
JoinDecision::Drop(DropReason::LeadingPicture)
);
assert_eq!(j.push(&sample(au(4, &[hevc(7)]), 98)), JoinDecision::Keep);
assert_eq!(j.push(&sample(au(4, &[hevc(1)]), 104)), JoinDecision::Keep);
assert_eq!(j.push(&sample(au(4, &[hevc(9)]), 99)), JoinDecision::Keep);
}
#[test]
fn hevc_irap_kinds_and_reserved_types() {
for (t, want) in [
(19, Some(RapKind::Idr)),
(20, Some(RapKind::Idr)),
(16, Some(RapKind::Bla)),
(17, Some(RapKind::Bla)),
(18, Some(RapKind::Bla)),
(21, Some(RapKind::Cra)),
(22, None),
(23, None),
(1, None),
(6, None),
] {
let a = au(4, &[hevc(t)]);
assert_eq!(rap_kind(NalCodec::Hevc, &a), want, "type {t}");
}
}
#[test]
fn vvc_uses_pts_for_leading_pictures() {
let vvc = |t: u8| vec![0x00, t << 3, 0x01];
let mut j = LiveJoin::new(NalCodec::Vvc, 4);
assert_eq!(
j.push(&sample(au(4, &[vvc(9)]), 50)),
JoinDecision::JoinPoint(RapKind::Cra)
);
assert_eq!(
j.push(&sample(au(4, &[vvc(1)]), 40)),
JoinDecision::Drop(DropReason::LeadingPicture)
);
assert_eq!(j.push(&sample(au(4, &[vvc(1)]), 60)), JoinDecision::Keep);
}
#[test]
fn labels() {
assert_eq!(RapKind::Idr.to_string(), "IDR");
assert_eq!(DropReason::LeadingPicture.to_string(), "leading picture");
assert_eq!(
JoinPolicy::SelfContainedOnly.to_string(),
"self-contained only"
);
}
}