use super::{VideoDepacketizer, VideoPacketizer};
use crate::types::{RtpPacket, VideoCodec, VideoFrame};
const NAL_TYPE_IDR: u8 = 5;
const NAL_TYPE_SPS: u8 = 7;
const NAL_TYPE_PPS: u8 = 8;
const NAL_TYPE_STAP_A: u8 = 24;
const NAL_TYPE_FU_A: u8 = 28;
const START_CODE: &[u8] = &[0x00, 0x00, 0x00, 0x01];
pub struct H264Depacketizer {
nals: Vec<Vec<u8>>,
current_ts: u32,
fua_buf: Vec<u8>,
fua_nri: u8,
fua_active: bool,
has_idr: bool,
sps: Option<Vec<u8>>,
pps: Option<Vec<u8>>,
started: bool,
}
impl Default for H264Depacketizer {
fn default() -> Self {
Self::new()
}
}
impl H264Depacketizer {
pub fn new() -> Self {
Self {
nals: Vec::new(),
current_ts: 0,
fua_buf: Vec::new(),
fua_nri: 0,
fua_active: false,
has_idr: false,
sps: None,
pps: None,
started: false,
}
}
fn emit_frame(&mut self) -> Option<VideoFrame> {
if self.nals.is_empty() {
return None;
}
let keyframe = self.has_idr;
let total: usize = self.nals.iter().map(|n| START_CODE.len() + n.len()).sum();
let mut data = Vec::with_capacity(total + 2 * (START_CODE.len() + 64));
if keyframe {
if let Some(ref sps) = self.sps {
data.extend_from_slice(START_CODE);
data.extend_from_slice(sps);
}
if let Some(ref pps) = self.pps {
data.extend_from_slice(START_CODE);
data.extend_from_slice(pps);
}
}
for nal in self.nals.drain(..) {
let nal_type = nal[0] & 0x1F;
if keyframe && (nal_type == NAL_TYPE_SPS || nal_type == NAL_TYPE_PPS) {
continue;
}
data.extend_from_slice(START_CODE);
data.extend_from_slice(&nal);
}
self.has_idr = false;
if data.is_empty() {
return None;
}
Some(VideoFrame {
codec: VideoCodec::H264,
keyframe,
timestamp: self.current_ts,
data,
})
}
fn process_nal(&mut self, nal: &[u8]) {
if nal.is_empty() {
return;
}
let nal_type = nal[0] & 0x1F;
match nal_type {
NAL_TYPE_SPS => {
let v = nal.to_vec();
self.sps = Some(v.clone());
self.nals.push(v);
}
NAL_TYPE_PPS => {
let v = nal.to_vec();
self.pps = Some(v.clone());
self.nals.push(v);
}
NAL_TYPE_IDR => {
self.has_idr = true;
self.nals.push(nal.to_vec());
}
_ => {
self.nals.push(nal.to_vec());
}
}
}
}
impl VideoDepacketizer for H264Depacketizer {
fn depacketize(&mut self, pkt: &RtpPacket) -> Option<VideoFrame> {
if pkt.payload.is_empty() {
return None;
}
let mut result = None;
if self.started && pkt.header.timestamp != self.current_ts {
if !self.nals.is_empty() {
result = self.emit_frame();
}
if self.fua_active {
self.fua_buf.clear();
self.fua_active = false;
}
}
self.current_ts = pkt.header.timestamp;
self.started = true;
let first_byte = pkt.payload[0];
let nal_type = first_byte & 0x1F;
match nal_type {
1..=23 => {
self.process_nal(&pkt.payload);
}
NAL_TYPE_STAP_A => {
let mut offset = 1; while offset + 2 <= pkt.payload.len() {
let size =
u16::from_be_bytes([pkt.payload[offset], pkt.payload[offset + 1]]) as usize;
offset += 2;
if size == 0 || offset + size > pkt.payload.len() {
break;
}
self.process_nal(&pkt.payload[offset..offset + size]);
offset += size;
}
}
NAL_TYPE_FU_A => {
if pkt.payload.len() < 2 {
return result;
}
let fu_header = pkt.payload[1];
let start = (fu_header & 0x80) != 0;
let end = (fu_header & 0x40) != 0;
let frag_type = fu_header & 0x1F;
if start {
self.fua_nri = first_byte & 0xE0;
self.fua_buf.clear();
self.fua_buf.push(self.fua_nri | frag_type);
self.fua_buf.extend_from_slice(&pkt.payload[2..]);
self.fua_active = true;
} else if self.fua_active {
self.fua_buf.extend_from_slice(&pkt.payload[2..]);
}
if end && self.fua_active {
self.process_nal(&self.fua_buf.clone());
self.fua_buf.clear(); self.fua_active = false;
}
}
_ => {
}
}
if pkt.header.marker && !self.nals.is_empty() {
return self.emit_frame().or(result);
}
result
}
}
pub struct H264Packetizer;
impl Default for H264Packetizer {
fn default() -> Self {
Self::new()
}
}
impl H264Packetizer {
pub fn new() -> Self {
Self
}
}
impl VideoPacketizer for H264Packetizer {
fn packetize(&mut self, frame: &VideoFrame, mtu: usize) -> Vec<Vec<u8>> {
if frame.data.is_empty() || mtu < 3 {
return Vec::new();
}
let nals = extract_nals(&frame.data);
let mut payloads = Vec::with_capacity(nals.len());
for nal in &nals {
if nal.is_empty() {
continue;
}
if nal.len() <= mtu {
payloads.push(nal.to_vec());
} else {
let nri = nal[0] & 0xE0;
let nal_type = nal[0] & 0x1F;
let fu_indicator = nri | NAL_TYPE_FU_A;
let frag_data = &nal[1..]; let max_frag = mtu - 2; let mut offset = 0;
let mut first = true;
while offset < frag_data.len() {
let end_offset = (offset + max_frag).min(frag_data.len());
let last = end_offset == frag_data.len();
let mut fu_header = nal_type;
if first {
fu_header |= 0x80; }
if last {
fu_header |= 0x40; }
let mut payload = Vec::with_capacity(2 + (end_offset - offset));
payload.push(fu_indicator);
payload.push(fu_header);
payload.extend_from_slice(&frag_data[offset..end_offset]);
payloads.push(payload);
offset = end_offset;
first = false;
}
}
}
payloads
}
}
fn extract_nals(data: &[u8]) -> Vec<&[u8]> {
let mut nals = Vec::new();
let mut i = 0;
let len = data.len();
while i < len {
if i + 3 <= len && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 1 {
i += 3;
break;
}
if i + 4 <= len && data[i] == 0 && data[i + 1] == 0 && data[i + 2] == 0 && data[i + 3] == 1
{
i += 4;
break;
}
i += 1;
}
let mut nal_start = i;
while i < len {
if i + 3 <= len && data[i] == 0 && data[i + 1] == 0 {
if data[i + 2] == 1 {
if i > nal_start {
nals.push(&data[nal_start..i]);
}
i += 3;
nal_start = i;
continue;
}
if i + 4 <= len && data[i + 2] == 0 && data[i + 3] == 1 {
if i > nal_start {
nals.push(&data[nal_start..i]);
}
i += 4;
nal_start = i;
continue;
}
}
i += 1;
}
if nal_start < len {
nals.push(&data[nal_start..len]);
}
nals
}
#[cfg(test)]
mod tests {
use super::*;
use crate::types::RtpHeader;
fn make_rtp(payload: Vec<u8>, ts: u32, marker: bool, seq: u16) -> RtpPacket {
RtpPacket {
header: RtpHeader {
version: 2,
marker,
payload_type: 96,
sequence_number: seq,
timestamp: ts,
ssrc: 1234,
},
payload,
}
}
#[test]
fn extract_nals_annex_b() {
let mut data = Vec::new();
data.extend_from_slice(START_CODE);
data.push(0x67); data.extend_from_slice(&[0x42, 0x00, 0x1f]);
data.extend_from_slice(START_CODE);
data.push(0x68); data.extend_from_slice(&[0xce, 0x38, 0x80]);
let nals = extract_nals(&data);
assert_eq!(nals.len(), 2);
assert_eq!(nals[0][0] & 0x1F, NAL_TYPE_SPS);
assert_eq!(nals[1][0] & 0x1F, NAL_TYPE_PPS);
}
#[test]
fn extract_nals_3byte_start_code() {
let data = [0x00, 0x00, 0x01, 0x65, 0xAA, 0xBB]; let nals = extract_nals(&data);
assert_eq!(nals.len(), 1);
assert_eq!(nals[0], vec![0x65, 0xAA, 0xBB]);
}
#[test]
fn single_nal_depacketize() {
let mut depkt = H264Depacketizer::new();
let nal = vec![0x41, 0x01, 0x02, 0x03]; let pkt = make_rtp(nal.clone(), 1000, true, 1);
let frame = depkt.depacketize(&pkt).unwrap();
assert!(!frame.keyframe);
assert_eq!(frame.timestamp, 1000);
assert!(frame.data.starts_with(START_CODE));
assert_eq!(&frame.data[4..], &nal[..]);
}
#[test]
fn single_nal_idr_is_keyframe() {
let mut depkt = H264Depacketizer::new();
let nal = vec![0x65, 0xFF, 0xFE]; let pkt = make_rtp(nal, 2000, true, 1);
let frame = depkt.depacketize(&pkt).unwrap();
assert!(frame.keyframe);
}
#[test]
fn stap_a_depacketize() {
let mut depkt = H264Depacketizer::new();
let sps = vec![0x67, 0x42, 0x00, 0x1f]; let pps = vec![0x68, 0xce, 0x38, 0x80]; let idr = vec![0x65, 0xAA];
let mut payload = vec![NAL_TYPE_STAP_A]; payload.extend_from_slice(&(sps.len() as u16).to_be_bytes());
payload.extend_from_slice(&sps);
payload.extend_from_slice(&(pps.len() as u16).to_be_bytes());
payload.extend_from_slice(&pps);
payload.extend_from_slice(&(idr.len() as u16).to_be_bytes());
payload.extend_from_slice(&idr);
let pkt = make_rtp(payload, 3000, true, 1);
let frame = depkt.depacketize(&pkt).unwrap();
assert!(frame.keyframe);
assert_eq!(frame.timestamp, 3000);
assert!(frame.data.len() > 12);
}
#[test]
fn stap_a_zero_length_nal_does_not_loop() {
let mut depkt = H264Depacketizer::new();
let payload = vec![
NAL_TYPE_STAP_A, 0x00,
0x00, 0x00,
0x03, 0x41,
0x01,
0x02, ];
let pkt = make_rtp(payload, 9000, true, 1);
let frame = depkt.depacketize(&pkt);
assert!(frame.is_none());
}
#[test]
fn fua_reassembly() {
let mut depkt = H264Depacketizer::new();
let original_nal = [0x65, 0x01, 0x02, 0x03, 0x04, 0x05]; let nri = original_nal[0] & 0xE0; let nal_type = original_nal[0] & 0x1F;
let mut p1 = vec![nri | NAL_TYPE_FU_A]; p1.push(0x80 | nal_type); p1.extend_from_slice(&original_nal[1..3]); let pkt1 = make_rtp(p1, 4000, false, 1);
let mut p2 = vec![nri | NAL_TYPE_FU_A];
p2.push(nal_type); p2.extend_from_slice(&original_nal[3..5]);
let pkt2 = make_rtp(p2, 4000, false, 2);
let mut p3 = vec![nri | NAL_TYPE_FU_A];
p3.push(0x40 | nal_type); p3.extend_from_slice(&original_nal[5..]);
let pkt3 = make_rtp(p3, 4000, true, 3);
assert!(depkt.depacketize(&pkt1).is_none());
assert!(depkt.depacketize(&pkt2).is_none());
let frame = depkt.depacketize(&pkt3).unwrap();
assert!(frame.keyframe);
assert_eq!(frame.timestamp, 4000);
assert!(frame.data.starts_with(START_CODE));
}
#[test]
fn packetize_small_nal() {
let mut pkt = H264Packetizer::new();
let mut data = Vec::new();
data.extend_from_slice(START_CODE);
data.extend_from_slice(&[0x41, 0x01, 0x02]);
let frame = VideoFrame {
codec: VideoCodec::H264,
keyframe: false,
timestamp: 100,
data,
};
let payloads = pkt.packetize(&frame, 1200);
assert_eq!(payloads.len(), 1);
assert_eq!(payloads[0], vec![0x41, 0x01, 0x02]);
}
#[test]
fn packetize_fua_fragmentation() {
let mut pkt = H264Packetizer::new();
let mut data = Vec::new();
data.extend_from_slice(START_CODE);
let mut nal = vec![0x65]; nal.extend_from_slice(&[0xAA; 100]); data.extend_from_slice(&nal);
let frame = VideoFrame {
codec: VideoCodec::H264,
keyframe: true,
timestamp: 200,
data,
};
let payloads = pkt.packetize(&frame, 50);
assert!(payloads.len() > 1);
assert_eq!(payloads[0][0] & 0x1F, NAL_TYPE_FU_A);
assert_ne!(payloads[0][1] & 0x80, 0);
let last = payloads.last().unwrap();
assert_ne!(last[1] & 0x40, 0);
if payloads.len() > 2 {
assert_eq!(payloads[1][1] & 0xC0, 0);
}
}
#[test]
fn packetize_depacketize_round_trip() {
let mut packetizer = H264Packetizer::new();
let mut depacketizer = H264Depacketizer::new();
let mut data = Vec::new();
data.extend_from_slice(START_CODE);
data.extend_from_slice(&[0x67, 0x42, 0x00, 0x1f]); data.extend_from_slice(START_CODE);
data.extend_from_slice(&[0x68, 0xce, 0x38, 0x80]); data.extend_from_slice(START_CODE);
let mut idr = vec![0x65]; idr.extend_from_slice(&[0xBB; 50]);
data.extend_from_slice(&idr);
let frame = VideoFrame {
codec: VideoCodec::H264,
keyframe: true,
timestamp: 9000,
data: data.clone(),
};
let payloads = packetizer.packetize(&frame, 1200);
assert!(!payloads.is_empty());
let mut result = None;
for (i, pl) in payloads.iter().enumerate() {
let marker = i == payloads.len() - 1;
let pkt = make_rtp(pl.clone(), 9000, marker, i as u16);
if let Some(f) = depacketizer.depacketize(&pkt) {
result = Some(f);
}
}
let out = result.unwrap();
assert!(out.keyframe);
assert_eq!(out.timestamp, 9000);
assert!(out.data.len() > 10);
}
#[test]
fn packetize_depacketize_non_idr_round_trip() {
let mut packetizer = H264Packetizer::new();
let mut depacketizer = H264Depacketizer::new();
let mut data = Vec::new();
data.extend_from_slice(START_CODE);
data.extend_from_slice(&[0x41, 0x01, 0x02, 0x03]);
let frame = VideoFrame {
codec: VideoCodec::H264,
keyframe: false,
timestamp: 10000,
data,
};
let payloads = packetizer.packetize(&frame, 1200);
let mut result = None;
for (i, pl) in payloads.iter().enumerate() {
let marker = i == payloads.len() - 1;
let pkt = make_rtp(pl.clone(), 10000, marker, i as u16);
if let Some(f) = depacketizer.depacketize(&pkt) {
result = Some(f);
}
}
let out = result.unwrap();
assert!(!out.keyframe);
}
#[test]
fn multi_frame_depacketize() {
let mut depkt = H264Depacketizer::new();
let pkt1 = make_rtp(vec![0x41, 0x01], 1000, true, 1);
let f1 = depkt.depacketize(&pkt1).unwrap();
assert_eq!(f1.timestamp, 1000);
let pkt2 = make_rtp(vec![0x41, 0x02], 2000, true, 2);
let f2 = depkt.depacketize(&pkt2).unwrap();
assert_eq!(f2.timestamp, 2000);
}
#[test]
fn sps_pps_cached_and_prepended() {
let mut depkt = H264Depacketizer::new();
let sps = vec![0x67, 0x42, 0x00, 0x1f];
let pps = vec![0x68, 0xce, 0x38];
let mut stap = vec![NAL_TYPE_STAP_A];
stap.extend_from_slice(&(sps.len() as u16).to_be_bytes());
stap.extend_from_slice(&sps);
stap.extend_from_slice(&(pps.len() as u16).to_be_bytes());
stap.extend_from_slice(&pps);
let pkt1 = make_rtp(stap, 1000, true, 1);
let _f1 = depkt.depacketize(&pkt1);
let idr = vec![0x65, 0xAA, 0xBB];
let pkt2 = make_rtp(idr, 2000, true, 2);
let frame = depkt.depacketize(&pkt2).unwrap();
assert!(frame.keyframe);
assert!(frame.data.starts_with(START_CODE));
let sps_pos = frame.data.windows(4).position(|w| w == &sps[..4]);
assert!(sps_pos.is_some());
}
#[test]
fn empty_payload_ignored() {
let mut depkt = H264Depacketizer::new();
let pkt = make_rtp(vec![], 1000, true, 1);
assert!(depkt.depacketize(&pkt).is_none());
}
#[test]
fn packetize_empty_frame() {
let mut pkt = H264Packetizer::new();
let frame = VideoFrame {
codec: VideoCodec::H264,
keyframe: false,
timestamp: 0,
data: Vec::new(),
};
assert!(pkt.packetize(&frame, 1200).is_empty());
}
}