use alloc::vec;
use alloc::vec::Vec;
use core::fmt;
use core::marker::PhantomData;
use broadcast_common::{Package, Parse, Serialize, Unpackage};
use crate::aac_asc::AudioSpecificConfig;
use crate::avc_config::{AVCConfigurationBox, AVCDecoderConfigurationRecord};
use crate::error::{Error, Result};
use crate::media::{Media, Track};
use crate::mp4esds::{
DecoderConfigDescriptor, DecoderSpecificInfo, ESDescriptor, EsdsBox, ObjectTypeIndication,
SLConfigDescriptor, StreamType as EsdsStreamType,
};
use crate::pipeline::{CodecConfig, Sample, TrackSpec};
pub(crate) const FLV_SIGNATURE: [u8; 3] = *b"FLV";
const FLV_VERSION: u8 = 1;
pub(crate) const FLV_HEADER_LEN: usize = 9;
const TYPE_FLAG_AUDIO: u8 = 0x04;
const TYPE_FLAG_VIDEO: u8 = 0x01;
pub(crate) const TAG_HEADER_LEN: usize = 11;
pub(crate) const PREV_TAG_SIZE_LEN: usize = 4;
pub(crate) const MAX_FLV_HEADER_LEN: usize = 1024;
pub(crate) mod tag_type {
pub const AUDIO: u8 = 8;
pub const VIDEO: u8 = 9;
pub const SCRIPT: u8 = 18;
}
pub(crate) const CODEC_ID_AVC: u8 = 7;
pub(crate) const FRAME_TYPE_KEYFRAME: u8 = 1;
const FRAME_TYPE_INTER: u8 = 2;
pub(crate) mod avc_packet_type {
pub const SEQUENCE_HEADER: u8 = 0;
pub const NALU: u8 = 1;
pub const END_OF_SEQUENCE: u8 = 2;
}
pub(crate) const SOUND_FORMAT_AAC: u8 = 10;
const SOUND_RATE_44K: u8 = 3;
const SOUND_SIZE_16BIT: u8 = 1;
const SOUND_TYPE_STEREO: u8 = 1;
const SOUND_TYPE_MONO: u8 = 0;
pub(crate) mod aac_packet_type {
pub const SEQUENCE_HEADER: u8 = 0;
pub const RAW: u8 = 1;
}
pub(crate) const FLV_TIMESCALE: u32 = 1000;
const OTI_MPEG4_AUDIO: u8 = 0x40;
const STREAM_TYPE_AUDIO: u8 = 0x05;
const ESDS_AUDIO_ES_ID: u16 = 1;
const SL_CONFIG_PREDEFINED_MP4: u8 = 0x02;
pub(crate) const AUDIO_SAMPLE_SIZE_BITS: u16 = 16;
#[derive(Debug, PartialEq, Eq)]
#[non_exhaustive]
pub enum FlvError {
BadSignature([u8; 3]),
TagOverrun {
offset: usize,
need: usize,
have: usize,
},
NoSupportedTrack,
HeaderTooLarge {
declared: u32,
max: usize,
},
UnsupportedCodec {
codec: &'static str,
},
Codec(Error),
}
impl fmt::Display for FlvError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
FlvError::BadSignature(sig) => {
write!(f, "bad FLV signature: {sig:02X?} (expected \"FLV\")")
}
FlvError::TagOverrun { offset, need, have } => write!(
f,
"FLV tag at offset {offset} overruns buffer: need {need}, have {have}"
),
FlvError::NoSupportedTrack => {
write!(
f,
"FLV carried no supported track (need AVC video or AAC audio)"
)
}
FlvError::HeaderTooLarge { declared, max } => write!(
f,
"FLV header DataOffset {declared} exceeds the maximum accepted {max} bytes"
),
FlvError::UnsupportedCodec { codec } => {
write!(
f,
"codec {codec} has no FLV carriage in this crate (only AVC + AAC)"
)
}
FlvError::Codec(e) => write!(f, "FLV codec config: {e}"),
}
}
}
#[cfg(feature = "std")]
impl std::error::Error for FlvError {}
impl From<Error> for FlvError {
fn from(e: Error) -> Self {
FlvError::Codec(e)
}
}
#[derive(Debug, Default, Clone)]
pub struct FlvDemux<'a> {
_marker: PhantomData<&'a [u8]>,
}
impl FlvDemux<'_> {
pub fn new() -> Self {
Self {
_marker: PhantomData,
}
}
}
struct FlvTag<'a> {
tag_type: u8,
timestamp: u32,
body: &'a [u8],
}
fn iter_tags(input: &[u8]) -> Result<Vec<FlvTag<'_>>> {
if input.len() < FLV_HEADER_LEN + PREV_TAG_SIZE_LEN {
return Err(Error::BufferTooShort {
need: FLV_HEADER_LEN + PREV_TAG_SIZE_LEN,
have: input.len(),
what: "FLV header",
});
}
let data_offset = u32::from_be_bytes([input[5], input[6], input[7], input[8]]) as usize;
let mut off = data_offset.max(FLV_HEADER_LEN) + PREV_TAG_SIZE_LEN;
let mut tags = Vec::new();
while off + TAG_HEADER_LEN <= input.len() {
let tag_type = input[off];
let data_size =
u32::from_be_bytes([0, input[off + 1], input[off + 2], input[off + 3]]) as usize;
let ts_lo = u32::from_be_bytes([0, input[off + 4], input[off + 5], input[off + 6]]);
let ts_ext = input[off + 7] as u32;
let timestamp = (ts_ext << 24) | ts_lo;
let body_start = off + TAG_HEADER_LEN;
let body_end = body_start + data_size;
if body_end + PREV_TAG_SIZE_LEN > input.len() {
return Err(Error::from(FlvErrorAsError(FlvError::TagOverrun {
offset: off,
need: body_end + PREV_TAG_SIZE_LEN,
have: input.len(),
})));
}
tags.push(FlvTag {
tag_type,
timestamp,
body: &input[body_start..body_end],
});
off = body_end + PREV_TAG_SIZE_LEN;
}
Ok(tags)
}
struct FlvErrorAsError(FlvError);
impl From<FlvErrorAsError> for Error {
fn from(w: FlvErrorAsError) -> Self {
match w.0 {
FlvError::TagOverrun { need, have, .. } => Error::BufferTooShort {
need,
have,
what: "FLV tag body",
},
other => Error::InvalidInput(match other {
FlvError::BadSignature(_) => "FLV bad signature",
FlvError::NoSupportedTrack => "FLV no supported track",
FlvError::UnsupportedCodec { .. } => "FLV unsupported codec",
FlvError::HeaderTooLarge { .. } => "FLV header DataOffset too large",
_ => "FLV error",
}),
}
}
}
impl<'a> Unpackage for FlvDemux<'a> {
type Input = &'a [u8];
type Media = Media;
type Error = FlvError;
fn unpackage(&mut self, input: &'a [u8]) -> core::result::Result<Media, FlvError> {
if input.len() < FLV_HEADER_LEN {
return Err(FlvError::Codec(Error::BufferTooShort {
need: FLV_HEADER_LEN,
have: input.len(),
what: "FLV header",
}));
}
if input[0..3] != FLV_SIGNATURE {
return Err(FlvError::BadSignature([input[0], input[1], input[2]]));
}
let tags = iter_tags(input).map_err(FlvError::Codec)?;
let mut avc_config: Option<AVCConfigurationBox> = None;
let mut video_samples: Vec<Sample> = Vec::new();
let mut last_video_dts: Option<u32> = None;
let mut aac_esds: Option<EsdsBox> = None;
let mut aac_channels: u16 = 0;
let mut aac_rate: u32 = 0;
let mut audio_samples: Vec<Sample> = Vec::new();
let mut last_audio_dts: Option<u32> = None;
for tag in &tags {
match tag.tag_type {
tag_type::VIDEO => {
if tag.body.len() < 2 {
continue;
}
let frame_type = tag.body[0] >> 4;
let codec_id = tag.body[0] & 0x0F;
if codec_id != CODEC_ID_AVC {
continue; }
let avc_packet_type = tag.body[1];
if tag.body.len() < 5 {
continue;
}
let composition_time = read_si24(&tag.body[2..5]);
let data = &tag.body[5..];
match avc_packet_type {
avc_packet_type::SEQUENCE_HEADER => {
if avc_config.is_none() && !data.is_empty() {
let record = AVCDecoderConfigurationRecord::parse(data)
.map_err(FlvError::Codec)?;
avc_config = Some(AVCConfigurationBox::new(record));
}
}
avc_packet_type::NALU => {
let dts = tag.timestamp;
let duration = delta_duration(&mut last_video_dts, dts);
let dts_abs = dts as i64;
let pts_abs = dts_abs + composition_time as i64;
video_samples.push(Sample {
data: data.to_vec().into(),
dts: Some(dts_abs),
pts: Some(pts_abs),
duration: Some(duration),
flags: crate::ir::SampleFlags::new(
frame_type == FRAME_TYPE_KEYFRAME,
),
provenance: None,
});
}
avc_packet_type::END_OF_SEQUENCE => {}
_ => {}
}
}
tag_type::AUDIO => {
if tag.body.is_empty() {
continue;
}
let sound_format = tag.body[0] >> 4;
if sound_format != SOUND_FORMAT_AAC {
continue; }
if tag.body.len() < 2 {
continue;
}
let aac_pkt_type = tag.body[1];
let data = &tag.body[2..];
match aac_pkt_type {
aac_packet_type::SEQUENCE_HEADER => {
if aac_esds.is_none() && !data.is_empty() {
let asc =
AudioSpecificConfig::parse(data).map_err(FlvError::Codec)?;
aac_channels = asc.channel_configuration.raw() as u16;
aac_rate = asc_rate_hz(&asc);
aac_esds = Some(build_aac_esds(data.to_vec()));
}
}
aac_packet_type::RAW => {
let dts = tag.timestamp;
let duration = delta_duration(&mut last_audio_dts, dts);
let dts_abs = dts as i64;
audio_samples.push(Sample {
data: data.to_vec().into(),
dts: Some(dts_abs),
pts: Some(dts_abs),
duration: Some(duration),
flags: crate::ir::SampleFlags::SYNC,
provenance: None,
});
}
_ => {}
}
}
tag_type::SCRIPT => { }
_ => { }
}
}
backfill_last_duration(&mut video_samples);
backfill_last_duration(&mut audio_samples);
let mut tracks: Vec<Track> = Vec::new();
let mut track_id = 1u32;
if let Some(config) = avc_config {
if !video_samples.is_empty() {
let (width, height) = config
.config
.sps
.first()
.and_then(|sps| crate::sps::decode_avc_sps(&sps.0).ok())
.map(|i| (i.width as u16, i.height as u16))
.unwrap_or((0, 0));
let anchor = anchor_of(&video_samples);
tracks.push(Track::new_at(
TrackSpec::new(
track_id,
FLV_TIMESCALE,
CodecConfig::Avc {
config,
width,
height,
},
),
video_samples,
anchor,
));
track_id += 1;
}
}
if let Some(esds) = aac_esds {
if !audio_samples.is_empty() {
let anchor = anchor_of(&audio_samples);
tracks.push(Track::new_at(
TrackSpec::new(
track_id,
FLV_TIMESCALE,
CodecConfig::Aac {
esds,
channel_count: aac_channels,
sample_rate: aac_rate,
sample_size: AUDIO_SAMPLE_SIZE_BITS,
},
),
audio_samples,
anchor,
));
}
}
if tracks.is_empty() {
return Err(FlvError::NoSupportedTrack);
}
Ok(Media::new(tracks, FLV_TIMESCALE))
}
}
pub(crate) fn read_si24(b: &[u8]) -> i32 {
let raw = ((b[0] as u32) << 16) | ((b[1] as u32) << 8) | (b[2] as u32);
if raw & 0x0080_0000 != 0 {
(raw | 0xFF00_0000) as i32
} else {
raw as i32
}
}
fn delta_duration(prev: &mut Option<u32>, dts: u32) -> u32 {
let dur = match *prev {
Some(p) => dts.saturating_sub(p),
None => 0,
};
*prev = Some(dts);
dur
}
fn backfill_last_duration(samples: &mut [Sample]) {
let n = samples.len();
if n == 0 {
return;
}
for i in 0..n.saturating_sub(1) {
samples[i].duration = samples[i + 1].duration;
}
if n >= 2 {
samples[n - 1].duration = samples[n - 2].duration;
}
}
fn anchor_of(samples: &[Sample]) -> u64 {
samples
.first()
.and_then(|s| s.dts)
.map(|d| d.max(0) as u64)
.unwrap_or(0)
}
pub(crate) fn build_aac_esds(asc_bytes: Vec<u8>) -> EsdsBox {
EsdsBox::new(ESDescriptor {
es_id: ESDS_AUDIO_ES_ID,
stream_dependence_flag: false,
url_flag: false,
ocr_stream_flag: false,
stream_priority: 0,
depends_on_es_id: None,
url: None,
ocr_es_id: None,
decoder_config: Some(DecoderConfigDescriptor {
object_type_indication: ObjectTypeIndication(OTI_MPEG4_AUDIO),
stream_type: EsdsStreamType(STREAM_TYPE_AUDIO),
up_stream: false,
buffer_size_db: 0,
max_bitrate: 0,
avg_bitrate: 0,
decoder_specific_info: Some(DecoderSpecificInfo { data: asc_bytes }),
}),
sl_config: Some(SLConfigDescriptor {
body: vec![SL_CONFIG_PREDEFINED_MP4],
}),
})
}
pub(crate) fn asc_rate_hz(asc: &AudioSpecificConfig) -> u32 {
if let Some(f) = asc.sampling_frequency {
return f;
}
match asc.sampling_frequency_index.raw() {
0 => 96000,
1 => 88200,
2 => 64000,
3 => 48000,
4 => 44100,
5 => 32000,
6 => 24000,
7 => 22050,
8 => 16000,
9 => 12000,
10 => 11025,
11 => 8000,
12 => 7350,
_ => 0,
}
}
#[derive(Debug, Default, Clone)]
pub struct FlvMux;
impl FlvMux {
pub fn new() -> Self {
Self
}
}
struct OutTag {
tag_type: u8,
timestamp: u32,
body: Vec<u8>,
}
impl OutTag {
fn write_into(&self, out: &mut Vec<u8>) {
let data_size = self.body.len();
let start = out.len();
out.push(self.tag_type);
out.extend_from_slice(&(data_size as u32).to_be_bytes()[1..]); let ts = self.timestamp;
out.push((ts >> 16) as u8);
out.push((ts >> 8) as u8);
out.push(ts as u8);
out.push((ts >> 24) as u8); out.extend_from_slice(&[0, 0, 0]); out.extend_from_slice(&self.body);
let tag_size = (out.len() - start) as u32;
out.extend_from_slice(&tag_size.to_be_bytes()); }
}
impl Package for FlvMux {
type Media = Media;
type Output = Vec<u8>;
type Error = FlvError;
fn package(&mut self, media: &Media) -> core::result::Result<Vec<u8>, FlvError> {
let mut video: Option<&Track> = None;
let mut audio: Option<&Track> = None;
for t in &media.tracks {
match &t.spec.config {
CodecConfig::Avc { .. } if video.is_none() => video = Some(t),
CodecConfig::Aac { .. } if audio.is_none() => audio = Some(t),
CodecConfig::Avc { .. } | CodecConfig::Aac { .. } => {}
other => {
return Err(FlvError::UnsupportedCodec {
codec: codec_name(other),
});
}
}
}
if video.is_none() && audio.is_none() {
return Err(FlvError::NoSupportedTrack);
}
let mut out = Vec::new();
let mut type_flags = 0u8;
if video.is_some() {
type_flags |= TYPE_FLAG_VIDEO;
}
if audio.is_some() {
type_flags |= TYPE_FLAG_AUDIO;
}
out.extend_from_slice(&FLV_SIGNATURE);
out.push(FLV_VERSION);
out.push(type_flags);
out.extend_from_slice(&(FLV_HEADER_LEN as u32).to_be_bytes());
out.extend_from_slice(&0u32.to_be_bytes());
let (width, height) = match video.map(|t| &t.spec.config) {
Some(CodecConfig::Avc { width, height, .. }) => (*width, *height),
_ => (0, 0),
};
let duration_s = media_duration_seconds(media);
let meta = build_onmetadata(duration_s, width, height, video.is_some(), audio.is_some());
OutTag {
tag_type: tag_type::SCRIPT,
timestamp: 0,
body: meta,
}
.write_into(&mut out);
if let Some(vt) = video {
if let CodecConfig::Avc { config, .. } = &vt.spec.config {
let mut avcc = vec![0u8; config.config.serialized_len()];
let n = config
.config
.serialize_into(&mut avcc)
.map_err(FlvError::Codec)?;
avcc.truncate(n);
let mut body = Vec::with_capacity(5 + avcc.len());
body.push((FRAME_TYPE_KEYFRAME << 4) | CODEC_ID_AVC);
body.push(avc_packet_type::SEQUENCE_HEADER);
body.extend_from_slice(&[0, 0, 0]); body.extend_from_slice(&avcc);
OutTag {
tag_type: tag_type::VIDEO,
timestamp: 0,
body,
}
.write_into(&mut out);
}
}
let (sound_type, asc_bytes) = if let Some(at) = audio {
if let CodecConfig::Aac {
esds,
channel_count,
..
} = &at.spec.config
{
let asc = esds_asc_bytes(esds)?;
let st = if *channel_count <= 1 {
SOUND_TYPE_MONO
} else {
SOUND_TYPE_STEREO
};
let mut body = Vec::with_capacity(2 + asc.len());
body.push(audio_tag_header_byte(st));
body.push(aac_packet_type::SEQUENCE_HEADER);
body.extend_from_slice(&asc);
OutTag {
tag_type: tag_type::AUDIO,
timestamp: 0,
body,
}
.write_into(&mut out);
(st, asc)
} else {
(SOUND_TYPE_STEREO, Vec::new())
}
} else {
(SOUND_TYPE_STEREO, Vec::new())
};
let _ = asc_bytes;
let mut items: Vec<(u32, u32, OutTag)> = Vec::new(); let mut seq = 0u32;
if let Some(vt) = video {
let mut dts = 0u32;
for s in &vt.samples {
let comp = s.composition_offset();
let mut body = Vec::with_capacity(5 + s.data.len());
let ft = if s.flags.is_sync {
FRAME_TYPE_KEYFRAME
} else {
FRAME_TYPE_INTER
};
body.push((ft << 4) | CODEC_ID_AVC);
body.push(avc_packet_type::NALU);
body.push((comp >> 16) as u8);
body.push((comp >> 8) as u8);
body.push(comp as u8);
body.extend_from_slice(&s.data);
items.push((
dts,
seq,
OutTag {
tag_type: tag_type::VIDEO,
timestamp: dts,
body,
},
));
seq += 1;
dts = dts.saturating_add(s.duration.unwrap_or(0));
}
}
if let Some(at) = audio {
let mut dts = 0u32;
for s in &at.samples {
let mut body = Vec::with_capacity(2 + s.data.len());
body.push(audio_tag_header_byte(sound_type));
body.push(aac_packet_type::RAW);
body.extend_from_slice(&s.data);
items.push((
dts,
seq,
OutTag {
tag_type: tag_type::AUDIO,
timestamp: dts,
body,
},
));
seq += 1;
dts = dts.saturating_add(s.duration.unwrap_or(0));
}
}
items.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
for (_, _, tag) in &items {
tag.write_into(&mut out);
}
Ok(out)
}
}
fn audio_tag_header_byte(sound_type: u8) -> u8 {
(SOUND_FORMAT_AAC << 4) | (SOUND_RATE_44K << 2) | (SOUND_SIZE_16BIT << 1) | (sound_type & 1)
}
fn esds_asc_bytes(esds: &EsdsBox) -> core::result::Result<Vec<u8>, FlvError> {
esds.es_descriptor
.decoder_config
.as_ref()
.and_then(|dc| dc.decoder_specific_info.as_ref())
.map(|dsi| dsi.data.clone())
.ok_or(FlvError::Codec(Error::InvalidInput(
"AAC esds has no AudioSpecificConfig (DecoderSpecificInfo)",
)))
}
fn codec_name(c: &CodecConfig) -> &'static str {
match c {
CodecConfig::Avc { .. } => "AVC",
CodecConfig::Hevc { .. } => "HEVC",
CodecConfig::Vvc { .. } => "VVC",
CodecConfig::Aac { .. } => "AAC",
CodecConfig::Ac3 { .. } => "AC-3",
CodecConfig::Eac3 { .. } => "E-AC-3",
CodecConfig::Av1 { .. } => "AV1",
CodecConfig::Vp9 { .. } => "VP9",
CodecConfig::Opus { .. } => "Opus",
CodecConfig::Flac { .. } => "FLAC",
CodecConfig::Ac4 { .. } => "AC-4",
CodecConfig::MpegH { .. } => "MPEG-H",
CodecConfig::Mpeg2Video { .. } => "MPEG-2 video",
CodecConfig::MpegAudio { .. } => "MPEG audio",
CodecConfig::Dts { .. } => "DTS",
CodecConfig::Vp8 { .. } => "VP8",
CodecConfig::Vorbis { .. } => "Vorbis",
CodecConfig::Data { .. } => "Data",
CodecConfig::Subtitle { .. } => "Subtitle",
}
}
fn media_duration_seconds(media: &Media) -> f64 {
let mut max = 0.0f64;
for t in &media.tracks {
let ticks: u64 = t
.samples
.iter()
.map(|s| s.duration.unwrap_or(0) as u64)
.sum();
let ts = if t.spec.timescale == 0 {
FLV_TIMESCALE
} else {
t.spec.timescale
} as f64;
let secs = ticks as f64 / ts;
if secs > max {
max = secs;
}
}
max
}
const AMF0_NUMBER: u8 = 0x00;
const AMF0_BOOLEAN: u8 = 0x01;
const AMF0_STRING: u8 = 0x02;
const AMF0_ECMA_ARRAY: u8 = 0x08;
const AMF0_OBJECT_END: u8 = 0x09;
const META_VIDEOCODECID_AVC: f64 = 7.0;
const META_AUDIOCODECID_AAC: f64 = 10.0;
fn amf0_string(out: &mut Vec<u8>, s: &str) {
out.extend_from_slice(&(s.len() as u16).to_be_bytes());
out.extend_from_slice(s.as_bytes());
}
fn amf0_named_number(out: &mut Vec<u8>, key: &str, v: f64) {
amf0_string(out, key);
out.push(AMF0_NUMBER);
out.extend_from_slice(&v.to_be_bytes());
}
fn amf0_named_bool(out: &mut Vec<u8>, key: &str, v: bool) {
amf0_string(out, key);
out.push(AMF0_BOOLEAN);
out.push(v as u8);
}
fn build_onmetadata(
duration: f64,
width: u16,
height: u16,
has_video: bool,
has_audio: bool,
) -> Vec<u8> {
let mut out = Vec::new();
out.push(AMF0_STRING);
amf0_string(&mut out, "onMetaData");
out.push(AMF0_ECMA_ARRAY);
let mut props: Vec<(&str, Prop)> = Vec::new();
props.push(("duration", Prop::Num(duration)));
if has_video {
props.push(("width", Prop::Num(width as f64)));
props.push(("height", Prop::Num(height as f64)));
props.push(("videocodecid", Prop::Num(META_VIDEOCODECID_AVC)));
}
if has_audio {
props.push(("audiocodecid", Prop::Num(META_AUDIOCODECID_AAC)));
props.push(("stereo", Prop::Bool(true)));
}
out.extend_from_slice(&(props.len() as u32).to_be_bytes());
for (k, v) in &props {
match v {
Prop::Num(n) => amf0_named_number(&mut out, k, *n),
Prop::Bool(b) => amf0_named_bool(&mut out, k, *b),
}
}
out.extend_from_slice(&0u16.to_be_bytes());
out.push(AMF0_OBJECT_END);
out
}
enum Prop {
Num(f64),
Bool(bool),
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn si24_sign_extends() {
assert_eq!(read_si24(&[0x00, 0x00, 0x50]), 80);
assert_eq!(read_si24(&[0x00, 0x00, 0x00]), 0);
assert_eq!(read_si24(&[0xFF, 0xFF, 0xFF]), -1);
}
#[test]
fn audio_header_byte_layout() {
assert_eq!(audio_tag_header_byte(SOUND_TYPE_STEREO), 0xAF);
assert_eq!(audio_tag_header_byte(SOUND_TYPE_MONO), 0xAE);
}
}