use std::time::Duration;
use bytes::Bytes;
use moq_mux::catalog::hang::CatalogExt;
use moq_mux::container::Frame as MuxFrame;
use moq_net::Timestamp;
use super::encoder::{Codec, Config, Encoder, Input};
use crate::resample::Resampler;
use crate::{Error, Frame};
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct Options {
pub track: Option<String>,
pub codec: Codec,
pub sample_rate: Option<u32>,
pub channels: Option<u32>,
pub bitrate: Option<u32>,
pub fec: bool,
pub dtx: bool,
pub frame_duration: Duration,
}
impl Default for Options {
fn default() -> Self {
Self {
track: None,
codec: Codec::default(),
sample_rate: None,
channels: None,
bitrate: None,
fec: false,
dtx: false,
frame_duration: Duration::from_millis(20),
}
}
}
impl Options {
fn config(&self, input: Input) -> Config {
Config {
input,
codec: self.codec,
sample_rate: self.sample_rate,
channels: self.channels,
bitrate: self.bitrate,
fec: self.fec,
dtx: self.dtx,
frame_duration: self.frame_duration,
}
}
}
pub struct Producer<E: CatalogExt = ()> {
encoder: Encoder,
resampler: Option<Resampler>,
track: moq_mux::container::Producer<moq_mux::container::legacy::Wire>,
rendition: Rendition<E>,
pending: Vec<f32>,
frames_produced: u64,
epoch_us: Option<u64>,
pending_discontinuity: bool,
decoder_boundary: bool,
}
struct Terminal {
packets: Vec<Bytes>,
end: Timestamp,
start: Timestamp,
frame_size: usize,
codec_rate: u32,
}
impl<E: CatalogExt> Producer<E> {
pub fn new(
broadcast: &mut moq_net::broadcast::Producer,
catalog: moq_mux::catalog::Producer<E>,
input: Input,
options: &Options,
) -> Result<Self, Error> {
let encoder = Encoder::new(&options.config(input))?;
let input = &encoder.config().input;
let resampler = if input.sample_rate == encoder.codec_rate() {
None
} else {
let chunk_frames =
((input.sample_rate as u128 * encoder.config().frame_duration.as_micros()) / 1_000_000) as usize;
Some(Resampler::new(
input.sample_rate,
encoder.codec_rate(),
input.channels,
chunk_frames,
)?)
};
let track = match &options.track {
Some(name) => broadcast.create_track(name.clone(), catalog.track_info())?,
None => broadcast.unique_track(&format!(".{}", options.codec), catalog.track_info())?,
};
let name = track.name().to_string();
let track = catalog.media_producer(track, moq_mux::container::legacy::Wire)?;
let mut catalog_mut = catalog.clone();
let mut config = encoder.catalog();
config.timeline = Some(catalog.timeline(&name)?.section());
catalog_mut.lock().audio.insert(&name, config)?;
Ok(Self {
encoder,
resampler,
track,
rendition: Rendition { catalog, name },
pending: Vec::new(),
frames_produced: 0,
epoch_us: None,
pending_discontinuity: false,
decoder_boundary: true,
})
}
pub fn track_name(&self) -> &str {
&self.rendition.name
}
pub fn track(&self) -> &moq_net::track::Producer {
self.track.track()
}
pub fn bitrate(&self) -> u64 {
self.encoder.bitrate()
}
pub fn set_bitrate(&mut self, bitrate: u64) -> Result<(), Error> {
self.encoder.set_bitrate(bitrate)
}
pub fn reset_epoch(&mut self) {
if self.encoder.started() && !self.decoder_boundary {
self.pending_discontinuity = true;
}
self.reset_state();
}
fn reset_state(&mut self) {
self.epoch_us = None;
self.frames_produced = 0;
self.pending.clear();
self.encoder.reset();
if let Some(resampler) = self.resampler.as_mut() {
resampler.reset();
}
}
pub fn write(&mut self, frame: &Frame) -> Result<(), Error> {
if self.pending_discontinuity {
self.track.discontinuity()?;
self.pending_discontinuity = false;
self.decoder_boundary = true;
}
let timestamp_us = u64::try_from(frame.timestamp.as_micros())
.map_err(|_| Error::Unsupported(format!("frame timestamp {:?} out of range", frame.timestamp)))?;
let epoch_us = *self.epoch_us.get_or_insert(timestamp_us);
let input = &self.encoder.config().input;
let (format, channels) = (input.format, input.channels);
let pcm = format.as_interleaved_f32(frame.data.as_ref(), channels)?;
let pcm: Vec<f32> = match self.resampler.as_mut() {
Some(r) => r.process(&pcm)?,
None => pcm.into_owned(),
};
self.pending.extend(pcm);
self.publish_full_frames(epoch_us)
}
fn publish_full_frames(&mut self, epoch_us: u64) -> Result<(), Error> {
let frame_samples = self.encoder.frame_size() * self.encoder.codec_channels() as usize;
while self.pending.len() >= frame_samples {
let chunk: Vec<f32> = self.pending.drain(..frame_samples).collect();
let packet = self.encoder.encode(&chunk)?;
let timestamp = Self::timestamp(epoch_us, self.frames_produced, self.encoder.codec_rate())?;
self.frames_produced += self.encoder.frame_size() as u64;
Self::publish(&mut self.track, packet, timestamp)?;
self.decoder_boundary = false;
}
Ok(())
}
fn timestamp(epoch_us: u64, frames_produced: u64, codec_rate: u32) -> Result<Timestamp, Error> {
let offset_us = (frames_produced * 1_000_000) / codec_rate as u64;
Ok(Timestamp::from_micros(epoch_us + offset_us)?)
}
fn publish(
track: &mut moq_mux::container::Producer<moq_mux::container::legacy::Wire>,
payload: Bytes,
timestamp: Timestamp,
) -> Result<(), Error> {
let mux_frame = MuxFrame {
timestamp,
payload,
keyframe: true,
duration: None,
};
track.write(mux_frame)?;
track.cut(None)?;
Ok(())
}
fn publish_terminal(
track: &mut moq_mux::container::Producer<moq_mux::container::legacy::Wire>,
terminal: Terminal,
) -> Result<(), Error> {
track.write(MuxFrame {
timestamp: terminal.end,
payload: Bytes::new(),
keyframe: true,
duration: None,
})?;
for (index, packet) in terminal.packets.into_iter().enumerate() {
let offset = Timestamp::from_scale((index * terminal.frame_size) as u64, terminal.codec_rate as u64)?
.convert(terminal.start.scale())?;
track.write(MuxFrame {
timestamp: terminal.start.checked_add(offset)?,
payload: packet,
keyframe: false,
duration: None,
})?;
}
track.cut(Some(terminal.end))?;
Ok(())
}
pub fn discontinuity(&mut self) -> Result<(), Error> {
self.track.discontinuity()?;
self.pending_discontinuity = false;
self.decoder_boundary = true;
self.reset_state();
Ok(())
}
pub fn finish(mut self) -> Result<(), Error> {
if let Some(resampler) = self.resampler.take() {
self.pending.extend(resampler.flush()?);
}
let epoch_us = self.epoch_us.unwrap_or(0);
self.publish_full_frames(epoch_us)?;
let frame_size = self.encoder.frame_size();
let codec_rate = self.encoder.codec_rate();
let channels = self.encoder.codec_channels() as usize;
let source_frames = self.pending.len() / channels;
let start = Self::timestamp(epoch_us, self.frames_produced, codec_rate)?;
let end = Self::timestamp(epoch_us, self.frames_produced + source_frames as u64, codec_rate)?;
let finish = self.encoder.finish(&self.pending)?;
let discard_padding = finish.discard_padding();
let packets = finish.into_packets();
if discard_padding > 0 {
Self::publish_terminal(
&mut self.track,
Terminal {
packets,
end,
start,
frame_size,
codec_rate,
},
)?;
} else {
for packet in packets {
let timestamp = Self::timestamp(epoch_us, self.frames_produced, codec_rate)?;
Self::publish(&mut self.track, packet, timestamp)?;
self.frames_produced += frame_size as u64;
}
}
self.track.finish()?;
Ok(())
}
pub fn abort(self, err: moq_net::Error) {
self.track.abort(err);
}
}
struct Rendition<E: CatalogExt> {
catalog: moq_mux::catalog::Producer<E>,
name: String,
}
impl<E: CatalogExt> Drop for Rendition<E> {
fn drop(&mut self) {
self.catalog.lock().audio.remove(&self.name);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::Format;
use crate::decode::{Config as DecodeConfig, Consumer as AudioConsumer};
#[tokio::test]
async fn finish_publishes_the_opus_lookahead_tail() {
for frames in [960, 860] {
let input = Input {
format: Format::F32,
sample_rate: 48_000,
channels: 1,
};
let options = Options {
track: Some("audio".to_string()),
bitrate: Some(128_000),
..Options::default()
};
let decoder_config = Encoder::new(&options.config(input.clone())).unwrap().catalog();
let mut broadcast = moq_net::broadcast::Info::new().produce();
let catalog = moq_mux::catalog::Producer::new(&mut broadcast).unwrap();
let consumer = broadcast.consume();
let mut producer = Producer::new(&mut broadcast, catalog, input, &options).unwrap();
let mut audio = AudioConsumer::new(&consumer, &decoder_config, "audio", DecodeConfig::new())
.await
.unwrap();
let mut pcm = vec![0.0f32; frames];
let impulse = pcm.len() - 100;
pcm[impulse] = 1.0;
let data: Vec<u8> = pcm.iter().flat_map(|sample| sample.to_le_bytes()).collect();
producer
.write(&Frame {
timestamp: Timestamp::ZERO,
data: Bytes::from(data),
})
.unwrap();
producer.finish().unwrap();
let mut decoded = Vec::new();
while let Some(frame) = audio.read().await.unwrap() {
let pcm = Format::F32.as_interleaved_f32(&frame.data, 1).unwrap();
decoded.extend_from_slice(&pcm);
}
assert_eq!(decoded.len(), frames, "terminal padding extended the source");
let peak = decoded.iter().fold(0.0f32, |peak, sample| peak.max(sample.abs()));
assert!(peak > 0.1, "the {frames}-frame Opus tail lost the impulse: peak {peak}");
}
}
#[tokio::test]
async fn finish_publishes_the_resampled_tail() {
let input = Input {
format: Format::F32,
sample_rate: 44_100,
channels: 1,
};
let mut broadcast = moq_net::broadcast::Info::new().produce();
let catalog = moq_mux::catalog::Producer::new(&mut broadcast).unwrap();
let consumer = broadcast.consume();
let options = Options {
track: Some("audio".to_string()),
..Options::default()
};
let mut producer = Producer::new(&mut broadcast, catalog, input.clone(), &options).unwrap();
let mut track = moq_mux::container::Consumer::new(
consumer
.track("audio")
.unwrap()
.subscribe(moq_net::track::Subscription::default())
.await
.unwrap(),
moq_mux::catalog::hang::Container::Legacy,
);
let data: Vec<u8> = vec![0.25f32; 8_838].iter().flat_map(|s| s.to_le_bytes()).collect();
producer
.write(&Frame {
timestamp: moq_net::Timestamp::ZERO,
data: data.into(),
})
.unwrap();
producer.finish().unwrap();
let mut packets = 0;
while track.read().await.unwrap().is_some() {
packets += 1;
}
assert_eq!(packets, 11);
}
#[tokio::test]
async fn reset_epoch_drops_the_resampler_buffer_too() {
let input = Input {
format: Format::F32,
sample_rate: 44_100,
channels: 1,
};
let mut broadcast = moq_net::broadcast::Info::new().produce();
let catalog = moq_mux::catalog::Producer::new(&mut broadcast).unwrap();
let consumer = broadcast.consume();
let options = Options {
track: Some("audio".to_string()),
..Options::default()
};
let mut producer = Producer::new(&mut broadcast, catalog, input.clone(), &options).unwrap();
let mut track = moq_mux::container::Consumer::new(
consumer
.track("audio")
.unwrap()
.subscribe(moq_net::track::Subscription::default())
.await
.unwrap(),
moq_mux::catalog::hang::Container::Legacy,
);
let data: Vec<u8> = vec![0.25f32; 441].iter().flat_map(|s| s.to_le_bytes()).collect();
producer
.write(&Frame {
timestamp: moq_net::Timestamp::ZERO,
data: data.into(),
})
.unwrap();
producer.reset_epoch();
producer.finish().unwrap();
assert!(track.read().await.unwrap().is_none());
}
#[tokio::test]
async fn reset_epoch_drops_the_encoder_lookahead() {
let mut broadcast = moq_net::broadcast::Info::new().produce();
let catalog = moq_mux::catalog::Producer::new(&mut broadcast).unwrap();
let consumer = broadcast.consume();
let options = Options {
track: Some("audio".to_string()),
..Options::default()
};
let mut producer = Producer::new(
&mut broadcast,
catalog,
Input {
channels: 1,
..Input::default()
},
&options,
)
.unwrap();
let mut track = moq_mux::container::Consumer::new(
consumer
.track("audio")
.unwrap()
.subscribe(moq_net::track::Subscription::default())
.await
.unwrap(),
moq_mux::catalog::hang::Container::Legacy,
);
producer.write(&full_frame(1_000_000)).unwrap();
producer.reset_epoch();
producer.finish().unwrap();
assert!(track.read().await.unwrap().is_some());
assert!(track.read().await.unwrap().is_none());
}
#[tokio::test]
async fn reset_epoch_restarts_the_decoder() {
let input = Input {
format: Format::F32,
sample_rate: 48_000,
channels: 1,
};
let options = Options {
track: Some("audio".to_string()),
..Options::default()
};
let decoder_config = Encoder::new(&options.config(input.clone())).unwrap().catalog();
let mut broadcast = moq_net::broadcast::Info::new().produce();
let catalog = moq_mux::catalog::Producer::new(&mut broadcast).unwrap();
let subscriber = broadcast.consume();
let mut producer = Producer::new(&mut broadcast, catalog, input, &options).unwrap();
let mut audio = AudioConsumer::new(
&subscriber,
&decoder_config,
"audio",
DecodeConfig {
latency_max: Some(Duration::from_millis(500)),
..DecodeConfig::new()
},
)
.await
.unwrap();
producer.write(&full_frame(0)).unwrap();
let first = audio.read().await.unwrap().expect("first epoch packet");
assert_eq!(first.data.len() / size_of::<f32>(), 960 - 312);
producer.reset_epoch();
producer.write(&full_frame(1_000_000)).unwrap();
producer.finish().unwrap();
let mut resumed_frames = 0;
while let Some(frame) = audio.read().await.unwrap() {
assert!(frame.timestamp.as_micros() >= 1_000_000);
resumed_frames += frame.data.len() / size_of::<f32>();
}
assert_eq!(resumed_frames, 960, "the resumed epoch must trim its own pre-skip once");
}
fn full_frame(timestamp_us: u64) -> Frame {
let mut data = Vec::with_capacity(960 * 4);
for _ in 0..960 {
data.extend_from_slice(&0.1f32.to_le_bytes());
}
Frame {
timestamp: Timestamp::from_micros(timestamp_us).unwrap(),
data: data.into(),
}
}
async fn published_pts(frames: &[Frame], reset_before: Option<usize>) -> Vec<u128> {
let mut broadcast = moq_net::broadcast::Info::new().produce();
let catalog = moq_mux::catalog::Producer::new(&mut broadcast).unwrap();
let consumer = broadcast.consume();
let input = Input {
format: Format::F32,
sample_rate: 48_000,
channels: 1,
};
let options = Options {
track: Some("audio".to_string()),
..Options::default()
};
let mut producer = Producer::new(&mut broadcast, catalog, input, &options).unwrap();
let track = consumer.track("audio").unwrap().subscribe(None).await.unwrap();
let mut reader = moq_mux::container::Consumer::new(track, moq_mux::container::legacy::Wire);
let mut pts = Vec::new();
for (i, frame) in frames.iter().enumerate() {
if reset_before == Some(i) {
producer.reset_epoch();
}
producer.write(frame).unwrap();
let read = reader.read().await.unwrap().expect("a packet per full frame");
pts.push(read.timestamp.as_micros());
}
pts
}
#[tokio::test]
async fn epoch_anchors_to_first_frame_timestamp() {
let pts = published_pts(&[full_frame(1_000_000)], None).await;
assert_eq!(pts, vec![1_000_000]);
}
#[tokio::test]
async fn pts_advances_by_frame_duration_ignoring_later_timestamps() {
let pts = published_pts(&[full_frame(1_000), full_frame(999_999)], None).await;
assert_eq!(pts, vec![1_000, 1_000 + 20_000]);
}
#[tokio::test]
async fn reset_epoch_reanchors_so_the_gap_lands_in_pts() {
let pts = published_pts(&[full_frame(0), full_frame(5_000_000)], Some(1)).await;
assert_eq!(pts, vec![0, 5_000_000]);
}
#[tokio::test]
async fn default_options_derive_the_track_name() {
let mut broadcast = moq_net::broadcast::Info::new().produce();
let catalog = moq_mux::catalog::Producer::new(&mut broadcast).unwrap();
let first = Producer::new(&mut broadcast, catalog.clone(), Input::default(), &Options::default()).unwrap();
assert_eq!(first.track_name(), "0.opus");
let second = Producer::new(&mut broadcast, catalog, Input::default(), &Options::default()).unwrap();
assert_eq!(second.track_name(), "1.opus");
}
}