use std::{
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
thread::{self, JoinHandle},
time::Duration,
};
use crate::pp_log::{PpLog, pp_info, pp_trace};
use crossbeam_channel::{
Receiver, RecvTimeoutError, SendTimeoutError, Sender, TrySendError, bounded, select,
};
use thiserror::Error as ThisError;
use crate::{
buffer::MediaBuffer,
bus::{Bus, BusEvent},
control::{self, ControlMsg, ControlReceiver, ControlSender},
element::{Element, ElementType, Sink, element_pp_log},
error::Result,
};
#[derive(Debug, ThisError)]
pub enum QueueError {
#[error("downstream channel closed")]
ChannelClosed,
#[error("downstream didn't accept a buffer within {after:?} — send timed out")]
SendTimedOut { after: Duration },
}
const STOP_POLL_INTERVAL: Duration = Duration::from_millis(20);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OverflowPolicy {
Block(Duration),
DropNewest,
}
impl Default for OverflowPolicy {
fn default() -> Self {
OverflowPolicy::Block(Duration::MAX)
}
}
pub struct Queue {
pp_log: PpLog,
name: Arc<str>,
tx: Sender<MediaBuffer>,
policy: OverflowPolicy,
bus: Bus,
handle: Option<JoinHandle<()>>,
control: ControlSender,
stop: Arc<AtomicBool>,
}
impl Queue {
pub fn spawn(
name: impl Into<String>,
capacity: usize,
downstream: Box<dyn Sink>,
bus: Bus,
pipeline_id: Option<&str>,
) -> Queue {
Self::spawn_with_policy(
name,
capacity,
downstream,
bus,
OverflowPolicy::default(),
pipeline_id,
)
}
pub fn spawn_with_policy(
name: impl Into<String>,
capacity: usize,
downstream: Box<dyn Sink>,
bus: Bus,
policy: OverflowPolicy,
pipeline_id: Option<&str>,
) -> Queue {
let name: Arc<str> = name.into().into();
let pp_log = element_pp_log(ElementType::Queue, &name, pipeline_id);
pp_info!(pp_log: &pp_log, "spawned: capacity={capacity}, policy={policy:?}");
let (tx, rx) = bounded::<MediaBuffer>(capacity);
let (control_tx, control_rx) = control::channel();
let worker_name = name.clone();
let worker_bus = bus.clone();
let worker_pp_log = pp_log.clone();
let stop = Arc::new(AtomicBool::new(false));
let worker_stop = stop.clone();
let handle = thread::Builder::new()
.name(format!("queue:{worker_name}"))
.spawn(move || {
worker_loop(
rx,
control_rx,
downstream,
worker_bus,
worker_name,
worker_pp_log,
worker_stop,
)
})
.expect("failed to spawn queue worker thread");
Queue {
name,
pp_log,
tx,
policy,
bus,
handle: Some(handle),
control: control_tx,
stop,
}
}
}
impl Element for Queue {
fn name(&self) -> Arc<str> {
self.name.clone()
}
fn element_type(&self) -> ElementType {
ElementType::Queue
}
fn pp_log(&self) -> &PpLog {
&self.pp_log
}
fn pp_log_mut(&mut self) -> &mut PpLog {
&mut self.pp_log
}
}
impl Sink for Queue {
fn consume(&mut self, buf: MediaBuffer) -> Result<()> {
if buf.is_eos() {
pp_trace!(pp_log: &self.pp_log, "event=eos phase=received");
let result = self
.tx
.send(buf)
.map_err(|_| QueueError::ChannelClosed.into());
match &result {
Ok(()) => pp_trace!(
pp_log: &self.pp_log,
"event=eos phase=queued outcome=ok"
),
Err(error) => pp_trace!(
pp_log: &self.pp_log,
"event=eos phase=queued outcome=error error={error}"
),
}
return result;
}
match self.policy {
OverflowPolicy::Block(timeout) => match self.tx.send_timeout(buf, timeout) {
Ok(()) => Ok(()),
Err(SendTimeoutError::Timeout(_)) => {
Err(QueueError::SendTimedOut { after: timeout }.into())
}
Err(SendTimeoutError::Disconnected(_)) => Err(QueueError::ChannelClosed.into()),
},
OverflowPolicy::DropNewest => match self.tx.try_send(buf) {
Ok(()) => Ok(()),
Err(TrySendError::Full(_)) => {
self.bus.post(
&self.pp_log,
BusEvent::Dropped {
element_type: ElementType::Queue,
name: self.name.clone(),
},
);
Ok(())
}
Err(TrySendError::Disconnected(_)) => Err(QueueError::ChannelClosed.into()),
},
}
}
fn control(&mut self, msg: ControlMsg) -> Result<()> {
pp_trace!(
pp_log: &self.pp_log,
"event=control control={msg:?} phase=received"
);
self.control.send(msg);
pp_trace!(
pp_log: &self.pp_log,
"event=control control={msg:?} phase=completed outcome=ok"
);
Ok(())
}
}
impl Drop for Queue {
fn drop(&mut self) {
if let Some(handle) = self.handle.take() {
self.stop.store(true, Ordering::Relaxed);
pp_info!(pp_log: &self.pp_log, "dropped: joining worker");
let _ = handle.join();
}
}
}
fn worker_loop(
data_rx: Receiver<MediaBuffer>,
control_rx: ControlReceiver,
mut downstream: Box<dyn Sink>,
bus: Bus,
name: Arc<str>,
pp_log: PpLog,
stop: Arc<AtomicBool>,
) {
pp_info!(pp_log: &pp_log, "worker: starting");
let error_reporter = QueueErrorReporter {
bus: &bus,
name: &name,
pp_log: &pp_log,
};
loop {
if let Some((msg, ack)) = control_rx.try_recv() {
if apply_control(
&data_rx,
&mut downstream,
msg,
&ack,
&control_rx,
&error_reporter,
&stop,
) {
pp_info!(pp_log: &pp_log, "worker: stopped");
return;
}
continue;
}
select! {
recv(control_rx.rx) -> req => {
match req {
Ok(req) => {
if apply_control(
&data_rx,
&mut downstream,
req.msg,
&req.ack,
&control_rx,
&error_reporter,
&stop,
) {
pp_info!(pp_log: &pp_log, "worker: stopped");
return;
}
}
Err(_) => {
pp_info!(pp_log: &pp_log, "worker: control channel gone, ending");
return; }
}
}
recv(data_rx) -> buf => {
match buf {
Ok(buf) => {
let is_eos = buf.is_eos();
match downstream.consume(buf) {
Ok(()) => {
if is_eos {
pp_trace!(
pp_log: &pp_log,
"event=eos phase=completed outcome=ok"
);
bus.post(
&pp_log,
BusEvent::Eos {
element_type: ElementType::Queue,
name: name.clone(),
},
);
return;
}
}
Err(error) => {
if is_eos {
pp_trace!(
pp_log: &pp_log,
"event=eos phase=completed outcome=error error={error}"
);
}
error_reporter.post(error);
}
}
}
Err(_) => {
pp_info!(pp_log: &pp_log, "worker: producer (this Queue) gone, ending");
return;
}
}
}
default(STOP_POLL_INTERVAL) => {
if stop.load(Ordering::Relaxed) {
pp_info!(pp_log: &pp_log, "worker: stop flag set, ending");
return;
}
}
}
}
}
fn apply_control(
data_rx: &Receiver<MediaBuffer>,
downstream: &mut Box<dyn Sink>,
msg: ControlMsg,
ack: &Sender<()>,
control_rx: &ControlReceiver,
error_reporter: &QueueErrorReporter<'_>,
stop: &AtomicBool,
) -> bool {
pp_trace!(
pp_log: error_reporter.pp_log,
"event=control control={msg:?} phase=forwarding"
);
discard_stale_data(data_rx, msg);
forward_control(downstream, msg, error_reporter);
let is_stop = msg == ControlMsg::Stop;
let _ = ack.send(());
if is_stop {
return true;
}
if msg != ControlMsg::Pause {
return false;
}
loop {
let (msg, ack) = match control_rx.rx.recv_timeout(STOP_POLL_INTERVAL) {
Ok(req) => (req.msg, req.ack),
Err(RecvTimeoutError::Timeout) => {
if stop.load(Ordering::Relaxed) {
pp_info!(pp_log: error_reporter.pp_log, "worker: stop flag set while paused, ending");
return true;
}
continue;
}
Err(RecvTimeoutError::Disconnected) => {
pp_info!(pp_log: error_reporter.pp_log, "worker: control channel gone while paused, ending");
return true; }
};
pp_trace!(
pp_log: error_reporter.pp_log,
"event=control control={msg:?} phase=forwarding"
);
discard_stale_data(data_rx, msg);
forward_control(downstream, msg, error_reporter);
let is_stop = msg == ControlMsg::Stop;
let _ = ack.send(());
if is_stop {
return true;
}
if msg == ControlMsg::Resume {
return false;
}
}
}
struct QueueErrorReporter<'a> {
bus: &'a Bus,
name: &'a Arc<str>,
pp_log: &'a PpLog,
}
impl QueueErrorReporter<'_> {
fn post(&self, error: crate::error::Error) {
self.bus.post(
self.pp_log,
BusEvent::Error {
element_type: ElementType::Queue,
name: self.name.clone(),
error,
},
);
}
}
fn forward_control(
downstream: &mut Box<dyn Sink>,
msg: ControlMsg,
error_reporter: &QueueErrorReporter<'_>,
) {
if let Err(error) = downstream.control(msg) {
error_reporter.post(error);
}
}
fn discard_stale_data(data_rx: &Receiver<MediaBuffer>, msg: ControlMsg) {
if matches!(msg, ControlMsg::Seek(_)) {
while data_rx.try_recv().is_ok() {}
}
}
#[cfg(test)]
mod tests {
use std::{
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
thread,
time::Duration,
};
use super::*;
use crate::bus::Bus;
struct SlowCounter {
pp_log: PpLog,
count: Arc<AtomicUsize>,
}
impl Element for SlowCounter {
fn name(&self) -> Arc<str> {
"slow-counter".into()
}
fn element_type(&self) -> ElementType {
ElementType::Other
}
fn pp_log(&self) -> &PpLog {
&self.pp_log
}
fn pp_log_mut(&mut self) -> &mut PpLog {
&mut self.pp_log
}
}
impl Sink for SlowCounter {
fn consume(&mut self, buf: MediaBuffer) -> Result<()> {
if let MediaBuffer::Packet(_) = buf {
thread::sleep(Duration::from_millis(20));
self.count.fetch_add(1, Ordering::SeqCst);
}
Ok(())
}
fn control(&mut self, _msg: ControlMsg) -> Result<()> {
Ok(())
}
}
fn packet() -> MediaBuffer {
MediaBuffer::Packet(Arc::new(ffmpeg_next::Packet::empty()))
}
#[test]
fn block_never_drops() {
let count = Arc::new(AtomicUsize::new(0));
let sink = SlowCounter {
count: count.clone(),
pp_log: element_pp_log(ElementType::Other, "slow-counter", None),
};
let (bus, bus_rx) = Bus::new();
let mut queue = Queue::spawn_with_policy(
"test",
1,
Box::new(sink),
bus,
OverflowPolicy::default(),
None,
);
for _ in 0..10 {
queue.consume(packet()).unwrap();
}
queue.consume(MediaBuffer::Eos).unwrap();
drop(queue);
assert_eq!(count.load(Ordering::SeqCst), 10);
assert!(!bus_rx.iter().any(|e| matches!(e, BusEvent::Dropped { .. })));
}
#[test]
fn block_with_a_finite_timeout_errors_instead_of_blocking_forever() {
let count = Arc::new(AtomicUsize::new(0));
let sink = SlowCounter {
count: count.clone(),
pp_log: element_pp_log(ElementType::Other, "slow-counter", None),
};
let (bus, _bus_rx) = Bus::new();
let mut queue = Queue::spawn_with_policy(
"test",
1,
Box::new(sink),
bus,
OverflowPolicy::Block(Duration::from_millis(5)),
None,
);
let mut timed_out = 0;
for _ in 0..10 {
match queue.consume(packet()) {
Ok(()) => {}
Err(_) => timed_out += 1,
}
}
queue.consume(MediaBuffer::Eos).unwrap();
drop(queue);
assert!(
timed_out > 0,
"expected at least one send to time out against a downstream that can't keep up"
);
}
#[test]
fn drop_newest_drops_when_full_and_reports_on_bus() {
let count = Arc::new(AtomicUsize::new(0));
let sink = SlowCounter {
count: count.clone(),
pp_log: element_pp_log(ElementType::Other, "slow-counter", None),
};
let (bus, bus_rx) = Bus::new();
let mut queue = Queue::spawn_with_policy(
"test",
1,
Box::new(sink),
bus,
OverflowPolicy::DropNewest,
None,
);
for _ in 0..10 {
queue.consume(packet()).unwrap();
}
queue.consume(MediaBuffer::Eos).unwrap(); drop(queue);
let processed = count.load(Ordering::SeqCst);
let dropped = bus_rx
.iter()
.filter(|e| matches!(e, BusEvent::Dropped { .. }))
.count();
assert!(
processed < 10,
"expected some packets to be dropped, but all {processed} were processed"
);
assert!(dropped > 0, "expected at least one BusEvent::Dropped");
assert_eq!(processed + dropped, 10);
}
#[test]
fn pause_stops_delivery_and_resume_lets_it_continue() {
let count = Arc::new(AtomicUsize::new(0));
let sink = SlowCounter {
count: count.clone(),
pp_log: element_pp_log(ElementType::Other, "slow-counter", None),
};
let (bus, _bus_rx) = Bus::new();
let mut queue = Queue::spawn_with_policy(
"test",
8,
Box::new(sink),
bus,
OverflowPolicy::default(),
None,
);
queue.control(ControlMsg::Pause).unwrap();
for _ in 0..3 {
queue.consume(packet()).unwrap();
}
thread::sleep(Duration::from_millis(100));
assert_eq!(count.load(Ordering::SeqCst), 0);
queue.control(ControlMsg::Resume).unwrap();
queue.consume(MediaBuffer::Eos).unwrap();
drop(queue);
assert_eq!(count.load(Ordering::SeqCst), 3);
}
#[test]
fn dropping_without_stop_or_eos_does_not_hang() {
let count = Arc::new(AtomicUsize::new(0));
let sink = SlowCounter {
count: count.clone(),
pp_log: element_pp_log(ElementType::Other, "slow-counter", None),
};
let (bus, _bus_rx) = Bus::new();
let queue = Queue::spawn_with_policy(
"test",
8,
Box::new(sink),
bus,
OverflowPolicy::default(),
None,
);
drop(queue);
}
#[test]
fn dropping_while_paused_does_not_hang() {
let count = Arc::new(AtomicUsize::new(0));
let sink = SlowCounter {
count: count.clone(),
pp_log: element_pp_log(ElementType::Other, "slow-counter", None),
};
let (bus, _bus_rx) = Bus::new();
let mut queue = Queue::spawn_with_policy(
"test",
8,
Box::new(sink),
bus,
OverflowPolicy::default(),
None,
);
queue.control(ControlMsg::Pause).unwrap(); drop(queue);
}
#[test]
fn stop_is_synchronous_and_terminates_the_worker() {
let count = Arc::new(AtomicUsize::new(0));
let sink = SlowCounter {
count: count.clone(),
pp_log: element_pp_log(ElementType::Other, "slow-counter", None),
};
let (bus, _bus_rx) = Bus::new();
let mut queue = Queue::spawn_with_policy(
"test",
8,
Box::new(sink),
bus,
OverflowPolicy::default(),
None,
);
queue.consume(packet()).unwrap();
queue.control(ControlMsg::Stop).unwrap(); drop(queue); }
struct FailFirstThenCount {
pp_log: PpLog,
count: Arc<AtomicUsize>,
failed_once: bool,
}
impl Element for FailFirstThenCount {
fn name(&self) -> Arc<str> {
"fail-first".into()
}
fn element_type(&self) -> ElementType {
ElementType::Other
}
fn pp_log(&self) -> &PpLog {
&self.pp_log
}
fn pp_log_mut(&mut self) -> &mut PpLog {
&mut self.pp_log
}
}
impl Sink for FailFirstThenCount {
fn consume(&mut self, buf: MediaBuffer) -> Result<()> {
let MediaBuffer::Packet(_) = buf else {
return Ok(());
};
if !self.failed_once {
self.failed_once = true;
return Err(crate::error::Error::Other("simulated failure".into()));
}
self.count.fetch_add(1, Ordering::SeqCst);
Ok(())
}
fn control(&mut self, _msg: ControlMsg) -> Result<()> {
Ok(())
}
}
struct FailControl {
pp_log: PpLog,
}
impl Element for FailControl {
fn name(&self) -> Arc<str> {
"fail-control".into()
}
fn element_type(&self) -> ElementType {
ElementType::Other
}
fn pp_log(&self) -> &PpLog {
&self.pp_log
}
fn pp_log_mut(&mut self) -> &mut PpLog {
&mut self.pp_log
}
}
impl Sink for FailControl {
fn consume(&mut self, _buf: MediaBuffer) -> Result<()> {
Ok(())
}
fn control(&mut self, msg: ControlMsg) -> Result<()> {
Err(crate::error::Error::Other(format!(
"simulated {msg:?} failure"
)))
}
}
#[test]
fn a_failing_consume_drops_that_buffer_but_keeps_the_worker_alive() {
let count = Arc::new(AtomicUsize::new(0));
let sink = FailFirstThenCount {
count: count.clone(),
failed_once: false,
pp_log: element_pp_log(ElementType::Other, "fail-first", None),
};
let (bus, bus_rx) = Bus::new();
let mut queue = Queue::spawn_with_policy(
"test",
8,
Box::new(sink),
bus,
OverflowPolicy::default(),
None,
);
for _ in 0..3 {
queue.consume(packet()).unwrap();
}
queue.consume(MediaBuffer::Eos).unwrap();
drop(queue);
assert_eq!(count.load(Ordering::SeqCst), 2);
let errors = bus_rx
.iter()
.filter(|e| matches!(e, BusEvent::Error { .. }))
.count();
assert_eq!(
errors, 1,
"expected exactly one Error event, for the one buffer that failed"
);
}
#[test]
fn failing_control_is_reported_without_blocking_the_control_cascade() {
let sink = FailControl {
pp_log: element_pp_log(ElementType::Other, "fail-control", None),
};
let (bus, bus_rx) = Bus::new();
let mut queue = Queue::spawn_with_policy(
"test",
1,
Box::new(sink),
bus,
OverflowPolicy::default(),
None,
);
queue.control(ControlMsg::Pause).unwrap();
queue.control(ControlMsg::Resume).unwrap();
queue.control(ControlMsg::Stop).unwrap();
drop(queue);
let errors: Vec<_> = bus_rx
.iter()
.filter(|event| matches!(event, BusEvent::Error { .. }))
.collect();
assert_eq!(
errors.len(),
3,
"Pause, Resume, and Stop failures must each be reported once"
);
}
}