use std::task::{Poll, ready};
use std::time::Duration;
use moq_net::kio;
use moq_net::{BandwidthConsumer, BandwidthProducer, Version};
use url::Url;
use crate::{Client, Error};
#[derive(Clone, Debug, clap::Args, serde::Serialize, serde::Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Backoff {
#[arg(
id = "backoff-initial",
long,
default_value = "1s",
env = "MOQ_BACKOFF_INITIAL",
value_parser = humantime::parse_duration,
)]
#[serde(with = "humantime_serde")]
pub initial: Duration,
#[arg(id = "backoff-multiplier", long, default_value_t = 2, env = "MOQ_BACKOFF_MULTIPLIER")]
pub multiplier: u32,
#[arg(
id = "backoff-max",
long,
default_value = "30s",
env = "MOQ_BACKOFF_MAX",
value_parser = humantime::parse_duration,
)]
#[serde(with = "humantime_serde")]
pub max: Duration,
#[arg(
id = "backoff-timeout",
long,
default_value = "5m",
env = "MOQ_BACKOFF_TIMEOUT",
value_parser = humantime::parse_duration,
)]
#[serde(with = "humantime_serde")]
pub timeout: Duration,
}
impl Default for Backoff {
fn default() -> Self {
Self {
initial: Duration::from_secs(1),
multiplier: 2,
max: Duration::from_secs(30),
timeout: Duration::from_secs(300),
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Status {
Connected,
Disconnected,
}
#[derive(Default)]
struct State {
status: Option<Status>,
version: Option<Version>,
error: Option<Error>,
}
pub struct Reconnect {
abort: tokio::task::AbortHandle,
state: kio::Consumer<State>,
send_bandwidth: BandwidthConsumer,
recv_bandwidth: BandwidthConsumer,
last_reported: Option<Status>,
}
impl Reconnect {
pub(crate) fn new(client: Client, url: Url, backoff: Backoff) -> Self {
let producer = kio::Producer::<State>::default();
let state = producer.consume();
let send_bw = BandwidthProducer::new();
let recv_bw = BandwidthProducer::new();
let send_bandwidth = send_bw.consume();
let recv_bandwidth = recv_bw.consume();
let task = tokio::spawn(async move {
if let Err(err) = Self::run(&producer, &send_bw, &recv_bw, client, url, backoff).await {
tracing::error!(%err, "reconnect loop exited");
if let Ok(mut state) = producer.write() {
state.error = Some(err);
}
}
});
Self {
abort: task.abort_handle(),
state,
send_bandwidth,
recv_bandwidth,
last_reported: None,
}
}
async fn run(
state: &kio::Producer<State>,
send_bw: &BandwidthProducer,
recv_bw: &BandwidthProducer,
client: Client,
url: Url,
backoff: Backoff,
) -> crate::Result<()> {
let mut delay = backoff.initial;
let mut retry_start = tokio::time::Instant::now();
let mut last_error: Option<Error> = None;
loop {
if !backoff.timeout.is_zero() && retry_start.elapsed() > backoff.timeout {
let timeout = backoff.timeout;
let msg = match last_error {
Some(err) => format!("reconnect timed out after {timeout:?}: {err}"),
None => format!("reconnect timed out after {timeout:?}"),
};
return Err(Error::Reconnect(msg));
}
tracing::info!(%url, "connecting");
match client.connect(url.clone()).await {
Ok(session) => {
tracing::info!(%url, "connected");
if let Ok(mut state) = state.write() {
state.status = Some(Status::Connected);
state.version = Some(session.version());
}
let connected = tokio::time::Instant::now();
let closed = run_session(send_bw, recv_bw, &session).await;
if let Ok(mut state) = state.write() {
state.status = Some(Status::Disconnected);
state.version = None;
}
let _ = send_bw.set(None);
let _ = recv_bw.set(None);
if connected.elapsed() >= backoff.initial {
tracing::warn!(%url, "session closed, reconnecting");
delay = backoff.initial;
retry_start = tokio::time::Instant::now();
last_error = None;
} else {
if let Err(err) = closed {
let err = Error::from(err);
tracing::warn!(%url, %err, "session severed immediately, retrying");
last_error = Some(err);
} else {
tracing::warn!(%url, "session severed immediately, retrying");
}
}
}
Err(err) => {
if err.is_auth() {
return Err(err);
}
last_error = Some(err);
}
}
tracing::warn!(%url, ?delay, "reconnecting after backoff");
tokio::time::sleep(delay).await;
delay = std::cmp::min(delay * backoff.multiplier, backoff.max);
}
}
pub fn poll_status(&mut self, waiter: &kio::Waiter) -> Poll<crate::Result<Status>> {
let last = self.last_reported;
let status = match ready!(self.state.poll(waiter, |state| match state.status {
Some(status) if Some(status) != last => Poll::Ready(status),
_ => Poll::Pending,
})) {
Ok(status) => status,
Err(state) => return Poll::Ready(Err(terminal(&state))),
};
self.last_reported = Some(status);
Poll::Ready(Ok(status))
}
pub async fn status(&mut self) -> crate::Result<Status> {
kio::wait(|waiter| self.poll_status(waiter)).await
}
pub fn connected(&self) -> bool {
self.state.read().status == Some(Status::Connected)
}
pub fn version(&self) -> Option<Version> {
self.state.read().version
}
pub fn send_bandwidth(&self) -> BandwidthConsumer {
self.send_bandwidth.clone()
}
pub fn recv_bandwidth(&self) -> BandwidthConsumer {
self.recv_bandwidth.clone()
}
pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll<crate::Result<()>> {
ready!(self.state.poll_closed(waiter));
Poll::Ready(match &self.state.read().error {
Some(err) => Err(err.clone()),
None => Ok(()),
})
}
pub async fn closed(&self) -> crate::Result<()> {
kio::wait(|waiter| self.poll_closed(waiter)).await
}
}
async fn run_session(
send_bw: &BandwidthProducer,
recv_bw: &BandwidthProducer,
session: &moq_net::Session,
) -> Result<(), moq_net::Error> {
let mut send = session.send_bandwidth();
let mut recv = session.recv_bandwidth();
let closed = session.closed();
tokio::pin!(closed);
kio::wait(|waiter| {
poll_forward(&mut send, send_bw, waiter);
poll_forward(&mut recv, recv_bw, waiter);
waiter.poll_future(closed.as_mut())
})
.await
}
fn poll_forward(bw: &mut Option<BandwidthConsumer>, out: &BandwidthProducer, waiter: &kio::Waiter) {
loop {
let Some(consumer) = bw.as_mut() else { return };
let Poll::Ready(rate) = consumer.poll_changed(waiter) else {
return;
};
let _ = out.set(rate);
if rate.is_none() {
*bw = None;
return;
}
}
}
impl Drop for Reconnect {
fn drop(&mut self) {
self.abort.abort();
}
}
fn terminal(state: &State) -> Error {
match &state.error {
Some(err) => err.clone(),
None => Error::Reconnect("reconnect stopped".to_string()),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_backoff_default() {
let backoff = Backoff::default();
assert_eq!(backoff.initial, Duration::from_secs(1));
assert_eq!(backoff.multiplier, 2);
assert_eq!(backoff.max, Duration::from_secs(30));
assert_eq!(backoff.timeout, Duration::from_secs(300));
}
#[test]
fn poll_forward_mirrors_then_drops_on_none() {
let src = BandwidthProducer::new();
let out = BandwidthProducer::new();
let out_rx = out.consume();
let waiter = kio::Waiter::noop();
let mut bw = Some(src.consume());
poll_forward(&mut bw, &out, &waiter);
assert_eq!(out_rx.peek(), None);
assert!(bw.is_some());
src.set(Some(3_000)).unwrap();
poll_forward(&mut bw, &out, &waiter);
assert_eq!(out_rx.peek(), Some(3_000));
src.set(None).unwrap();
poll_forward(&mut bw, &out, &waiter);
assert_eq!(out_rx.peek(), None);
assert!(bw.is_none());
src.set(Some(9_000)).unwrap();
poll_forward(&mut bw, &out, &waiter);
assert_eq!(out_rx.peek(), None);
}
}