use std::collections::HashMap;
use std::fmt;
use std::net::SocketAddr;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context as _;
use bytes::Bytes;
use evento_core::cursor::{Args, ReadResult, Value};
use evento_core::{
Event, EventFilter, Executor, RoutingKey, TenantFilter, WriteError, WriteInterest,
};
use futures_util::{SinkExt, StreamExt};
use tokio::net::TcpStream;
use tokio::sync::{mpsc, oneshot, watch};
use tokio_rustls::rustls::pki_types::ServerName;
use tokio_rustls::TlsConnector;
use tokio_util::codec::{Framed, LengthDelimitedCodec};
use tokio_util::either::Either;
use ulid::Ulid;
use crate::wire::{
decode_tagged, encode_tagged, ClientFrame, HelloOk, RecordKind, Rejection, Request, Response,
ServerFrame, Token, MAX_FRAME_LENGTH,
};
type ClientStream = Either<TcpStream, tokio_rustls::client::TlsStream<TcpStream>>;
#[derive(Clone)]
struct Tls {
connector: TlsConnector,
server_name: ServerName<'static>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Address(String);
impl From<SocketAddr> for Address {
fn from(addr: SocketAddr) -> Self {
Address(addr.to_string())
}
}
impl From<&str> for Address {
fn from(addr: &str) -> Self {
Address(addr.to_owned())
}
}
impl From<String> for Address {
fn from(addr: String) -> Self {
Address(addr)
}
}
impl fmt::Display for Address {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.0)
}
}
#[derive(Clone)]
struct Dial {
addr: Address,
token: Option<Token>,
tls: Option<Tls>,
}
const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
const RECONNECT_MIN: Duration = Duration::from_millis(100);
const RECONNECT_MAX: Duration = Duration::from_secs(5);
#[derive(Clone, Copy)]
struct Backoff {
min: Duration,
max: Duration,
}
impl Backoff {
fn next_delay(&self, bound: &mut Duration) -> Duration {
let delay = bound.mul_f64(rand::random_range(0.0..=1.0));
*bound = bound.saturating_mul(2).min(self.max);
delay
}
}
const HELLO_ID: u64 = 0;
const REQ_CAPACITY: usize = 1024;
const OUT_CAPACITY: usize = 1024;
type Stable = Arc<AtomicU64>;
fn codec() -> LengthDelimitedCodec {
LengthDelimitedCodec::builder()
.max_frame_length(MAX_FRAME_LENGTH)
.new_codec()
}
type Dispatch = (u64, Request, oneshot::Sender<Response>);
enum WatchCommand {
Add {
watch: u64,
interest: WriteInterest,
tx: watch::Sender<u64>,
},
Remove { watch: u64 },
}
fn watch_frame(watch: u64, interest: &WriteInterest) -> ClientFrame {
ClientFrame::Watch {
watch,
filters: interest.filters.as_deref().map(<[EventFilter]>::to_vec),
routing_key: interest.routing_key.clone(),
tenant: interest.tenant.clone(),
}
}
#[derive(Clone)]
pub struct Client {
inner: Arc<Inner>,
}
struct Inner {
lanes: Vec<mpsc::Sender<Dispatch>>,
round_robin: AtomicUsize,
next_id: AtomicU64,
watches: mpsc::UnboundedSender<WatchCommand>,
next_watch: AtomicU64,
watch_all: std::sync::OnceLock<watch::Sender<u64>>,
runtime: tokio::runtime::Handle,
default_routing_key: Option<String>,
stable: Stable,
request_timeout: Duration,
}
pub struct ClientBuilder {
dial: Dial,
request_timeout: Duration,
connections: usize,
backoff: Backoff,
}
impl ClientBuilder {
pub fn token(mut self, token: impl Into<String>) -> Self {
self.dial.token = Some(Token(token.into()));
self
}
pub fn tls(mut self, connector: TlsConnector, server_name: ServerName<'static>) -> Self {
self.dial.tls = Some(Tls {
connector,
server_name,
});
self
}
pub fn request_timeout(mut self, timeout: Duration) -> Self {
self.request_timeout = timeout;
self
}
pub fn connections(mut self, n: usize) -> Self {
self.connections = n.max(1);
self
}
pub fn reconnect_backoff(mut self, min: Duration, max: Duration) -> Self {
self.backoff = Backoff {
min,
max: max.max(min),
};
self
}
pub async fn connect(self) -> anyhow::Result<Client> {
let stable: Stable = Arc::new(AtomicU64::new(0));
let (watches, watch_rx) = mpsc::unbounded_channel();
let mut watch_rx = Some(watch_rx);
let mut lanes = Vec::with_capacity(self.connections);
let mut default_routing_key = None;
for lane in 0..self.connections {
let (framed, hello) =
connect_and_hello(&self.dial, &stable, self.request_timeout).await?;
if lane == 0 {
default_routing_key = hello.default_routing_key;
}
let (req_tx, req_rx) = mpsc::channel(REQ_CAPACITY);
tokio::spawn(
Actor {
dial: self.dial.clone(),
req_rx,
watch_rx: if lane == 0 { watch_rx.take() } else { None },
watches: HashMap::new(),
stable: Arc::clone(&stable),
request_timeout: self.request_timeout,
backoff: self.backoff,
}
.run(framed),
);
lanes.push(req_tx);
}
Ok(Client {
inner: Arc::new(Inner {
lanes,
round_robin: AtomicUsize::new(0),
next_id: AtomicU64::new(HELLO_ID + 1),
watches,
next_watch: AtomicU64::new(0),
watch_all: std::sync::OnceLock::new(),
runtime: tokio::runtime::Handle::current(),
default_routing_key,
stable,
request_timeout: self.request_timeout,
}),
})
}
}
impl Client {
pub async fn connect(addr: impl Into<Address>) -> anyhow::Result<Client> {
Client::builder(addr).connect().await
}
pub fn builder(addr: impl Into<Address>) -> ClientBuilder {
ClientBuilder {
dial: Dial {
addr: addr.into(),
token: None,
tls: None,
},
request_timeout: DEFAULT_REQUEST_TIMEOUT,
connections: 1,
backoff: Backoff {
min: RECONNECT_MIN,
max: RECONNECT_MAX,
},
}
}
async fn request(&self, request: Request) -> anyhow::Result<Response> {
let id = self.inner.next_id.fetch_add(1, Ordering::Relaxed);
let lane = self.inner.round_robin.fetch_add(1, Ordering::Relaxed) % self.inner.lanes.len();
let (tx, rx) = oneshot::channel();
if self.inner.lanes[lane]
.send((id, request, tx))
.await
.is_err()
{
anyhow::bail!("remote executor connection is closed");
}
match tokio::time::timeout(self.inner.request_timeout, rx).await {
Ok(Ok(response)) => Ok(response),
Ok(Err(_)) => anyhow::bail!("remote executor connection lost"),
Err(_) => anyhow::bail!(
"remote request timed out after {:?}",
self.inner.request_timeout
),
}
}
}
impl Inner {
fn watch(&self, interest: WriteInterest) -> (u64, watch::Sender<u64>) {
let (tx, _) = watch::channel(0u64);
let watch = self.next_watch.fetch_add(1, Ordering::Relaxed);
let _ = self.watches.send(WatchCommand::Add {
watch,
interest,
tx: tx.clone(),
});
(watch, tx)
}
}
fn protocol_err(what: &str) -> anyhow::Error {
anyhow::anyhow!("remote protocol error: unexpected response to {what}")
}
#[async_trait::async_trait]
impl Executor for Client {
fn default_routing_key(&self) -> Option<&str> {
self.inner.default_routing_key.as_deref()
}
fn write_watch(&self) -> Option<watch::Receiver<u64>> {
let all = self.inner.watch_all.get_or_init(|| {
self.inner
.watch(WriteInterest {
filters: None,
routing_key: None,
tenant: TenantFilter::All,
})
.1
});
Some(all.subscribe())
}
fn write_watch_for(&self, interest: &WriteInterest) -> Option<watch::Receiver<u64>> {
let (watch, tx) = self.inner.watch(interest.clone());
let rx = tx.subscribe();
let watches = self.inner.watches.clone();
self.inner.runtime.spawn(async move {
tx.closed().await;
let _ = watches.send(WatchCommand::Remove { watch });
});
Some(rx)
}
async fn stable_timestamp(&self) -> anyhow::Result<Option<u64>> {
match self.inner.stable.load(Ordering::Acquire) {
0 => Ok(None),
v => Ok(Some(v)),
}
}
async fn write(&self, events: Vec<Event>) -> Result<(), WriteError> {
match self.request(Request::Write { events }).await {
Ok(Response::Write(Ok(()))) => Ok(()),
Ok(Response::Write(Err(e))) => Err(e.into()),
Ok(_) => Err(WriteError::Unknown(protocol_err("write"))),
Err(e) => Err(WriteError::Unknown(e)),
}
}
async fn replicate(&self, events: Vec<Event>) -> Result<(), WriteError> {
match self.request(Request::Replicate { events }).await {
Ok(Response::Write(Ok(()))) => Ok(()),
Ok(Response::Write(Err(e))) => Err(e.into()),
Ok(_) => Err(WriteError::Unknown(protocol_err("replicate"))),
Err(e) => Err(WriteError::Unknown(e)),
}
}
async fn read(
&self,
aggregators: Option<Arc<[EventFilter]>>,
routing_key: Option<RoutingKey>,
tenant: TenantFilter,
args: Args,
to_micros: Option<u64>,
) -> anyhow::Result<ReadResult<Event>> {
match self
.request(Request::Read {
aggregators: aggregators.map(|a| a.to_vec()),
routing_key,
tenant,
args,
to_micros,
})
.await?
{
Response::Read(Ok(wire)) => Ok(wire.into()),
Response::Read(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("read")),
}
}
async fn latest_timestamp(
&self,
aggregators: Option<Arc<[EventFilter]>>,
routing_key: Option<RoutingKey>,
tenant: TenantFilter,
) -> anyhow::Result<u64> {
match self
.request(Request::LatestTimestamp {
aggregators: aggregators.map(|a| a.to_vec()),
routing_key,
tenant,
})
.await?
{
Response::LatestTimestamp(Ok(v)) => Ok(v),
Response::LatestTimestamp(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("latest_timestamp")),
}
}
async fn get_subscriber_cursor(&self, key: String) -> anyhow::Result<Option<Value>> {
match self.request(Request::GetSubscriberCursor { key }).await? {
Response::SubscriberCursor(Ok(v)) => Ok(v),
Response::SubscriberCursor(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("get_subscriber_cursor")),
}
}
async fn is_subscriber_running(&self, key: String, worker_id: Ulid) -> anyhow::Result<bool> {
match self
.request(Request::IsSubscriberRunning { key, worker_id })
.await?
{
Response::SubscriberRunning(Ok(v)) => Ok(v),
Response::SubscriberRunning(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("is_subscriber_running")),
}
}
async fn subscriber_status(
&self,
key: String,
worker_id: Ulid,
) -> anyhow::Result<evento_core::SubscriberStatus> {
match self
.request(Request::SubscriberStatus { key, worker_id })
.await?
{
Response::SubscriberStatus(Ok(v)) => Ok(v),
Response::SubscriberStatus(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("subscriber_status")),
}
}
async fn latest_version(
&self,
tenant: String,
aggregate_type: String,
aggregate_id: String,
) -> anyhow::Result<u16> {
match self
.request(Request::LatestVersion {
tenant,
aggregate_type,
aggregate_id,
})
.await?
{
Response::LatestVersion(Ok(v)) => Ok(v),
Response::LatestVersion(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("latest_version")),
}
}
async fn stream_routing_key(
&self,
tenant: String,
aggregate_type: String,
aggregate_id: String,
) -> anyhow::Result<Option<Option<String>>> {
match self
.request(Request::StreamRoutingKey {
tenant,
aggregate_type,
aggregate_id,
})
.await?
{
Response::StreamRoutingKey(Ok(v)) => Ok(v),
Response::StreamRoutingKey(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("stream_routing_key")),
}
}
async fn upsert_subscriber(&self, key: String, worker_id: Ulid) -> anyhow::Result<()> {
match self
.request(Request::UpsertSubscriber { key, worker_id })
.await?
{
Response::Unit(Ok(())) => Ok(()),
Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("upsert_subscriber")),
}
}
async fn acknowledge(
&self,
key: String,
worker_id: Ulid,
cursor: Value,
lag: u64,
) -> anyhow::Result<bool> {
match self
.request(Request::Acknowledge {
key,
worker_id,
cursor,
lag,
})
.await?
{
Response::Acknowledge(Ok(v)) => Ok(v),
Response::Acknowledge(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("acknowledge")),
}
}
async fn get_snapshot(
&self,
tenant: String,
aggregate_type: String,
projection: String,
aggregate_revision: String,
id: String,
) -> anyhow::Result<Option<(Vec<u8>, Value)>> {
match self
.request(Request::GetSnapshot {
tenant,
aggregate_type,
projection,
aggregate_revision,
id,
})
.await?
{
Response::Snapshot(Ok(v)) => Ok(v),
Response::Snapshot(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("get_snapshot")),
}
}
async fn save_snapshot(
&self,
tenant: String,
aggregate_type: String,
projection: String,
aggregate_revision: String,
id: String,
data: Vec<u8>,
cursor: Value,
) -> anyhow::Result<()> {
match self
.request(Request::SaveSnapshot {
tenant,
aggregate_type,
projection,
aggregate_revision,
id,
data,
cursor,
})
.await?
{
Response::Unit(Ok(())) => Ok(()),
Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("save_snapshot")),
}
}
async fn delete_snapshot(
&self,
tenant: String,
aggregate_type: String,
projection: String,
id: String,
) -> anyhow::Result<()> {
match self
.request(Request::DeleteSnapshot {
tenant,
aggregate_type,
projection,
id,
})
.await?
{
Response::Unit(Ok(())) => Ok(()),
Response::Unit(Err(msg)) => Err(anyhow::anyhow!(msg)),
_ => Err(protocol_err("delete_snapshot")),
}
}
}
struct Actor {
dial: Dial,
req_rx: mpsc::Receiver<Dispatch>,
watch_rx: Option<mpsc::UnboundedReceiver<WatchCommand>>,
watches: HashMap<u64, (WriteInterest, watch::Sender<u64>)>,
stable: Stable,
request_timeout: Duration,
backoff: Backoff,
}
impl Actor {
async fn run(mut self, initial: Framed<ClientStream, LengthDelimitedCodec>) {
let mut conn = Some(initial);
let mut pending: HashMap<u64, oneshot::Sender<Response>> = HashMap::new();
loop {
let framed = match conn.take() {
Some(framed) => framed,
None => match self.reconnect().await {
Some(framed) => framed,
None => return, },
};
let (mut sink, mut inbound) = framed.split();
let (out_tx, mut out_rx) = mpsc::channel::<Bytes>(OUT_CAPACITY);
let writer = tokio::spawn(async move {
'writer: while let Some(bytes) = out_rx.recv().await {
if sink.feed(bytes).await.is_err() {
break;
}
while let Ok(bytes) = out_rx.try_recv() {
if sink.feed(bytes).await.is_err() {
break 'writer;
}
}
if sink.flush().await.is_err() {
break;
}
}
});
'connected: {
for (watch, (interest, _)) in &self.watches {
let Ok(bytes) =
encode_tagged(RecordKind::ClientFrame, &watch_frame(*watch, interest))
else {
continue;
};
if out_tx.send(Bytes::from(bytes)).await.is_err() {
break 'connected;
}
}
loop {
tokio::select! {
command = next_watch_command(&mut self.watch_rx) => {
let frame = match command {
WatchCommand::Add { watch, interest, tx } => {
let frame = watch_frame(watch, &interest);
self.watches.insert(watch, (interest, tx));
frame
}
WatchCommand::Remove { watch } => {
self.watches.remove(&watch);
ClientFrame::Unwatch { watch }
}
};
let Ok(bytes) = encode_tagged(RecordKind::ClientFrame, &frame) else {
continue;
};
if out_tx.send(Bytes::from(bytes)).await.is_err() {
break;
}
}
item = self.req_rx.recv() => {
let Some((id, request, reply_tx)) = item else { break };
let frame = ClientFrame::Request { id, request };
let Ok(bytes) = encode_tagged(RecordKind::ClientFrame, &frame) else {
continue; };
pending.insert(id, reply_tx);
if out_tx.send(Bytes::from(bytes)).await.is_err() {
pending.remove(&id);
break;
}
}
item = inbound.next() => {
let Some(Ok(bytes)) = item else { break };
match decode_tagged::<ServerFrame>(RecordKind::ServerFrame, &bytes) {
Ok(ServerFrame::Response { id, response, stable_timestamp }) => {
self.update_stable(stable_timestamp);
if let Some(tx) = pending.remove(&id) {
let _ = tx.send(response);
}
}
Ok(ServerFrame::Notify { watches, stable_timestamp }) => {
self.update_stable(stable_timestamp);
for watch in watches {
if let Some((_, tx)) = self.watches.get(&watch) {
tx.send_modify(|g| *g += 1);
}
}
}
Err(err) => {
tracing::debug!(?err, "undecodable frame, reconnecting");
break;
}
}
}
}
}
}
drop(out_tx);
let _ = writer.await;
pending.clear();
if self.req_rx.is_closed() {
return;
}
}
}
async fn reconnect(&mut self) -> Option<Framed<ClientStream, LengthDelimitedCodec>> {
let mut bound = self.backoff.min;
loop {
tokio::time::sleep(self.backoff.next_delay(&mut bound)).await;
if self.req_rx.is_closed() {
return None;
}
match connect_and_hello(&self.dial, &self.stable, self.request_timeout).await {
Ok((framed, _)) => return Some(framed),
Err(err) => {
if let Some(rejection) = err.downcast_ref::<Rejection>() {
tracing::warn!(%rejection, addr = %self.dial.addr, "the server refused the reconnect, backing off");
} else {
tracing::debug!(?err, addr = %self.dial.addr, "reconnect failed, backing off");
}
}
}
}
}
fn update_stable(&self, stable_timestamp: Option<u64>) {
if let Some(v) = stable_timestamp {
self.stable.fetch_max(v, Ordering::AcqRel);
}
}
}
async fn next_watch_command(
rx: &mut Option<mpsc::UnboundedReceiver<WatchCommand>>,
) -> WatchCommand {
if let Some(commands) = rx {
if let Some(command) = commands.recv().await {
return command;
}
*rx = None;
}
std::future::pending().await
}
async fn connect_and_hello(
dial: &Dial,
stable: &AtomicU64,
timeout: Duration,
) -> anyhow::Result<(Framed<ClientStream, LengthDelimitedCodec>, HelloOk)> {
let stream = TcpStream::connect(dial.addr.0.as_str())
.await
.with_context(|| format!("cannot connect to {}", dial.addr))?;
let _ = stream.set_nodelay(true);
let stream = match &dial.tls {
None => Either::Left(stream),
Some(tls) => Either::Right(
tokio::time::timeout(
timeout,
tls.connector.connect(tls.server_name.clone(), stream),
)
.await
.map_err(|_| anyhow::anyhow!("TLS handshake timed out after {timeout:?}"))??,
),
};
let mut framed = Framed::new(stream, codec());
let hello = ClientFrame::Request {
id: HELLO_ID,
request: Request::Hello {
token: dial.token.clone(),
},
};
framed
.send(Bytes::from(encode_tagged(RecordKind::ClientFrame, &hello)?))
.await?;
let hello = tokio::time::timeout(timeout, async {
loop {
let Some(item) = framed.next().await else {
anyhow::bail!("connection closed during hello");
};
match decode_tagged::<ServerFrame>(RecordKind::ServerFrame, &item?)? {
ServerFrame::Response {
id: HELLO_ID,
response: Response::Hello(reply),
stable_timestamp,
} => {
if let Some(v) = stable_timestamp {
stable.fetch_max(v, Ordering::AcqRel);
}
let ok = reply
.map_err(anyhow::Error::new)
.context("the server refused the connection")?;
return Ok(ok);
}
ServerFrame::Notify {
stable_timestamp, ..
} => {
if let Some(v) = stable_timestamp {
stable.fetch_max(v, Ordering::AcqRel);
}
}
_ => anyhow::bail!("unexpected frame during hello"),
}
}
})
.await
.map_err(|_| anyhow::anyhow!("hello timed out after {timeout:?}"))??;
Ok((framed, hello))
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn a_watch_is_registered_and_dropped_with_its_receivers() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (stream, _) = listener.accept().await.unwrap();
let mut wire = Framed::new(stream, codec());
let mut seen = Vec::new();
while seen.len() < 4 {
let bytes = wire.next().await.unwrap().unwrap();
match decode_tagged::<ClientFrame>(RecordKind::ClientFrame, &bytes).unwrap() {
ClientFrame::Request { id, .. } => {
let reply = ServerFrame::Response {
id,
response: Response::Hello(Ok(HelloOk {
default_routing_key: None,
})),
stable_timestamp: None,
};
let bytes = encode_tagged(RecordKind::ServerFrame, &reply).unwrap();
wire.send(Bytes::from(bytes)).await.unwrap();
}
frame => seen.push(frame),
}
}
seen
});
let client = Client::connect(addr).await.unwrap();
let interest = WriteInterest {
filters: Some([EventFilter::by_type_raw("test/Account")].into()),
routing_key: Some(RoutingKey::Value(None)),
tenant: TenantFilter::All,
};
let accounts = client.write_watch_for(&interest).unwrap();
let clone = accounts.clone();
let all = client.write_watch().unwrap();
let all_again = client.write_watch().unwrap();
drop(accounts);
let cards = client
.write_watch_for(&WriteInterest {
filters: Some([EventFilter::by_type_raw("test/Card")].into()),
..interest
})
.unwrap();
drop(clone);
let seen = tokio::time::timeout(Duration::from_secs(5), server)
.await
.expect("four watch frames within 5s")
.unwrap();
let [ClientFrame::Watch {
watch: accounts,
filters: Some(filters),
routing_key: Some(RoutingKey::Value(None)),
tenant: TenantFilter::All,
}, ClientFrame::Watch {
watch: unfiltered,
filters: None,
routing_key: None,
tenant: TenantFilter::All,
}, ClientFrame::Watch {
watch: cards_id, ..
}, ClientFrame::Unwatch { watch: dropped }] = &seen[..]
else {
panic!("unexpected frames: {seen:?}");
};
assert_eq!(filters, &[EventFilter::by_type_raw("test/Account")]);
assert_eq!(dropped, accounts);
assert!(accounts != unfiltered && unfiltered != cards_id && accounts != cards_id);
drop((all, all_again, cards));
}
#[test]
fn the_reconnect_delay_stays_below_a_doubling_bound() {
let backoff = Backoff {
min: Duration::from_millis(100),
max: Duration::from_secs(5),
};
let mut bound = backoff.min;
let mut bounds = Vec::new();
for _ in 0..8 {
let before = bound;
assert!(backoff.next_delay(&mut bound) <= before);
bounds.push(before.as_millis());
}
assert_eq!(bounds, [100, 200, 400, 800, 1600, 3200, 5000, 5000]);
}
#[test]
fn two_clients_do_not_share_a_reconnect_schedule() {
let backoff = Backoff {
min: Duration::from_secs(5),
max: Duration::from_secs(5),
};
let schedule = || {
let mut bound = backoff.min;
(0..16)
.map(|_| backoff.next_delay(&mut bound))
.collect::<Vec<_>>()
};
assert_ne!(schedule(), schedule());
}
#[test]
fn a_reconnect_maximum_below_the_minimum_is_raised_to_it() {
let builder = Client::builder("127.0.0.1:1")
.reconnect_backoff(Duration::from_secs(2), Duration::from_secs(1));
assert_eq!(builder.backoff.min, Duration::from_secs(2));
assert_eq!(builder.backoff.max, Duration::from_secs(2));
}
}