use core::fmt;
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::time::Duration;
use tokio::sync::watch;
use tokio::task::JoinHandle;
use tokio::time::Instant;
use shep_core::protocol::{HelloAck, Request, Response};
use crate::client::{Client, RequestError};
use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
use crate::events::EventStream;
pub const RECONNECT_MIN_DELAY: Duration = Duration::from_millis(50);
pub const RECONNECT_MAX_DELAY: Duration = Duration::from_secs(5);
fn next_delay(current: Duration) -> Duration {
current.saturating_mul(2).min(RECONNECT_MAX_DELAY)
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum LinkState {
Connected,
Reconnecting,
Refused {
daemon_version: Option<String>,
message: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[must_use = "which of the two it was decides whether a dog exits unreachable or refused"]
#[non_exhaustive]
pub enum LinkLost {
Budget {
waited: Duration,
},
Refused {
daemon_version: Option<String>,
message: String,
},
}
impl fmt::Display for LinkLost {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Budget { waited } => write!(f, "no shepherd answered within {waited:?}"),
Self::Refused { message, .. } => {
write!(f, "the shepherd refused this connection: {message}")
}
}
}
}
impl core::error::Error for LinkLost {}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[must_use = "the verdict says whether ids held from before the reconnect still mean anything"]
pub enum Reconnected {
SameDaemon,
NewDaemon,
}
impl Client {
pub async fn reconnect(&mut self) -> Result<Reconnected, ConnectError> {
self.reconnect_within(HANDSHAKE_TIMEOUT).await
}
pub async fn reconnect_within(
&mut self,
budget: Duration,
) -> Result<Reconnected, ConnectError> {
let started = Instant::now();
let socket = self.socket().to_path_buf();
let dog_name = self.dog_name().map(str::to_owned);
let predecessor = self.daemon().pid;
let mut delay = RECONNECT_MIN_DELAY;
loop {
let left = budget.saturating_sub(started.elapsed());
let attempt =
Client::connect_as(&socket, left.min(HANDSHAKE_TIMEOUT), dog_name.as_deref()).await;
let failed = match attempt {
Ok(fresh) => {
let verdict = if fresh.daemon().pid == predecessor {
Reconnected::SameDaemon
} else {
Reconnected::NewDaemon
};
self.replace_connection(fresh);
return Ok(verdict);
}
Err(refusal @ ConnectError::ProtocolMismatch { .. }) => return Err(refusal),
Err(transient) => transient,
};
if budget.saturating_sub(started.elapsed()) <= delay {
return Err(failed);
}
tokio::time::sleep(delay).await;
delay = next_delay(delay);
}
}
}
pub struct ReconnectingClient {
shared: Arc<Shared>,
supervisor: JoinHandle<()>,
}
impl fmt::Debug for ReconnectingClient {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ReconnectingClient")
.field("socket", &self.shared.socket)
.field("dog_name", &self.shared.dog_name)
.field("link", &self.link())
.field("ack", &self.daemon())
.finish_non_exhaustive()
}
}
struct Shared {
socket: PathBuf,
handshake_timeout: Duration,
dog_name: Option<String>,
state: RwLock<State>,
link: watch::Sender<LinkState>,
}
struct State {
client: Arc<Client>,
}
impl Shared {
fn read(&self) -> RwLockReadGuard<'_, State> {
self.state.read().unwrap_or_else(PoisonError::into_inner)
}
fn write(&self) -> RwLockWriteGuard<'_, State> {
self.state.write().unwrap_or_else(PoisonError::into_inner)
}
fn client(&self) -> Arc<Client> {
Arc::clone(&self.read().client)
}
fn set_link(&self, link: LinkState) {
self.link.send_replace(link);
}
fn install(&self, client: Client) {
self.write().client = Arc::new(client);
self.set_link(LinkState::Connected);
}
}
impl ReconnectingClient {
pub async fn connect(socket: &Path) -> Result<Self, ConnectError> {
Self::connect_with_timeout(socket, HANDSHAKE_TIMEOUT).await
}
pub async fn connect_with_timeout(
socket: &Path,
timeout: Duration,
) -> Result<Self, ConnectError> {
Self::connect_inner(socket, timeout, None).await
}
pub async fn connect_as_dog(socket: &Path, name: &str) -> Result<Self, ConnectError> {
Self::connect_as_dog_with_timeout(socket, HANDSHAKE_TIMEOUT, name).await
}
pub async fn connect_as_dog_with_timeout(
socket: &Path,
timeout: Duration,
name: &str,
) -> Result<Self, ConnectError> {
Self::connect_inner(socket, timeout, Some(name)).await
}
async fn connect_inner(
socket: &Path,
timeout: Duration,
dog_name: Option<&str>,
) -> Result<Self, ConnectError> {
let client = Client::connect_as(socket, timeout, dog_name).await?;
let shared = Arc::new(Shared {
socket: socket.to_path_buf(),
handshake_timeout: timeout,
dog_name: dog_name.map(str::to_owned),
link: watch::Sender::new(LinkState::Connected),
state: RwLock::new(State {
client: Arc::new(client),
}),
});
let supervisor = tokio::spawn(supervise(Arc::clone(&shared)));
Ok(Self { shared, supervisor })
}
#[must_use]
pub fn dog_name(&self) -> Option<&str> {
self.shared.dog_name.as_deref()
}
#[must_use]
pub fn daemon(&self) -> HelloAck {
self.shared.read().client.daemon().clone()
}
#[must_use]
pub fn socket(&self) -> &Path {
&self.shared.socket
}
#[must_use]
pub fn link(&self) -> LinkState {
self.shared.link.borrow().clone()
}
pub async fn request(&self, body: Request) -> Result<Response, RequestError> {
self.request_with_deadline(body, None).await
}
pub async fn request_with_deadline(
&self,
body: Request,
deadline: Option<Duration>,
) -> Result<Response, RequestError> {
self.shared
.client()
.request_with_deadline(body, deadline)
.await
}
pub async fn subscribe(&self, topics: Vec<String>) -> Result<EventStream, RequestError> {
self.shared.client().subscribe(topics).await
}
pub async fn connected_within(&self, budget: Duration) -> Result<(), LinkLost> {
let started = Instant::now();
let mut link = self.shared.link.subscribe();
loop {
match link.borrow_and_update().clone() {
LinkState::Connected => return Ok(()),
LinkState::Refused {
daemon_version,
message,
} => {
return Err(LinkLost::Refused {
daemon_version,
message,
});
}
_ => {}
}
let left = budget.saturating_sub(started.elapsed());
if left.is_zero() {
return Err(LinkLost::Budget {
waited: started.elapsed(),
});
}
match tokio::time::timeout(left, link.changed()).await {
Ok(Ok(())) => {}
Ok(Err(_)) | Err(_) => {
return Err(LinkLost::Budget {
waited: started.elapsed(),
});
}
}
}
}
pub async fn link_lost(&self, budget: Duration) -> LinkLost {
let mut link = self.shared.link.subscribe();
loop {
loop {
let up = matches!(*link.borrow_and_update(), LinkState::Connected);
if !up {
break;
}
if link.changed().await.is_err() {
core::future::pending::<()>().await;
}
}
match self.connected_within(budget).await {
Ok(()) => {}
Err(lost) => return lost,
}
}
}
}
impl Drop for ReconnectingClient {
fn drop(&mut self) {
self.supervisor.abort();
}
}
async fn supervise(shared: Arc<Shared>) {
loop {
let generation = shared.client();
generation.closed().await;
drop(generation);
shared.set_link(LinkState::Reconnecting);
let mut delay = RECONNECT_MIN_DELAY;
loop {
match Client::connect_as(
&shared.socket,
shared.handshake_timeout,
shared.dog_name.as_deref(),
)
.await
{
Ok(fresh) => {
shared.install(fresh);
break;
}
Err(ConnectError::ProtocolMismatch {
daemon_version,
message,
..
}) => {
shared.set_link(LinkState::Refused {
daemon_version,
message,
});
return;
}
Err(_transient) => {
tokio::time::sleep(delay).await;
delay = next_delay(delay);
}
}
}
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use shep_core::protocol::{PROTOCOL_VERSION, RpcError, RpcErrorCode};
use super::*;
use crate::testing::{Handovers, Handshake, control_address, fake_daemon_across_handovers};
const BOUND: Duration = Duration::from_secs(5);
const NEGATIVE_WINDOW: Duration = Duration::from_millis(400);
fn ack_from(pid: u32) -> HelloAck {
HelloAck {
daemon_version: format!("0.0.{pid}"),
protocol: PROTOCOL_VERSION,
pid,
min_supported: None,
}
}
async fn await_reconnect(client: &ReconnectingClient, shepherds: &Handovers, accepts: u32) {
let seen = tokio::time::timeout(BOUND, async {
loop {
if shepherds.accepted() >= accepts && client.link() == LinkState::Connected {
return;
}
tokio::task::yield_now().await;
}
})
.await;
assert!(
seen.is_ok(),
"no reconnect within {BOUND:?}: {} accepts, link {:?}",
shepherds.accepted(),
client.link()
);
}
async fn subscribe_then_lose_it(client: &ReconnectingClient, shepherds: &Handovers) {
let mut events = client
.subscribe(vec!["process.*".to_owned()])
.await
.expect("the first subscription must be answered");
shepherds.cut().await;
let ended =
tokio::time::timeout(BOUND, async { while events.next().await.is_some() {} }).await;
assert!(ended.is_ok(), "the stream must end within {BOUND:?}");
}
async fn await_refusal(client: &ReconnectingClient) -> LinkState {
let seen = tokio::time::timeout(BOUND, async {
loop {
let link = client.link();
if matches!(link, LinkState::Refused { .. }) {
return link;
}
tokio::task::yield_now().await;
}
})
.await;
seen.unwrap_or_else(|_| panic!("the link never reached a refusal within {BOUND:?}"))
}
async fn reconnect_ok(client: &mut Client) -> Reconnected {
tokio::time::timeout(BOUND, client.reconnect())
.await
.expect("the reconnect must not hang")
.expect("the successor accepted this handshake")
}
#[tokio::test]
async fn a_dropped_connection_is_re_established_and_the_next_request_is_served() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
let before = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("the first request must not hang");
assert!(before.is_ok(), "before the handover: {before:?}");
shepherds.cut().await;
await_reconnect(&client, &shepherds, 2).await;
let after = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("the request after a handover must not hang");
assert!(
after.is_ok(),
"a request after the handover must be served, got {after:?}"
);
}
#[tokio::test]
async fn the_ack_follows_the_successor_rather_than_the_predecessor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
assert_eq!(client.daemon().pid, 11);
assert_eq!(client.daemon().daemon_version, "0.0.11");
shepherds.cut().await;
await_reconnect(&client, &shepherds, 2).await;
assert_eq!(
client.daemon().daemon_version,
"0.0.22",
"the version must come from the daemon now answering"
);
}
#[tokio::test]
async fn a_dogs_name_rides_every_handshake_including_the_refused_one() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client sent 2".to_string(),
daemon_version: Some("0.2.0".to_string()),
}),
],
);
let client = ReconnectingClient::connect_as_dog(&path, "metrics")
.await
.unwrap();
assert_eq!(client.dog_name(), Some("metrics"));
shepherds.cut().await;
await_refusal(&client).await;
let named: Vec<Option<String>> = shepherds
.hellos()
.into_iter()
.map(|hello| hello.dog_name)
.collect();
assert_eq!(
named,
vec![Some("metrics".to_string()), Some("metrics".to_string())],
"every generation must be told which dog is talking to it"
);
}
#[tokio::test]
async fn a_client_that_is_not_a_dog_names_none() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
assert_eq!(client.dog_name(), None);
shepherds.cut().await;
await_reconnect(&client, &shepherds, 2).await;
assert!(
shepherds
.hellos()
.iter()
.all(|hello| hello.dog_name.is_none()),
"an unnamed client must stay unnamed across a reconnect: {:?}",
shepherds.hellos()
);
}
#[tokio::test]
async fn an_in_flight_request_fails_and_is_never_re_sent_to_the_successor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut_on_next_request();
let lost = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("an abandoned request must fail, not hang");
assert_eq!(
lost,
Err(RequestError::Closed),
"an in-flight request must fail when the daemon is replaced"
);
await_reconnect(&client, &shepherds, 2).await;
let served = tokio::time::timeout(BOUND, client.request(Request::ListFlock))
.await
.expect("the request after a handover must not hang");
assert!(served.is_ok(), "after the handover: {served:?}");
let successor: Vec<Request> = shepherds
.envelopes()
.into_iter()
.filter(|(generation, _)| *generation == 2)
.map(|(_, envelope)| envelope.body)
.collect();
assert_eq!(
successor,
vec![Request::ListFlock],
"the successor must see only the request issued after the reconnect"
);
}
#[tokio::test]
async fn a_refused_reconnect_stops_rather_than_spinning() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client speaks 2".into(),
daemon_version: Some("0.9.9".into()),
}),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
let link = await_refusal(&client).await;
let LinkState::Refused {
daemon_version,
message,
} = link
else {
unreachable!("await_refusal only returns a refusal");
};
assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
assert!(message.contains("protocol 3"), "{message}");
assert_eq!(
shepherds.accepted(),
2,
"one initial connection plus exactly one refused reconnect"
);
let spun = tokio::time::timeout(NEGATIVE_WINDOW, async {
while shepherds.accepted() < 3 {
tokio::task::yield_now().await;
}
})
.await;
assert!(
spun.is_err(),
"the supervisor retried a refusal: {} accepts",
shepherds.accepted()
);
}
#[tokio::test]
async fn a_reconnect_retries_past_a_successor_that_is_not_accepting_yet() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Drop,
Handshake::Drop,
Handshake::Accept(ack_from(44)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
await_reconnect(&client, &shepherds, 4).await;
assert_eq!(
shepherds.accepted(),
4,
"two unanswered handshakes must be retried past, not given up on"
);
let served = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("the request after the retries must not hang");
assert!(served.is_ok(), "after the retries: {served:?}");
}
#[tokio::test]
async fn dropping_the_handle_stops_the_supervisor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![Handshake::Accept(ack_from(11)), Handshake::Drop],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
let reached = tokio::time::timeout(BOUND, async {
while shepherds.accepted() < 2 {
tokio::task::yield_now().await;
}
})
.await;
assert!(reached.is_ok(), "the supervisor never retried at all");
drop(client);
let grew = tokio::time::timeout(NEGATIVE_WINDOW, async {
while shepherds.accepted() < 3 {
tokio::task::yield_now().await;
}
})
.await;
assert!(
grew.is_err(),
"the supervisor kept reconnecting after its handle was dropped: {} accepts",
shepherds.accepted()
);
}
#[test]
fn the_reconnect_ladder_doubles_then_holds_at_the_ceiling() {
let mut delay = RECONNECT_MIN_DELAY;
let mut walked = vec![delay];
for _ in 0..8 {
delay = next_delay(delay);
walked.push(delay);
}
assert_eq!(
walked,
[50, 100, 200, 400, 800, 1600, 3200, 5000, 5000].map(Duration::from_millis)
);
assert_eq!(next_delay(Duration::MAX), RECONNECT_MAX_DELAY);
}
#[tokio::test]
async fn a_reconnect_to_the_same_daemon_reports_same_daemon() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(11)),
],
);
let mut client = Client::connect(&path).await.unwrap();
shepherds.cut().await;
let verdict = reconnect_ok(&mut client).await;
assert_eq!(verdict, Reconnected::SameDaemon);
}
#[tokio::test]
async fn a_reconnect_to_a_daemon_with_another_pid_reports_new_daemon() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let mut client = Client::connect(&path).await.unwrap();
shepherds.cut().await;
let verdict = reconnect_ok(&mut client).await;
assert_eq!(verdict, Reconnected::NewDaemon);
}
#[tokio::test]
async fn a_reconnected_client_sends_its_next_request_to_the_successor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let mut client = Client::connect(&path).await.unwrap();
shepherds.cut().await;
let _ = reconnect_ok(&mut client).await;
let served = tokio::time::timeout(BOUND, client.request(Request::ListFlock))
.await
.expect("the request after a reconnect must not hang");
assert!(served.is_ok(), "after the reconnect: {served:?}");
let successor: Vec<Request> = shepherds
.envelopes()
.into_iter()
.filter(|(generation, _)| *generation == 2)
.map(|(_, envelope)| envelope.body)
.collect();
assert_eq!(successor, vec![Request::ListFlock]);
}
#[tokio::test]
async fn a_reconnect_updates_the_ack_to_the_daemon_now_answering() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let mut client = Client::connect(&path).await.unwrap();
assert_eq!(client.daemon().daemon_version, "0.0.11");
shepherds.cut().await;
let _ = reconnect_ok(&mut client).await;
assert_eq!(client.daemon().pid, 22);
assert_eq!(client.daemon().daemon_version, "0.0.22");
}
#[tokio::test]
async fn a_reconnect_retries_past_a_successor_still_coming_up() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Drop,
Handshake::Drop,
Handshake::Accept(ack_from(44)),
],
);
let mut client = Client::connect(&path).await.unwrap();
shepherds.cut().await;
let verdict = reconnect_ok(&mut client).await;
assert_eq!(verdict, Reconnected::NewDaemon);
assert_eq!(
shepherds.accepted(),
4,
"two unanswered handshakes must be retried past, not given up on"
);
}
#[tokio::test]
async fn a_reconnect_returns_a_refusal_rather_than_retrying_it() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client speaks 2".into(),
daemon_version: Some("0.9.9".into()),
}),
],
);
let mut client = Client::connect(&path).await.unwrap();
shepherds.cut().await;
let refused = tokio::time::timeout(BOUND, client.reconnect_within(BOUND))
.await
.expect("a refusal must be reported, not waited out")
.expect_err("the successor refused this handshake");
let ConnectError::ProtocolMismatch {
daemon_version,
message,
..
} = refused
else {
panic!("a refusal must arrive as a mismatch, got {refused:?}");
};
assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
assert!(message.contains("protocol 3"), "{message}");
assert_eq!(
shepherds.accepted(),
2,
"one initial connection plus exactly one refused reconnect"
);
}
#[tokio::test]
async fn a_reconnect_gives_up_when_its_budget_is_spent() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let mut client = Client::connect(&path).await.unwrap();
drop(shepherds);
let budget = Duration::from_millis(200);
let started = tokio::time::Instant::now();
let gave_up = tokio::time::timeout(BOUND, client.reconnect_within(budget))
.await
.expect("a spent budget must return, not hang");
let elapsed = started.elapsed();
assert!(
gave_up.is_err(),
"a daemon that is gone must not report a reconnect: {gave_up:?}"
);
assert!(
elapsed >= RECONNECT_MIN_DELAY,
"gave up without retrying once, after {elapsed:?}"
);
}
#[tokio::test]
async fn a_dogs_name_rides_a_reconnect() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let mut client = Client::connect_as(&path, HANDSHAKE_TIMEOUT, Some("metrics"))
.await
.unwrap();
shepherds.cut().await;
let _ = reconnect_ok(&mut client).await;
let named: Vec<Option<String>> = shepherds
.hellos()
.into_iter()
.map(|hello| hello.dog_name)
.collect();
assert_eq!(
named,
vec![Some("metrics".to_string()), Some("metrics".to_string())],
"every generation must be told which dog is talking to it"
);
}
#[tokio::test]
async fn a_client_that_is_not_a_dog_stays_anonymous_across_a_reconnect() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let mut client = Client::connect(&path).await.unwrap();
shepherds.cut().await;
let _ = reconnect_ok(&mut client).await;
assert!(
shepherds
.hellos()
.iter()
.all(|hello| hello.dog_name.is_none()),
"an unnamed client must stay unnamed: {:?}",
shepherds.hellos()
);
}
#[tokio::test]
async fn an_upgrade_handover_keeps_its_pid_and_reports_same_daemon() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let upgraded = |version: &str| HelloAck {
daemon_version: version.to_string(),
protocol: PROTOCOL_VERSION,
pid: 11,
min_supported: None,
};
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(upgraded("0.8.0")),
Handshake::Accept(upgraded("0.9.0")),
],
);
let mut client = Client::connect(&path).await.unwrap();
shepherds.cut().await;
let verdict = reconnect_ok(&mut client).await;
assert_eq!(verdict, Reconnected::SameDaemon);
assert_eq!(
client.daemon().daemon_version,
"0.9.0",
"the ack must still follow the build now answering"
);
}
#[tokio::test]
async fn a_restart_onto_the_same_build_reports_new_daemon() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let same_build = |pid: u32| HelloAck {
daemon_version: "0.9.0".to_string(),
protocol: PROTOCOL_VERSION,
pid,
min_supported: None,
};
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(same_build(11)),
Handshake::Accept(same_build(22)),
],
);
let mut client = Client::connect(&path).await.unwrap();
shepherds.cut().await;
let verdict = reconnect_ok(&mut client).await;
assert_eq!(verdict, Reconnected::NewDaemon);
}
#[tokio::test]
async fn a_cancelled_reconnect_leaves_the_handle_on_its_old_connection() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let mut client = Client::connect(&path).await.unwrap();
let cancelled =
tokio::time::timeout(Duration::from_millis(1), client.reconnect_within(BOUND)).await;
assert!(
cancelled.is_err(),
"the reconnect must still have been in flight when it was cancelled"
);
assert_eq!(
client.daemon().pid,
11,
"a cancelled reconnect must not disturb the connection in hand"
);
let served = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("the old connection must still answer");
assert!(served.is_ok(), "after the cancelled reconnect: {served:?}");
assert_eq!(
shepherds.accepted(),
1,
"the fake serves one connection at a time, so the cancelled dial \
never got past the backlog"
);
}
#[tokio::test]
async fn a_wait_for_the_link_returns_once_the_successor_is_up() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(11)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
subscribe_then_lose_it(&client, &shepherds).await;
let waited = tokio::time::timeout(BOUND, client.connected_within(BOUND))
.await
.expect("the wait must not outlive the bound");
assert_eq!(waited, Ok(()), "a successor did come up");
assert_eq!(client.link(), LinkState::Connected);
}
#[tokio::test]
async fn a_wait_on_a_live_link_answers_at_once_even_with_no_budget_left() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
let answered = tokio::time::timeout(BOUND, client.connected_within(Duration::ZERO))
.await
.expect("a live link must answer without waiting");
assert_eq!(answered, Ok(()));
}
#[tokio::test]
async fn a_wait_for_a_shepherd_that_never_comes_spends_its_budget_and_gives_up() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
subscribe_then_lose_it(&client, &shepherds).await;
drop(shepherds);
let budget = Duration::from_millis(200);
let started = tokio::time::Instant::now();
let lost = tokio::time::timeout(BOUND, client.connected_within(budget))
.await
.expect("a spent budget must return, not hang");
let elapsed = started.elapsed();
assert!(
matches!(lost, Err(LinkLost::Budget { .. })),
"expected a spent budget, got {lost:?}"
);
assert!(
elapsed >= budget,
"gave up after {elapsed:?}, short of the {budget:?} it was given"
);
}
#[tokio::test]
async fn a_wait_ends_on_a_refusal_rather_than_serving_out_the_budget() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client speaks 2".into(),
daemon_version: Some("0.9.9".into()),
}),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
subscribe_then_lose_it(&client, &shepherds).await;
let lost = tokio::time::timeout(BOUND, client.connected_within(Duration::from_secs(3600)))
.await
.expect("a refusal must end the wait long before its budget");
let Err(LinkLost::Refused {
daemon_version,
message,
}) = lost
else {
panic!("expected a refusal, got {lost:?}");
};
assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
assert!(message.contains("protocol 3"), "{message}");
}
#[tokio::test]
async fn a_watch_for_a_lost_shepherd_stays_quiet_while_the_link_is_up() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
let fired =
tokio::time::timeout(NEGATIVE_WINDOW, client.link_lost(Duration::from_millis(1))).await;
assert!(
fired.is_err(),
"a connected client reported its shepherd lost: {fired:?}"
);
}
#[tokio::test]
async fn a_watch_for_a_lost_shepherd_rides_out_a_handover() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(11)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
let fired = tokio::time::timeout(NEGATIVE_WINDOW, client.link_lost(BOUND)).await;
assert!(
fired.is_err(),
"a handover was reported as a lost shepherd: {fired:?}"
);
assert_eq!(client.link(), LinkState::Connected);
}
#[tokio::test]
async fn a_watch_for_a_lost_shepherd_ends_on_a_refusal_rather_than_a_budget() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client speaks 2".into(),
daemon_version: Some("0.9.9".into()),
}),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
subscribe_then_lose_it(&client, &shepherds).await;
let lost = tokio::time::timeout(BOUND, client.link_lost(Duration::from_secs(3600)))
.await
.expect("a refusal must end the watch long before its budget");
let LinkLost::Refused {
daemon_version,
message,
} = lost
else {
panic!("expected a refusal, got {lost:?}");
};
assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
assert!(message.contains("protocol 3"), "{message}");
}
#[tokio::test]
async fn a_watch_for_a_lost_shepherd_fires_once_the_budget_is_spent() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
drop(shepherds);
let budget = Duration::from_millis(200);
let lost = tokio::time::timeout(BOUND, client.link_lost(budget))
.await
.expect("a spent budget must fire, not hang");
assert!(
matches!(lost, LinkLost::Budget { .. }),
"expected a spent budget, got {lost:?}"
);
}
#[test]
fn a_lost_link_says_which_of_the_two_it_was() {
let budget = LinkLost::Budget {
waited: Duration::from_secs(5),
};
assert_eq!(budget.to_string(), "no shepherd answered within 5s");
let refused = LinkLost::Refused {
daemon_version: Some("0.9.9".into()),
message: "daemon speaks protocol 3, client speaks 2".into(),
};
assert_eq!(
refused.to_string(),
"the shepherd refused this connection: daemon speaks protocol 3, client speaks 2"
);
}
#[test]
fn reconnected_debug_is_the_verdict_and_nothing_else() {
assert_eq!(format!("{:?}", Reconnected::SameDaemon), "SameDaemon");
assert_eq!(format!("{:?}", Reconnected::NewDaemon), "NewDaemon");
}
#[tokio::test]
async fn client_debug_names_the_socket_the_ack_and_the_dog() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let ack = ack_from(11);
let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack.clone())]);
let client = Client::connect_as(&path, HANDSHAKE_TIMEOUT, Some("metrics"))
.await
.unwrap();
assert_eq!(
format!("{client:?}"),
format!("Client {{ socket: {path:?}, ack: {ack:?}, dog_name: Some(\"metrics\"), .. }}")
);
}
}