use core::fmt;
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::time::Duration;
use tokio::task::JoinHandle;
use shep_core::protocol::{HelloAck, Request, Response};
use crate::client::{Client, RequestError};
use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
use crate::events::EventStream;
pub const RECONNECT_MIN_DELAY: Duration = Duration::from_millis(50);
pub const RECONNECT_MAX_DELAY: Duration = Duration::from_secs(5);
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum LinkState {
Connected,
Reconnecting,
Refused {
daemon_version: Option<String>,
message: String,
},
}
pub struct ReconnectingClient {
shared: Arc<Shared>,
supervisor: JoinHandle<()>,
}
impl fmt::Debug for ReconnectingClient {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ReconnectingClient")
.field("socket", &self.shared.socket)
.field("dog_name", &self.shared.dog_name)
.field("link", &self.link())
.field("ack", &self.daemon())
.finish_non_exhaustive()
}
}
struct Shared {
socket: PathBuf,
handshake_timeout: Duration,
dog_name: Option<String>,
state: RwLock<State>,
}
struct State {
client: Arc<Client>,
link: LinkState,
}
impl Shared {
fn read(&self) -> RwLockReadGuard<'_, State> {
self.state.read().unwrap_or_else(PoisonError::into_inner)
}
fn write(&self) -> RwLockWriteGuard<'_, State> {
self.state.write().unwrap_or_else(PoisonError::into_inner)
}
fn client(&self) -> Arc<Client> {
Arc::clone(&self.read().client)
}
fn set_link(&self, link: LinkState) {
self.write().link = link;
}
fn install(&self, client: Client) {
let mut state = self.write();
state.client = Arc::new(client);
state.link = LinkState::Connected;
}
}
impl ReconnectingClient {
pub async fn connect(socket: &Path) -> Result<Self, ConnectError> {
Self::connect_with_timeout(socket, HANDSHAKE_TIMEOUT).await
}
pub async fn connect_with_timeout(
socket: &Path,
timeout: Duration,
) -> Result<Self, ConnectError> {
Self::connect_inner(socket, timeout, None).await
}
pub async fn connect_as_dog(socket: &Path, name: &str) -> Result<Self, ConnectError> {
Self::connect_as_dog_with_timeout(socket, HANDSHAKE_TIMEOUT, name).await
}
pub async fn connect_as_dog_with_timeout(
socket: &Path,
timeout: Duration,
name: &str,
) -> Result<Self, ConnectError> {
Self::connect_inner(socket, timeout, Some(name)).await
}
async fn connect_inner(
socket: &Path,
timeout: Duration,
dog_name: Option<&str>,
) -> Result<Self, ConnectError> {
let client = Client::connect_as(socket, timeout, dog_name).await?;
let shared = Arc::new(Shared {
socket: socket.to_path_buf(),
handshake_timeout: timeout,
dog_name: dog_name.map(str::to_owned),
state: RwLock::new(State {
client: Arc::new(client),
link: LinkState::Connected,
}),
});
let supervisor = tokio::spawn(supervise(Arc::clone(&shared)));
Ok(Self { shared, supervisor })
}
#[must_use]
pub fn dog_name(&self) -> Option<&str> {
self.shared.dog_name.as_deref()
}
#[must_use]
pub fn daemon(&self) -> HelloAck {
self.shared.read().client.daemon().clone()
}
#[must_use]
pub fn socket(&self) -> &Path {
&self.shared.socket
}
#[must_use]
pub fn link(&self) -> LinkState {
self.shared.read().link.clone()
}
pub async fn request(&self, body: Request) -> Result<Response, RequestError> {
self.request_with_deadline(body, None).await
}
pub async fn request_with_deadline(
&self,
body: Request,
deadline: Option<Duration>,
) -> Result<Response, RequestError> {
self.shared
.client()
.request_with_deadline(body, deadline)
.await
}
pub async fn subscribe(&self, topics: Vec<String>) -> Result<EventStream, RequestError> {
self.shared.client().subscribe(topics).await
}
}
impl Drop for ReconnectingClient {
fn drop(&mut self) {
self.supervisor.abort();
}
}
async fn supervise(shared: Arc<Shared>) {
loop {
let generation = shared.client();
generation.closed().await;
drop(generation);
shared.set_link(LinkState::Reconnecting);
let mut delay = RECONNECT_MIN_DELAY;
loop {
match Client::connect_as(
&shared.socket,
shared.handshake_timeout,
shared.dog_name.as_deref(),
)
.await
{
Ok(fresh) => {
shared.install(fresh);
break;
}
Err(ConnectError::ProtocolMismatch {
daemon_version,
message,
..
}) => {
shared.set_link(LinkState::Refused {
daemon_version,
message,
});
return;
}
Err(_transient) => {
tokio::time::sleep(delay).await;
delay = (delay * 2).min(RECONNECT_MAX_DELAY);
}
}
}
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use shep_core::protocol::{PROTOCOL_VERSION, RpcError, RpcErrorCode};
use super::*;
use crate::testing::{Handovers, Handshake, control_address, fake_daemon_across_handovers};
const BOUND: Duration = Duration::from_secs(5);
const NEGATIVE_WINDOW: Duration = Duration::from_millis(400);
fn ack_from(pid: u32) -> HelloAck {
HelloAck {
daemon_version: format!("0.0.{pid}"),
protocol: PROTOCOL_VERSION,
pid,
}
}
async fn await_reconnect(client: &ReconnectingClient, shepherds: &Handovers, accepts: u32) {
let seen = tokio::time::timeout(BOUND, async {
loop {
if shepherds.accepted() >= accepts && client.link() == LinkState::Connected {
return;
}
tokio::task::yield_now().await;
}
})
.await;
assert!(
seen.is_ok(),
"no reconnect within {BOUND:?}: {} accepts, link {:?}",
shepherds.accepted(),
client.link()
);
}
async fn await_refusal(client: &ReconnectingClient) -> LinkState {
let seen = tokio::time::timeout(BOUND, async {
loop {
let link = client.link();
if matches!(link, LinkState::Refused { .. }) {
return link;
}
tokio::task::yield_now().await;
}
})
.await;
seen.unwrap_or_else(|_| panic!("the link never reached a refusal within {BOUND:?}"))
}
#[tokio::test]
async fn a_dropped_connection_is_re_established_and_the_next_request_is_served() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
let before = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("the first request must not hang");
assert!(before.is_ok(), "before the handover: {before:?}");
shepherds.cut().await;
await_reconnect(&client, &shepherds, 2).await;
let after = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("the request after a handover must not hang");
assert!(
after.is_ok(),
"a request after the handover must be served, got {after:?}"
);
}
#[tokio::test]
async fn the_ack_follows_the_successor_rather_than_the_predecessor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
assert_eq!(client.daemon().pid, 11);
assert_eq!(client.daemon().daemon_version, "0.0.11");
shepherds.cut().await;
await_reconnect(&client, &shepherds, 2).await;
assert_eq!(
client.daemon().daemon_version,
"0.0.22",
"the version must come from the daemon now answering"
);
}
#[tokio::test]
async fn a_dogs_name_rides_every_handshake_including_the_refused_one() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client sent 2".to_string(),
daemon_version: Some("0.2.0".to_string()),
}),
],
);
let client = ReconnectingClient::connect_as_dog(&path, "metrics")
.await
.unwrap();
assert_eq!(client.dog_name(), Some("metrics"));
shepherds.cut().await;
await_refusal(&client).await;
let named: Vec<Option<String>> = shepherds
.hellos()
.into_iter()
.map(|hello| hello.dog_name)
.collect();
assert_eq!(
named,
vec![Some("metrics".to_string()), Some("metrics".to_string())],
"every generation must be told which dog is talking to it"
);
}
#[tokio::test]
async fn a_client_that_is_not_a_dog_names_none() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
assert_eq!(client.dog_name(), None);
shepherds.cut().await;
await_reconnect(&client, &shepherds, 2).await;
assert!(
shepherds
.hellos()
.iter()
.all(|hello| hello.dog_name.is_none()),
"an unnamed client must stay unnamed across a reconnect: {:?}",
shepherds.hellos()
);
}
#[tokio::test]
async fn an_in_flight_request_fails_and_is_never_re_sent_to_the_successor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut_on_next_request();
let lost = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("an abandoned request must fail, not hang");
assert_eq!(
lost,
Err(RequestError::Closed),
"an in-flight request must fail when the daemon is replaced"
);
await_reconnect(&client, &shepherds, 2).await;
let served = tokio::time::timeout(BOUND, client.request(Request::ListFlock))
.await
.expect("the request after a handover must not hang");
assert!(served.is_ok(), "after the handover: {served:?}");
let successor: Vec<Request> = shepherds
.envelopes()
.into_iter()
.filter(|(generation, _)| *generation == 2)
.map(|(_, envelope)| envelope.body)
.collect();
assert_eq!(
successor,
vec![Request::ListFlock],
"the successor must see only the request issued after the reconnect"
);
}
#[tokio::test]
async fn a_refused_reconnect_stops_rather_than_spinning() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client speaks 2".into(),
daemon_version: Some("0.9.9".into()),
}),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
let link = await_refusal(&client).await;
let LinkState::Refused {
daemon_version,
message,
} = link
else {
unreachable!("await_refusal only returns a refusal");
};
assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
assert!(message.contains("protocol 3"), "{message}");
assert_eq!(
shepherds.accepted(),
2,
"one initial connection plus exactly one refused reconnect"
);
let spun = tokio::time::timeout(NEGATIVE_WINDOW, async {
while shepherds.accepted() < 3 {
tokio::task::yield_now().await;
}
})
.await;
assert!(
spun.is_err(),
"the supervisor retried a refusal: {} accepts",
shepherds.accepted()
);
}
#[tokio::test]
async fn a_reconnect_retries_past_a_successor_that_is_not_accepting_yet() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Drop,
Handshake::Drop,
Handshake::Accept(ack_from(44)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
await_reconnect(&client, &shepherds, 4).await;
assert_eq!(
shepherds.accepted(),
4,
"two unanswered handshakes must be retried past, not given up on"
);
let served = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("the request after the retries must not hang");
assert!(served.is_ok(), "after the retries: {served:?}");
}
#[tokio::test]
async fn dropping_the_handle_stops_the_supervisor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![Handshake::Accept(ack_from(11)), Handshake::Drop],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
let reached = tokio::time::timeout(BOUND, async {
while shepherds.accepted() < 2 {
tokio::task::yield_now().await;
}
})
.await;
assert!(reached.is_ok(), "the supervisor never retried at all");
drop(client);
let grew = tokio::time::timeout(NEGATIVE_WINDOW, async {
while shepherds.accepted() < 3 {
tokio::task::yield_now().await;
}
})
.await;
assert!(
grew.is_err(),
"the supervisor kept reconnecting after its handle was dropped: {} accepts",
shepherds.accepted()
);
}
}