use super::*;
use core::fmt;
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::time::Duration;
use tokio::sync::watch;
use tokio::task::JoinHandle;
use crate::client::{Client, RequestError};
use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
use crate::dogs::DogIdentity;
use crate::events::EventStream;
use shep_core::protocol::{HelloAck, Request, Response};
use tokio::time::Instant;
pub struct ReconnectingClient {
shared: Arc<Shared>,
supervisor: JoinHandle<()>,
}
impl fmt::Debug for ReconnectingClient {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ReconnectingClient")
.field("socket", &self.shared.socket)
.field("dog_name", &self.shared.dog_name)
.field("link", &self.link())
.field("ack", &self.daemon())
.finish_non_exhaustive()
}
}
pub(super) struct Shared {
socket: PathBuf,
handshake_timeout: Duration,
dog_name: Option<String>,
state: RwLock<State>,
link: watch::Sender<LinkState>,
}
pub(super) struct State {
client: Arc<Client>,
}
impl Drop for ReconnectingClient {
fn drop(&mut self) {
self.supervisor.abort();
}
}
pub(super) async fn supervise(shared: Arc<Shared>) {
loop {
let generation = shared.client();
generation.closed().await;
drop(generation);
shared.set_link(LinkState::Reconnecting);
let mut delay = RECONNECT_MIN_DELAY;
loop {
match Client::connect_as(
&shared.socket,
shared.handshake_timeout,
shared.dog_name.as_deref(),
)
.await
{
Ok(fresh) => {
shared.install(fresh);
break;
}
Err(ConnectError::ProtocolMismatch {
daemon_version,
message,
..
}) => {
shared.set_link(LinkState::Refused {
daemon_version,
message,
});
return;
}
Err(_transient) => {
tokio::time::sleep(delay).await;
delay = next_delay(delay);
}
}
}
}
}
impl Shared {
fn read(&self) -> RwLockReadGuard<'_, State> {
self.state.read().unwrap_or_else(PoisonError::into_inner)
}
fn write(&self) -> RwLockWriteGuard<'_, State> {
self.state.write().unwrap_or_else(PoisonError::into_inner)
}
fn client(&self) -> Arc<Client> {
Arc::clone(&self.read().client)
}
fn set_link(&self, link: LinkState) {
self.link.send_replace(link);
}
fn install(&self, client: Client) {
self.write().client = Arc::new(client);
self.set_link(LinkState::Connected);
}
}
impl ReconnectingClient {
pub async fn connect(socket: &Path) -> Result<Self, ConnectError> {
Self::connect_with_timeout(socket, HANDSHAKE_TIMEOUT).await
}
pub async fn connect_with_timeout(
socket: &Path,
timeout: Duration,
) -> Result<Self, ConnectError> {
Self::connect_inner(socket, timeout, None).await
}
pub async fn connect_as_dog(socket: &Path, name: &str) -> Result<Self, ConnectError> {
Self::connect_as_dog_with_timeout(socket, HANDSHAKE_TIMEOUT, name).await
}
pub async fn connect_as_dog_with_timeout(
socket: &Path,
timeout: Duration,
name: &str,
) -> Result<Self, ConnectError> {
Self::connect_inner(socket, timeout, Some(name)).await
}
pub async fn connect_as(socket: &Path, identity: &DogIdentity) -> Result<Self, ConnectError> {
Self::connect_inner(socket, HANDSHAKE_TIMEOUT, identity.handshake()).await
}
async fn connect_inner(
socket: &Path,
timeout: Duration,
dog_name: Option<&str>,
) -> Result<Self, ConnectError> {
let client = Client::connect_as(socket, timeout, dog_name).await?;
Ok(Self::supervised(client, socket, timeout, dog_name))
}
pub(crate) async fn connect_over_link(
link: shep_core::transport::ClientStream,
socket: &Path,
timeout: Duration,
dog_name: Option<&str>,
) -> Result<Self, ConnectError> {
let client = Client::connect_over(link, socket, timeout, dog_name).await?;
Ok(Self::supervised(client, socket, timeout, dog_name))
}
fn supervised(
client: Client,
socket: &Path,
timeout: Duration,
dog_name: Option<&str>,
) -> Self {
let shared = Arc::new(Shared {
socket: socket.to_path_buf(),
handshake_timeout: timeout,
dog_name: dog_name.map(str::to_owned),
link: watch::Sender::new(LinkState::Connected),
state: RwLock::new(State {
client: Arc::new(client),
}),
});
let supervisor = tokio::spawn(supervise(Arc::clone(&shared)));
Self { shared, supervisor }
}
#[must_use]
pub fn dog_name(&self) -> Option<&str> {
self.shared.dog_name.as_deref()
}
#[must_use]
pub fn daemon(&self) -> HelloAck {
self.shared.read().client.daemon().clone()
}
#[must_use]
pub fn socket(&self) -> &Path {
&self.shared.socket
}
#[must_use]
pub fn link(&self) -> LinkState {
self.shared.link.borrow().clone()
}
pub async fn request(&self, body: Request) -> Result<Response, RequestError> {
self.request_with_deadline(body, None).await
}
pub async fn request_with_deadline(
&self,
body: Request,
deadline: Option<Duration>,
) -> Result<Response, RequestError> {
self.shared
.client()
.request_with_deadline(body, deadline)
.await
}
pub async fn subscribe(&self, topics: Vec<String>) -> Result<EventStream, RequestError> {
self.shared.client().subscribe(topics).await
}
pub async fn connected_within(&self, budget: Duration) -> Result<(), LinkLost> {
let started = Instant::now();
let mut link = self.shared.link.subscribe();
loop {
match link.borrow_and_update().clone() {
LinkState::Connected => return Ok(()),
LinkState::Refused {
daemon_version,
message,
} => {
return Err(LinkLost::Refused {
daemon_version,
message,
});
}
_ => {}
}
let left = budget.saturating_sub(started.elapsed());
if left.is_zero() {
return Err(LinkLost::Budget {
waited: started.elapsed(),
});
}
match tokio::time::timeout(left, link.changed()).await {
Ok(Ok(())) => {}
Ok(Err(_)) | Err(_) => {
return Err(LinkLost::Budget {
waited: started.elapsed(),
});
}
}
}
}
pub async fn link_lost(&self, budget: Duration) -> LinkLost {
let mut link = self.shared.link.subscribe();
loop {
loop {
let up = matches!(*link.borrow_and_update(), LinkState::Connected);
if !up {
break;
}
if link.changed().await.is_err() {
core::future::pending::<()>().await;
}
}
match self.connected_within(budget).await {
Ok(()) => {}
Err(lost) => return lost,
}
}
}
}
#[cfg(test)]
mod tests {
use std::time::Duration;
use crate::client::{Client, RequestError};
use crate::connection::HANDSHAKE_TIMEOUT;
use shep_core::protocol::Request;
use super::super::testing::*;
use super::*;
use crate::testing::{Handshake, control_address, fake_daemon_across_handovers};
use shep_core::protocol::{RpcError, RpcErrorCode};
#[tokio::test]
async fn a_dogs_name_rides_every_handshake_including_the_refused_one() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client sent 2".to_string(),
daemon_version: Some("0.2.0".to_string()),
}),
],
);
let client = ReconnectingClient::connect_as_dog(&path, "metrics")
.await
.unwrap();
assert_eq!(client.dog_name(), Some("metrics"));
shepherds.cut().await;
await_refusal(&client).await;
let named: Vec<Option<String>> = shepherds
.hellos()
.into_iter()
.map(|hello| hello.dog_name)
.collect();
assert_eq!(
named,
vec![Some("metrics".to_string()), Some("metrics".to_string())],
"every generation must be told which dog is talking to it"
);
}
#[tokio::test]
async fn an_identity_announces_its_handshake_name_or_none() {
let named = DogIdentity::named("rotate");
let anonymous = DogIdentity::from_env(&|_| None, "log-rotate");
for (identity, announced) in [(named, Some("rotate")), (anonymous, None)] {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let hello = crate::testing::fake_daemon(&path, Ok(crate::testing::sample_ack())).await;
let client = ReconnectingClient::connect_as(&path, &identity)
.await
.unwrap();
let hello = tokio::time::timeout(Duration::from_secs(5), hello)
.await
.expect("the fake daemon read the hello")
.unwrap();
assert_eq!(hello.dog_name.as_deref(), announced);
assert_eq!(client.dog_name(), announced);
}
}
#[tokio::test]
async fn an_in_flight_request_fails_and_is_never_re_sent_to_the_successor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut_on_next_request();
let lost = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("an abandoned request must fail, not hang");
assert_eq!(
lost,
Err(RequestError::Closed),
"an in-flight request must fail when the daemon is replaced"
);
await_reconnect(&client, &shepherds, 2).await;
let served = tokio::time::timeout(BOUND, client.request(Request::ListFlock))
.await
.expect("the request after a handover must not hang");
assert!(served.is_ok(), "after the handover: {served:?}");
let successor: Vec<Request> = shepherds
.envelopes()
.into_iter()
.filter(|(generation, _)| *generation == 2)
.map(|(_, envelope)| envelope.body)
.collect();
assert_eq!(
successor,
vec![Request::ListFlock],
"the successor must see only the request issued after the reconnect"
);
}
#[tokio::test]
async fn dropping_the_handle_stops_the_supervisor() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![Handshake::Accept(ack_from(11)), Handshake::Drop],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
let reached = wait_until(BOUND, || shepherds.accepted() >= 2).await;
assert!(reached, "the supervisor never retried at all");
drop(client);
let grew = wait_until(NEGATIVE_WINDOW, || shepherds.accepted() >= 3).await;
assert!(
!grew,
"the supervisor kept reconnecting after its handle was dropped: {} accepts",
shepherds.accepted()
);
}
#[tokio::test]
async fn a_cancelled_reconnect_leaves_the_handle_on_its_old_connection() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(22)),
],
);
let mut client = Client::connect(&path).await.unwrap();
let cancelled =
tokio::time::timeout(Duration::from_millis(1), client.reconnect_within(BOUND)).await;
assert!(
cancelled.is_err(),
"the reconnect must still have been in flight when it was cancelled"
);
assert_eq!(
client.daemon().pid,
11,
"a cancelled reconnect must not disturb the connection in hand"
);
let served = tokio::time::timeout(BOUND, client.request(Request::Ping))
.await
.expect("the old connection must still answer");
assert!(served.is_ok(), "after the cancelled reconnect: {served:?}");
assert_eq!(
shepherds.accepted(),
1,
"the fake serves one connection at a time, so the cancelled dial \
never got past the backlog"
);
}
#[tokio::test]
async fn a_wait_for_the_link_returns_once_the_successor_is_up() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(11)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
subscribe_then_lose_it(&client, &shepherds).await;
let waited = tokio::time::timeout(BOUND, client.connected_within(BOUND))
.await
.expect("the wait must not outlive the bound");
assert_eq!(waited, Ok(()), "a successor did come up");
assert_eq!(client.link(), LinkState::Connected);
}
#[tokio::test]
async fn a_wait_on_a_live_link_answers_at_once_even_with_no_budget_left() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
let answered = tokio::time::timeout(BOUND, client.connected_within(Duration::ZERO))
.await
.expect("a live link must answer without waiting");
assert_eq!(answered, Ok(()));
}
#[tokio::test]
async fn a_wait_for_a_shepherd_that_never_comes_spends_its_budget_and_gives_up() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
subscribe_then_lose_it(&client, &shepherds).await;
drop(shepherds);
let budget = Duration::from_millis(200);
let started = tokio::time::Instant::now();
let lost = tokio::time::timeout(BOUND, client.connected_within(budget))
.await
.expect("a spent budget must return, not hang");
let elapsed = started.elapsed();
assert!(
matches!(lost, Err(LinkLost::Budget { .. })),
"expected a spent budget, got {lost:?}"
);
assert!(
elapsed >= budget,
"gave up after {elapsed:?}, short of the {budget:?} it was given"
);
}
#[tokio::test]
async fn a_wait_ends_on_a_refusal_rather_than_serving_out_the_budget() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client speaks 2".into(),
daemon_version: Some("0.9.9".into()),
}),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
subscribe_then_lose_it(&client, &shepherds).await;
let lost = tokio::time::timeout(BOUND, client.connected_within(Duration::from_secs(3600)))
.await
.expect("a refusal must end the wait long before its budget");
let Err(LinkLost::Refused {
daemon_version,
message,
}) = lost
else {
panic!("expected a refusal, got {lost:?}");
};
assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
assert!(message.contains("protocol 3"), "{message}");
}
#[tokio::test]
async fn a_watch_for_a_lost_shepherd_stays_quiet_while_the_link_is_up() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
let fired =
tokio::time::timeout(NEGATIVE_WINDOW, client.link_lost(Duration::from_millis(1))).await;
assert!(
fired.is_err(),
"a connected client reported its shepherd lost: {fired:?}"
);
}
#[tokio::test]
async fn a_watch_for_a_lost_shepherd_rides_out_a_handover() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Accept(ack_from(11)),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
shepherds.cut().await;
let fired = tokio::time::timeout(NEGATIVE_WINDOW, client.link_lost(BOUND)).await;
assert!(
fired.is_err(),
"a handover was reported as a lost shepherd: {fired:?}"
);
assert_eq!(client.link(), LinkState::Connected);
}
#[tokio::test]
async fn a_watch_for_a_lost_shepherd_ends_on_a_refusal_rather_than_a_budget() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(
&path,
vec![
Handshake::Accept(ack_from(11)),
Handshake::Refuse(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: "daemon speaks protocol 3, client speaks 2".into(),
daemon_version: Some("0.9.9".into()),
}),
],
);
let client = ReconnectingClient::connect(&path).await.unwrap();
subscribe_then_lose_it(&client, &shepherds).await;
let lost = tokio::time::timeout(BOUND, client.link_lost(Duration::from_secs(3600)))
.await
.expect("a refusal must end the watch long before its budget");
let LinkLost::Refused {
daemon_version,
message,
} = lost
else {
panic!("expected a refusal, got {lost:?}");
};
assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
assert!(message.contains("protocol 3"), "{message}");
}
#[tokio::test]
async fn a_watch_for_a_lost_shepherd_fires_once_the_budget_is_spent() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
let client = ReconnectingClient::connect(&path).await.unwrap();
drop(shepherds);
let budget = Duration::from_millis(200);
let lost = tokio::time::timeout(BOUND, client.link_lost(budget))
.await
.expect("a spent budget must fire, not hang");
assert!(
matches!(lost, LinkLost::Budget { .. }),
"expected a spent budget, got {lost:?}"
);
}
#[tokio::test]
async fn client_debug_names_the_socket_the_host_the_ack_and_the_dog() {
let dir = tempfile::tempdir().unwrap();
let path = control_address(dir.path());
let ack = ack_from(11);
let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack.clone())]);
let client = Client::connect_as(&path, HANDSHAKE_TIMEOUT, Some("metrics"))
.await
.unwrap();
assert_eq!(
format!("{client:?}"),
format!(
"Client {{ socket: {path:?}, host: None, ack: {ack:?}, dog_name: Some(\"metrics\"), .. }}"
)
);
}
}