shep_client/reconnect/reconnecting.rs
1use super::*;
2use core::fmt;
3use std::collections::BTreeSet;
4use std::path::{Path, PathBuf};
5use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
6use std::time::Duration;
7use tokio::sync::watch;
8use tokio::task::JoinHandle;
9// tokio's Instant, not std's: it moves with `tokio::time::pause`, and the
10// budget below is measured against a `tokio::time::sleep` that does too.
11use crate::client::{Client, RequestError};
12use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
13use crate::dogs::DogIdentity;
14use crate::events::EventStream;
15use shep_core::protocol::{HelloAck, Request, Response};
16use tokio::time::Instant;
17
18/// A [`Client`] that re-establishes its own connection when the daemon it
19/// was talking to is replaced.
20///
21/// Built for dogs, which outlive the shepherd they connected to: a dog's
22/// process crosses a daemon handover as an ordinary child, so it is still
23/// running when its socket dies. The CLI uses a bare [`Client`] instead.
24///
25/// # Example
26///
27/// ```no_run
28/// use shep_client::{ReconnectingClient, shep_core::protocol::Request};
29///
30/// # async fn dog(socket: &std::path::Path) -> Result<(), Box<dyn core::error::Error>> {
31/// // `name` is what the daemon put in `$SHEP_DOG_NAME` when it spawned
32/// // this process; it lets the daemon act on a refusal (G8).
33/// let client = ReconnectingClient::connect_as_dog(socket, "metrics").await?;
34/// // Survives the daemon being replaced underneath it; a request that was
35/// // in flight at the moment it happened still fails rather than retrying.
36/// let _flock = client.request(Request::ListFlock).await?;
37/// # Ok(())
38/// # }
39/// # let _ = dog;
40/// ```
41pub struct ReconnectingClient {
42 shared: Arc<Shared>,
43 supervisor: JoinHandle<()>,
44}
45
46/// Manual, not derived: the socket path and the [`HelloAck`] carry no
47/// secret, and neither does [`LinkState`], whose one payload is the
48/// daemon's own refusal sentence. Manual because [`RwLock`] and
49/// [`JoinHandle`] derive into noise, and a derived impl would print the
50/// inner [`Client`] where the link state is the useful fact.
51impl fmt::Debug for ReconnectingClient {
52 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53 f.debug_struct("ReconnectingClient")
54 .field("socket", &self.shared.socket)
55 .field("dog_name", &self.shared.dog_name)
56 .field("link", &self.link())
57 .field("ack", &self.daemon())
58 .finish_non_exhaustive()
59 }
60}
61
62/// Everything the handle and its supervisor both reach: the address to
63/// reconnect to, the budget to do it under, and the current generation.
64pub(super) struct Shared {
65 socket: PathBuf,
66 handshake_timeout: Duration,
67 /// The name this client announces itself as a dog under, re-sent on
68 /// every reconnect. See [`ReconnectingClient::connect_as_dog`].
69 dog_name: Option<String>,
70 state: RwLock<State>,
71 /// What the supervisor last reported, and how a waiter hears it
72 /// change. The only home of the link state: a second copy beside the
73 /// generation would be one more thing to keep in step, and the two
74 /// have no reader that needs them to move together.
75 link: watch::Sender<LinkState>,
76}
77
78/// The generation of the connection in force right now.
79pub(super) struct State {
80 client: Arc<Client>,
81 /// Counts the connections made, from 1: what a daemon-side topic filter
82 /// belongs to, since a successor starts with none.
83 generation: u64,
84}
85
86/// The topics a connection's daemon-side filter is known to hold.
87///
88/// `Default` knows nothing, so the next subscription is sent.
89#[derive(Debug, Default)]
90pub(crate) struct Covered {
91 generation: u64,
92 topics: BTreeSet<String>,
93}
94
95/// Stops the supervisor when the handle goes away.
96///
97/// Without this, a dropped `ReconnectingClient` whose daemon is gone leaves
98/// a task reconnecting forever, holding the last generation's socket open
99/// with it. `abort` rather than a cooperative signal: the supervisor is
100/// parked on a connect attempt that would not see a flag.
101impl Drop for ReconnectingClient {
102 fn drop(&mut self) {
103 self.supervisor.abort();
104 }
105}
106
107/// The supervisor loop: wait for the current generation to die, then
108/// re-establish it, forever.
109///
110/// Ends only on a protocol refusal (see [`LinkState::Refused`]) or when the
111/// handle is dropped and this task is aborted.
112pub(super) async fn supervise(shared: Arc<Shared>) {
113 loop {
114 // held only for the death await, dropped before reconnecting so the
115 // dead socket goes away as soon as `install` replaces it
116 let generation = shared.client();
117 generation.closed().await;
118 drop(generation);
119 shared.set_link(LinkState::Reconnecting);
120
121 let mut delay = RECONNECT_MIN_DELAY;
122 loop {
123 match Client::connect_as(
124 &shared.socket,
125 shared.handshake_timeout,
126 shared.dog_name.as_deref(),
127 )
128 .await
129 {
130 Ok(fresh) => {
131 shared.install(fresh);
132 break;
133 }
134 Err(ConnectError::ProtocolMismatch {
135 daemon_version,
136 message,
137 ..
138 }) => {
139 shared.set_link(LinkState::Refused {
140 daemon_version,
141 message,
142 });
143 return;
144 }
145 // Everything else is a daemon that is not ready yet: the
146 // successor has not started accepting, or the socket is
147 // momentarily unbound. Those resolve on their own, so the
148 // only question is how often to ask.
149 Err(_transient) => {
150 tokio::time::sleep(delay).await;
151 delay = next_delay(delay);
152 }
153 }
154 }
155 }
156}
157
158impl Shared {
159 /// A read guard, treating a poisoned lock as ordinary data.
160 ///
161 /// Nothing inside a critical section here can panic: every one is a
162 /// clone or an assignment of a plain struct, so poisoning cannot
163 /// signal a torn value.
164 fn read(&self) -> RwLockReadGuard<'_, State> {
165 self.state.read().unwrap_or_else(PoisonError::into_inner)
166 }
167
168 /// A write guard. See [`Self::read`] for the poisoning argument.
169 fn write(&self) -> RwLockWriteGuard<'_, State> {
170 self.state.write().unwrap_or_else(PoisonError::into_inner)
171 }
172
173 /// The current generation, cloned out so the guard is dropped before
174 /// any caller awaits on it: no lock is ever held across an `await`.
175 fn client(&self) -> Arc<Client> {
176 Arc::clone(&self.read().client)
177 }
178
179 /// The current generation and its number, read together.
180 fn current(&self) -> (Arc<Client>, u64) {
181 let state = self.read();
182 (Arc::clone(&state.client), state.generation)
183 }
184
185 fn set_link(&self, link: LinkState) {
186 self.link.send_replace(link);
187 }
188
189 /// Swaps in a freshly handshaken generation, dropping the dead one.
190 ///
191 /// The generation lands before the link is announced, so a waiter woken
192 /// by [`LinkState::Connected`] finds the connection it was told about.
193 fn install(&self, client: Client) {
194 let mut state = self.write();
195 state.client = Arc::new(client);
196 state.generation += 1;
197 drop(state);
198 self.set_link(LinkState::Connected);
199 }
200}
201
202impl ReconnectingClient {
203 /// Connects to `socket`, performs the version handshake bounded by
204 /// [`HANDSHAKE_TIMEOUT`], and starts the supervisor that will
205 /// re-establish this connection whenever it dies.
206 ///
207 /// The FIRST connection is not supervised: a socket nobody is listening
208 /// on is a caller's error, not a handover, so this reports it rather
209 /// than retrying behind the caller's back.
210 ///
211 /// # Errors
212 ///
213 /// See [`Self::connect_with_timeout`].
214 pub async fn connect(socket: &Path) -> Result<Self, ConnectError> {
215 Self::connect_with_timeout(socket, HANDSHAKE_TIMEOUT).await
216 }
217
218 /// As [`Self::connect`], but with a caller-supplied handshake timeout,
219 /// used for the first connection and for every reconnect after it.
220 ///
221 /// # Errors
222 ///
223 /// - [`ConnectError::Connect`]: the initial `connect(2)` call failed.
224 /// - [`ConnectError::Wire`]: `Hello` failed to encode, or the reply failed to decode.
225 /// - [`ConnectError::Io`]: a framed read or write failed after connect.
226 /// - [`ConnectError::HandshakeClosed`]: the peer closed before a `HelloReply`.
227 /// - [`ConnectError::HandshakeTimeout`]: no `HelloReply` arrived within `timeout`.
228 /// - [`ConnectError::ProtocolMismatch`]: the daemon refused on protocol-version skew.
229 pub async fn connect_with_timeout(
230 socket: &Path,
231 timeout: Duration,
232 ) -> Result<Self, ConnectError> {
233 Self::connect_inner(socket, timeout, None).await
234 }
235
236 /// As [`Self::connect`], but announcing this client as the dog
237 /// registered under `name`, the value the daemon put in
238 /// `$SHEP_DOG_NAME` when it spawned this process.
239 ///
240 /// A dog should use this rather than [`Self::connect`]: the name makes
241 /// a refused handshake actionable, since a refusal never reaches a
242 /// request. Without it, the daemon still stops rather than spins on a
243 /// refusal, but cannot say which dog went stale or restart it.
244 ///
245 /// # Errors
246 ///
247 /// See [`Self::connect_with_timeout`].
248 pub async fn connect_as_dog(socket: &Path, name: &str) -> Result<Self, ConnectError> {
249 Self::connect_as_dog_with_timeout(socket, HANDSHAKE_TIMEOUT, name).await
250 }
251
252 /// As [`Self::connect_as_dog`], but with a caller-supplied handshake
253 /// timeout, used for the first connection and for every reconnect after
254 /// it.
255 ///
256 /// # Errors
257 ///
258 /// See [`Self::connect_with_timeout`].
259 pub async fn connect_as_dog_with_timeout(
260 socket: &Path,
261 timeout: Duration,
262 name: &str,
263 ) -> Result<Self, ConnectError> {
264 Self::connect_inner(socket, timeout, Some(name)).await
265 }
266
267 /// Connects as `identity` says: as its dog when a shepherd named this
268 /// process, and anonymously when none did.
269 ///
270 /// # Errors
271 ///
272 /// See [`Self::connect_with_timeout`].
273 pub async fn connect_as(socket: &Path, identity: &DogIdentity) -> Result<Self, ConnectError> {
274 Self::connect_inner(socket, HANDSHAKE_TIMEOUT, identity.handshake()).await
275 }
276
277 async fn connect_inner(
278 socket: &Path,
279 timeout: Duration,
280 dog_name: Option<&str>,
281 ) -> Result<Self, ConnectError> {
282 let client = Client::connect_as(socket, timeout, dog_name).await?;
283 Ok(Self::supervised(client, socket, timeout, dog_name))
284 }
285
286 /// As [`Self::connect_as`], with the first generation over `link`, a
287 /// connection the shepherd opened for this process.
288 ///
289 /// Every later generation dials `socket`, so a lost link is not the end
290 /// of the connection.
291 ///
292 /// # Errors
293 ///
294 /// See [`Client::connect_over`].
295 pub(crate) async fn connect_over_link(
296 link: shep_core::transport::ClientStream,
297 socket: &Path,
298 timeout: Duration,
299 dog_name: Option<&str>,
300 ) -> Result<Self, ConnectError> {
301 let client = Client::connect_over(link, socket, timeout, dog_name).await?;
302 Ok(Self::supervised(client, socket, timeout, dog_name))
303 }
304
305 fn supervised(
306 client: Client,
307 socket: &Path,
308 timeout: Duration,
309 dog_name: Option<&str>,
310 ) -> Self {
311 let shared = Arc::new(Shared {
312 socket: socket.to_path_buf(),
313 handshake_timeout: timeout,
314 dog_name: dog_name.map(str::to_owned),
315 link: watch::Sender::new(LinkState::Connected),
316 state: RwLock::new(State {
317 client: Arc::new(client),
318 generation: 1,
319 }),
320 });
321 let supervisor = tokio::spawn(supervise(Arc::clone(&shared)));
322 Self { shared, supervisor }
323 }
324
325 /// The name this client announces itself as a dog under, or `None` for
326 /// a caller that is not one.
327 #[must_use]
328 pub fn dog_name(&self) -> Option<&str> {
329 self.shared.dog_name.as_deref()
330 }
331
332 /// The handshake acknowledgement of the daemon this client is talking
333 /// to right now.
334 ///
335 /// Owned rather than borrowed, unlike [`Client::daemon`]: a reconnect can
336 /// replace the generation this ack came from at any moment. A cached ack
337 /// would describe the predecessor.
338 #[must_use]
339 pub fn daemon(&self) -> HelloAck {
340 self.shared.read().client.daemon().clone()
341 }
342
343 /// The path this client connects (and reconnects) through.
344 #[must_use]
345 pub fn socket(&self) -> &Path {
346 &self.shared.socket
347 }
348
349 /// What the supervisor is doing right now.
350 #[must_use]
351 pub fn link(&self) -> LinkState {
352 self.shared.link.borrow().clone()
353 }
354
355 /// Sends `body` with [`DEFAULT_DEADLINE`](crate::DEFAULT_DEADLINE) on
356 /// the current generation of the connection.
357 ///
358 /// # Errors
359 ///
360 /// See [`Self::request_with_deadline`].
361 pub async fn request(&self, body: Request) -> Result<Response, RequestError> {
362 self.request_with_deadline(body, None).await
363 }
364
365 /// Sends `body` with `deadline` on the current generation.
366 ///
367 /// Never retried: a request on the wire when the daemon was replaced
368 /// fails, and so does one issued while reconnecting. Only the
369 /// connection re-establishes; the caller decides whether resending is safe.
370 ///
371 /// # Errors
372 ///
373 /// - [`RequestError::Rpc`]: the daemon answered with a structured error.
374 /// - [`RequestError::Timeout`]: no reply within the request's budget.
375 /// - [`RequestError::Closed`]: the connection closed before a reply arrived, or already had.
376 /// - [`RequestError::Wire`]: `body` failed to encode.
377 pub async fn request_with_deadline(
378 &self,
379 body: Request,
380 deadline: Option<Duration>,
381 ) -> Result<Response, RequestError> {
382 self.shared
383 .client()
384 .request_with_deadline(body, deadline)
385 .await
386 }
387
388 /// Subscribes the current generation of the connection to `topics`.
389 ///
390 /// The returned stream belongs to one generation and is not re-armed
391 /// across a reconnect: it ends when that connection dies, like
392 /// [`Client::subscribe`]'s would. A consumer that wants events past a
393 /// handover subscribes again; re-arming here would silently swallow
394 /// the gap between the old connection dying and a new `Subscribe`.
395 ///
396 /// # Errors
397 ///
398 /// Same as [`Self::request`].
399 pub async fn subscribe(&self, topics: Vec<String>) -> Result<EventStream, RequestError> {
400 self.shared.client().subscribe(topics).await
401 }
402
403 /// As [`Self::subscribe`], except that a `Subscribe` is sent only when
404 /// `covered` does not already hold every topic on this generation.
405 ///
406 /// The shepherd answers a `Subscribe` by replacing the connection's
407 /// forwarder, and the event the old one holds in flight is lost. A
408 /// caller that keeps `topics` a union of what it needs and passes the
409 /// same `covered` each time asks only when the union grows.
410 ///
411 /// # Errors
412 ///
413 /// Same as [`Self::request`]. `covered` is forgotten, so the next call
414 /// asks again.
415 pub(crate) async fn subscribe_covered(
416 &self,
417 topics: Vec<String>,
418 covered: &mut Covered,
419 ) -> Result<EventStream, RequestError> {
420 let (client, generation) = self.shared.current();
421 let held = covered.generation == generation
422 && topics.iter().all(|topic| covered.topics.contains(topic));
423 let sent = if held {
424 client.events().await
425 } else {
426 client.subscribe(topics.clone()).await
427 };
428 match sent {
429 Ok(events) if held => Ok(events),
430 Ok(events) => {
431 *covered = Covered {
432 generation,
433 topics: topics.into_iter().collect(),
434 };
435 Ok(events)
436 }
437 Err(error) => {
438 *covered = Covered::default();
439 Err(error)
440 }
441 }
442 }
443
444 /// Waits for the supervisor to be on a connection again, for at most
445 /// `budget`, and returns at once when it already is.
446 ///
447 /// What a caller needing a fresh [`EventStream`] after a handover waits
448 /// on. A `Subscribe` issued against a dead generation fails at once with
449 /// [`RequestError::Closed`]. That says nothing about a successor on its way.
450 /// A returned `Ok` is where to try, never a promise: the report can still
451 /// say [`LinkState::Connected`] in the moment after a socket dies.
452 ///
453 /// # Errors
454 ///
455 /// - [`LinkLost::Refused`]: a successor refused on version skew, and the supervisor stopped.
456 /// - [`LinkLost::Budget`]: `budget` ran out with the supervisor still dialling.
457 pub async fn connected_within(&self, budget: Duration) -> Result<(), LinkLost> {
458 let started = Instant::now();
459 let mut link = self.shared.link.subscribe();
460 loop {
461 // Cloned out of the guard, which must not be held across an
462 // await.
463 match link.borrow_and_update().clone() {
464 LinkState::Connected => return Ok(()),
465 LinkState::Refused {
466 daemon_version,
467 message,
468 } => {
469 return Err(LinkLost::Refused {
470 daemon_version,
471 message,
472 });
473 }
474 _ => {}
475 }
476 let left = budget.saturating_sub(started.elapsed());
477 if left.is_zero() {
478 return Err(LinkLost::Budget {
479 waited: started.elapsed(),
480 });
481 }
482 match tokio::time::timeout(left, link.changed()).await {
483 Ok(Ok(())) => {}
484 // Out of budget, or a sender that is gone so the state can
485 // never move again. The second is unreachable while `&self`
486 // holds that sender.
487 Ok(Err(_)) | Err(_) => {
488 return Err(LinkLost::Budget {
489 waited: started.elapsed(),
490 });
491 }
492 }
493 }
494 }
495
496 // A `watch` receiver keeps only the latest value. A successor that
497 // connects and dies before this task next runs is never seen as
498 // `Connected`. Its outage joins the one before it into a single budget.
499 // A shepherd flapping that fast is one a dog cannot work with.
500 /// Waits until the link has been down for a whole `budget` without
501 /// coming back, or has been refused.
502 ///
503 /// The clock runs only while the link is down, so a connection that
504 /// drops and returns inside `budget` leaves the next one a full budget.
505 /// Never resolves while the link is up, which is what lets it sit in a
506 /// `select!` arm beside a caller's own work.
507 pub async fn link_lost(&self, budget: Duration) -> LinkLost {
508 let mut link = self.shared.link.subscribe();
509 loop {
510 // Parking here is the only await on the path a connected client
511 // takes. That is what keeps this future yielding. It also stops a
512 // live link spending the budget: `connected_within` answers at
513 // once while the link is up, so the outer loop would spin.
514 loop {
515 let up = matches!(*link.borrow_and_update(), LinkState::Connected);
516 if !up {
517 break;
518 }
519 if link.changed().await.is_err() {
520 // The state can never move again, so the link can never
521 // be lost. Unreachable while `&self` holds the sender.
522 core::future::pending::<()>().await;
523 }
524 }
525 match self.connected_within(budget).await {
526 // Back inside the budget: this was a handover, not a loss.
527 Ok(()) => {}
528 Err(lost) => return lost,
529 }
530 }
531 }
532}
533
534#[cfg(test)]
535mod tests {
536
537 use std::time::Duration;
538
539 // tokio's Instant, not std's: it moves with `tokio::time::pause`, and the
540 // budget below is measured against a `tokio::time::sleep` that does too.
541 use crate::client::{Client, RequestError};
542 use crate::connection::HANDSHAKE_TIMEOUT;
543 use shep_core::protocol::Request;
544
545 use super::super::testing::*;
546 use super::*;
547 use crate::testing::{Handshake, control_address, fake_daemon_across_handovers};
548 use shep_core::protocol::{RpcError, RpcErrorCode};
549
550 /// fails if a dog's name reaches the FIRST daemon and not its
551 /// successor. The refusal is the one that matters and it is the second
552 /// one here: a dog that named itself at boot and then reconnected
553 /// anonymously would leave the successor unable to say which dog it
554 /// just refused, which is precisely the case G8 exists for. A daemon's
555 /// predecessor is not around to be asked.
556 #[tokio::test]
557 async fn a_dogs_name_rides_every_handshake_including_the_refused_one() {
558 let dir = tempfile::tempdir().unwrap();
559 let path = control_address(dir.path());
560 let shepherds = fake_daemon_across_handovers(
561 &path,
562 vec![
563 Handshake::Accept(ack_from(11)),
564 Handshake::Refuse(RpcError {
565 code: RpcErrorCode::ProtocolMismatch,
566 message: "daemon speaks protocol 3, client sent 2".to_string(),
567 daemon_version: Some("0.2.0".to_string()),
568 }),
569 ],
570 );
571 let client = ReconnectingClient::connect_as_dog(&path, "metrics")
572 .await
573 .unwrap();
574 assert_eq!(client.dog_name(), Some("metrics"));
575
576 shepherds.cut().await;
577 await_refusal(&client).await;
578
579 let named: Vec<Option<String>> = shepherds
580 .hellos()
581 .into_iter()
582 .map(|hello| hello.dog_name)
583 .collect();
584 assert_eq!(
585 named,
586 vec![Some("metrics".to_string()), Some("metrics".to_string())],
587 "every generation must be told which dog is talking to it"
588 );
589 }
590
591 #[tokio::test]
592 async fn an_identity_announces_its_handshake_name_or_none() {
593 let named = DogIdentity::named("rotate");
594 let anonymous = DogIdentity::from_env(&|_| None, "log-rotate");
595 for (identity, announced) in [(named, Some("rotate")), (anonymous, None)] {
596 let dir = tempfile::tempdir().unwrap();
597 let path = control_address(dir.path());
598 let hello = crate::testing::fake_daemon(&path, Ok(crate::testing::sample_ack())).await;
599
600 let client = ReconnectingClient::connect_as(&path, &identity)
601 .await
602 .unwrap();
603
604 let hello = tokio::time::timeout(Duration::from_secs(5), hello)
605 .await
606 .expect("the fake daemon read the hello")
607 .unwrap();
608 assert_eq!(hello.dog_name.as_deref(), announced);
609 assert_eq!(client.dog_name(), announced);
610 }
611 }
612
613 /// fails if a request that was in flight when the daemon was replaced
614 /// is re-sent to the successor: a re-issued `Stop` could stop a sheep
615 /// twice, and the client cannot tell a request the daemon never saw
616 /// from one it already acted on.
617 ///
618 /// The proof is positional: the successor's first envelope must be the
619 /// request issued after the reconnect, not the abandoned `Ping`.
620 #[tokio::test]
621 async fn an_in_flight_request_fails_and_is_never_re_sent_to_the_successor() {
622 let dir = tempfile::tempdir().unwrap();
623 let path = control_address(dir.path());
624 let shepherds = fake_daemon_across_handovers(
625 &path,
626 vec![
627 Handshake::Accept(ack_from(11)),
628 Handshake::Accept(ack_from(22)),
629 ],
630 );
631 let client = ReconnectingClient::connect(&path).await.unwrap();
632
633 // the predecessor reads this envelope and dies without answering it
634 shepherds.cut_on_next_request();
635 let lost = tokio::time::timeout(BOUND, client.request(Request::Ping))
636 .await
637 .expect("an abandoned request must fail, not hang");
638 assert_eq!(
639 lost,
640 Err(RequestError::Closed),
641 "an in-flight request must fail when the daemon is replaced"
642 );
643
644 await_reconnect(&client, &shepherds, 2).await;
645 let served = tokio::time::timeout(BOUND, client.request(Request::ListFlock))
646 .await
647 .expect("the request after a handover must not hang");
648 assert!(served.is_ok(), "after the handover: {served:?}");
649
650 let successor: Vec<Request> = shepherds
651 .envelopes()
652 .into_iter()
653 .filter(|(generation, _)| *generation == 2)
654 .map(|(_, envelope)| envelope.body)
655 .collect();
656 assert_eq!(
657 successor,
658 vec![Request::ListFlock],
659 "the successor must see only the request issued after the reconnect"
660 );
661 }
662
663 /// fails if dropping the handle leaves the supervisor reconnecting
664 /// forever against an address nobody will answer.
665 ///
666 /// Asserts a negative: the bound is the assertion, and the accept
667 /// count must stay flat within it.
668 #[tokio::test]
669 async fn dropping_the_handle_stops_the_supervisor() {
670 let dir = tempfile::tempdir().unwrap();
671 let path = control_address(dir.path());
672 let shepherds = fake_daemon_across_handovers(
673 &path,
674 vec![Handshake::Accept(ack_from(11)), Handshake::Drop],
675 );
676 let client = ReconnectingClient::connect(&path).await.unwrap();
677
678 shepherds.cut().await;
679 // Let the supervisor get as far as its first (dropped) reconnect,
680 // so it is provably still running at the moment the handle goes.
681 let reached = wait_until(BOUND, || shepherds.accepted() >= 2).await;
682 assert!(reached, "the supervisor never retried at all");
683
684 drop(client);
685
686 let grew = wait_until(NEGATIVE_WINDOW, || shepherds.accepted() >= 3).await;
687 assert!(
688 !grew,
689 "the supervisor kept reconnecting after its handle was dropped: {} accepts",
690 shepherds.accepted()
691 );
692 }
693
694 // The old connection is held until a new one has handshaken. A caller
695 // whose future loses a `select!` race still has the connection it
696 // started with. The fake serves one connection at a time. That holds the
697 // reconnect open long enough for the 1ms budget to cancel it.
698 #[tokio::test]
699 async fn a_cancelled_reconnect_leaves_the_handle_on_its_old_connection() {
700 let dir = tempfile::tempdir().unwrap();
701 let path = control_address(dir.path());
702 let shepherds = fake_daemon_across_handovers(
703 &path,
704 vec![
705 Handshake::Accept(ack_from(11)),
706 Handshake::Accept(ack_from(22)),
707 ],
708 );
709 let mut client = Client::connect(&path).await.unwrap();
710
711 let cancelled =
712 tokio::time::timeout(Duration::from_millis(1), client.reconnect_within(BOUND)).await;
713 assert!(
714 cancelled.is_err(),
715 "the reconnect must still have been in flight when it was cancelled"
716 );
717
718 assert_eq!(
719 client.daemon().pid,
720 11,
721 "a cancelled reconnect must not disturb the connection in hand"
722 );
723 let served = tokio::time::timeout(BOUND, client.request(Request::Ping))
724 .await
725 .expect("the old connection must still answer");
726 assert!(served.is_ok(), "after the cancelled reconnect: {served:?}");
727 assert_eq!(
728 shepherds.accepted(),
729 1,
730 "the fake serves one connection at a time, so the cancelled dial \
731 never got past the backlog"
732 );
733 }
734
735 /// fails if a caller waiting out a handover is told the link came back
736 /// when it did not, or is left waiting after it did.
737 #[tokio::test]
738 async fn a_wait_for_the_link_returns_once_the_successor_is_up() {
739 let dir = tempfile::tempdir().unwrap();
740 let path = control_address(dir.path());
741 let shepherds = fake_daemon_across_handovers(
742 &path,
743 vec![
744 Handshake::Accept(ack_from(11)),
745 Handshake::Accept(ack_from(11)),
746 ],
747 );
748 let client = ReconnectingClient::connect(&path).await.unwrap();
749
750 subscribe_then_lose_it(&client, &shepherds).await;
751 let waited = tokio::time::timeout(BOUND, client.connected_within(BOUND))
752 .await
753 .expect("the wait must not outlive the bound");
754
755 assert_eq!(waited, Ok(()), "a successor did come up");
756 assert_eq!(client.link(), LinkState::Connected);
757 }
758
759 // Pins the trap rather than the convenience. A caller looping on this
760 // cannot use it as the loop's bound: a live link answers `Ok` without
761 // consulting the budget. `ClientEvents::resubscribe` in the CLI checks
762 // the budget itself for exactly this reason.
763 #[tokio::test]
764 async fn a_wait_on_a_live_link_answers_at_once_even_with_no_budget_left() {
765 let dir = tempfile::tempdir().unwrap();
766 let path = control_address(dir.path());
767 let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
768 let client = ReconnectingClient::connect(&path).await.unwrap();
769
770 let answered = tokio::time::timeout(BOUND, client.connected_within(Duration::ZERO))
771 .await
772 .expect("a live link must answer without waiting");
773
774 assert_eq!(answered, Ok(()));
775 }
776
777 /// fails if a dog whose shepherd is gone for good waits forever, which
778 /// is the lingering that lets it attach to an unrelated shepherd later.
779 #[tokio::test]
780 async fn a_wait_for_a_shepherd_that_never_comes_spends_its_budget_and_gives_up() {
781 let dir = tempfile::tempdir().unwrap();
782 let path = control_address(dir.path());
783 let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
784 let client = ReconnectingClient::connect(&path).await.unwrap();
785
786 subscribe_then_lose_it(&client, &shepherds).await;
787 // Gone for good, listener and all: nothing answers this address
788 // again, which is what a stopped shepherd leaves behind.
789 drop(shepherds);
790
791 let budget = Duration::from_millis(200);
792 let started = tokio::time::Instant::now();
793 let lost = tokio::time::timeout(BOUND, client.connected_within(budget))
794 .await
795 .expect("a spent budget must return, not hang");
796 let elapsed = started.elapsed();
797
798 assert!(
799 matches!(lost, Err(LinkLost::Budget { .. })),
800 "expected a spent budget, got {lost:?}"
801 );
802 assert!(
803 elapsed >= budget,
804 "gave up after {elapsed:?}, short of the {budget:?} it was given"
805 );
806 }
807
808 /// fails if a bark dog that cannot speak the protocol waits out its
809 /// whole budget before exiting: the daemon that refused is the party
810 /// that can fix it, so there is nothing to wait for.
811 #[tokio::test]
812 async fn a_wait_ends_on_a_refusal_rather_than_serving_out_the_budget() {
813 let dir = tempfile::tempdir().unwrap();
814 let path = control_address(dir.path());
815 let shepherds = fake_daemon_across_handovers(
816 &path,
817 vec![
818 Handshake::Accept(ack_from(11)),
819 Handshake::Refuse(RpcError {
820 code: RpcErrorCode::ProtocolMismatch,
821 message: "daemon speaks protocol 3, client speaks 2".into(),
822 daemon_version: Some("0.9.9".into()),
823 }),
824 ],
825 );
826 let client = ReconnectingClient::connect(&path).await.unwrap();
827
828 subscribe_then_lose_it(&client, &shepherds).await;
829 // An hour, so serving the budget out could never look like passing.
830 let lost = tokio::time::timeout(BOUND, client.connected_within(Duration::from_secs(3600)))
831 .await
832 .expect("a refusal must end the wait long before its budget");
833
834 let Err(LinkLost::Refused {
835 daemon_version,
836 message,
837 }) = lost
838 else {
839 panic!("expected a refusal, got {lost:?}");
840 };
841 assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
842 assert!(message.contains("protocol 3"), "{message}");
843 }
844
845 /// fails if a dog watching for a lost shepherd fires while its shepherd
846 /// is right there, which would exit every dog on a healthy flock.
847 #[tokio::test]
848 async fn a_watch_for_a_lost_shepherd_stays_quiet_while_the_link_is_up() {
849 let dir = tempfile::tempdir().unwrap();
850 let path = control_address(dir.path());
851 let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
852 let client = ReconnectingClient::connect(&path).await.unwrap();
853
854 let fired =
855 tokio::time::timeout(NEGATIVE_WINDOW, client.link_lost(Duration::from_millis(1))).await;
856
857 assert!(
858 fired.is_err(),
859 "a connected client reported its shepherd lost: {fired:?}"
860 );
861 }
862
863 /// fails if a handover looks like a lost shepherd, which is the whole
864 /// distinction: the shepherd execs a successor on purpose, and every
865 /// dog is meant to cross that without exiting.
866 #[tokio::test]
867 async fn a_watch_for_a_lost_shepherd_rides_out_a_handover() {
868 let dir = tempfile::tempdir().unwrap();
869 let path = control_address(dir.path());
870 let shepherds = fake_daemon_across_handovers(
871 &path,
872 vec![
873 Handshake::Accept(ack_from(11)),
874 Handshake::Accept(ack_from(11)),
875 ],
876 );
877 let client = ReconnectingClient::connect(&path).await.unwrap();
878
879 shepherds.cut().await;
880 // Far longer than the reconnect ladder's first rung, so a successor
881 // that is coming has room to arrive.
882 let fired = tokio::time::timeout(NEGATIVE_WINDOW, client.link_lost(BOUND)).await;
883
884 assert!(
885 fired.is_err(),
886 "a handover was reported as a lost shepherd: {fired:?}"
887 );
888 assert_eq!(client.link(), LinkState::Connected);
889 }
890
891 /// fails if a dog a shepherd refuses sits out its whole budget before
892 /// noticing. `link_lost` reaches a refusal only by delegating to
893 /// `connected_within`, so the early exit is one call away from being
894 /// lost in a refactor, and nothing else here would catch it.
895 #[tokio::test]
896 async fn a_watch_for_a_lost_shepherd_ends_on_a_refusal_rather_than_a_budget() {
897 let dir = tempfile::tempdir().unwrap();
898 let path = control_address(dir.path());
899 let shepherds = fake_daemon_across_handovers(
900 &path,
901 vec![
902 Handshake::Accept(ack_from(11)),
903 Handshake::Refuse(RpcError {
904 code: RpcErrorCode::ProtocolMismatch,
905 message: "daemon speaks protocol 3, client speaks 2".into(),
906 daemon_version: Some("0.9.9".into()),
907 }),
908 ],
909 );
910 let client = ReconnectingClient::connect(&path).await.unwrap();
911
912 subscribe_then_lose_it(&client, &shepherds).await;
913 // An hour, so serving the budget out could never look like passing.
914 let lost = tokio::time::timeout(BOUND, client.link_lost(Duration::from_secs(3600)))
915 .await
916 .expect("a refusal must end the watch long before its budget");
917
918 let LinkLost::Refused {
919 daemon_version,
920 message,
921 } = lost
922 else {
923 panic!("expected a refusal, got {lost:?}");
924 };
925 assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
926 assert!(message.contains("protocol 3"), "{message}");
927 }
928
929 /// fails if a dog whose shepherd stopped keeps waiting: the metrics dog
930 /// has no stream to end, so this watch is the only thing that tells it.
931 #[tokio::test]
932 async fn a_watch_for_a_lost_shepherd_fires_once_the_budget_is_spent() {
933 let dir = tempfile::tempdir().unwrap();
934 let path = control_address(dir.path());
935 let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
936 let client = ReconnectingClient::connect(&path).await.unwrap();
937
938 drop(shepherds);
939
940 let budget = Duration::from_millis(200);
941 let lost = tokio::time::timeout(BOUND, client.link_lost(budget))
942 .await
943 .expect("a spent budget must fire, not hang");
944
945 assert!(
946 matches!(lost, LinkLost::Budget { .. }),
947 "expected a spent budget, got {lost:?}"
948 );
949 }
950
951 /// fails if `Client`'s `Debug` starts printing its command channel, or
952 /// stops naming the dog it announces itself as. A derived impl would do
953 /// both, and the channel says nothing a reader can act on.
954 #[tokio::test]
955 async fn client_debug_names_the_socket_the_host_the_ack_and_the_dog() {
956 let dir = tempfile::tempdir().unwrap();
957 let path = control_address(dir.path());
958 let ack = ack_from(11);
959 let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack.clone())]);
960 let client = Client::connect_as(&path, HANDSHAKE_TIMEOUT, Some("metrics"))
961 .await
962 .unwrap();
963
964 assert_eq!(
965 format!("{client:?}"),
966 format!(
967 "Client {{ socket: {path:?}, host: None, ack: {ack:?}, dog_name: Some(\"metrics\"), .. }}"
968 )
969 );
970 }
971}