Skip to main content

shep_client/reconnect/
reconnecting.rs

1use super::*;
2use core::fmt;
3use std::collections::BTreeSet;
4use std::path::{Path, PathBuf};
5use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
6use std::time::Duration;
7use tokio::sync::watch;
8use tokio::task::JoinHandle;
9// tokio's Instant, not std's: it moves with `tokio::time::pause`, and the
10// budget below is measured against a `tokio::time::sleep` that does too.
11use crate::client::{Client, RequestError};
12use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
13use crate::dogs::DogIdentity;
14use crate::events::EventStream;
15use shep_core::protocol::{HelloAck, Request, Response};
16use tokio::time::Instant;
17
18/// A [`Client`] that re-establishes its own connection when the daemon it
19/// was talking to is replaced.
20///
21/// Built for dogs, which outlive the shepherd they connected to: a dog's
22/// process crosses a daemon handover as an ordinary child, so it is still
23/// running when its socket dies. The CLI uses a bare [`Client`] instead.
24///
25/// # Example
26///
27/// ```no_run
28/// use shep_client::{ReconnectingClient, shep_core::protocol::Request};
29///
30/// # async fn dog(socket: &std::path::Path) -> Result<(), Box<dyn core::error::Error>> {
31/// // `name` is what the daemon put in `$SHEP_DOG_NAME` when it spawned
32/// // this process; it lets the daemon act on a refusal (G8).
33/// let client = ReconnectingClient::connect_as_dog(socket, "metrics").await?;
34/// // Survives the daemon being replaced underneath it; a request that was
35/// // in flight at the moment it happened still fails rather than retrying.
36/// let _flock = client.request(Request::ListFlock).await?;
37/// # Ok(())
38/// # }
39/// # let _ = dog;
40/// ```
41pub struct ReconnectingClient {
42    shared: Arc<Shared>,
43    supervisor: JoinHandle<()>,
44}
45
46/// Manual, not derived: the socket path and the [`HelloAck`] carry no
47/// secret, and neither does [`LinkState`], whose one payload is the
48/// daemon's own refusal sentence. Manual because [`RwLock`] and
49/// [`JoinHandle`] derive into noise, and a derived impl would print the
50/// inner [`Client`] where the link state is the useful fact.
51impl fmt::Debug for ReconnectingClient {
52    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
53        f.debug_struct("ReconnectingClient")
54            .field("socket", &self.shared.socket)
55            .field("dog_name", &self.shared.dog_name)
56            .field("link", &self.link())
57            .field("ack", &self.daemon())
58            .finish_non_exhaustive()
59    }
60}
61
62/// Everything the handle and its supervisor both reach: the address to
63/// reconnect to, the budget to do it under, and the current generation.
64pub(super) struct Shared {
65    socket: PathBuf,
66    handshake_timeout: Duration,
67    /// The name this client announces itself as a dog under, re-sent on
68    /// every reconnect. See [`ReconnectingClient::connect_as_dog`].
69    dog_name: Option<String>,
70    state: RwLock<State>,
71    /// What the supervisor last reported, and how a waiter hears it
72    /// change. The only home of the link state: a second copy beside the
73    /// generation would be one more thing to keep in step, and the two
74    /// have no reader that needs them to move together.
75    link: watch::Sender<LinkState>,
76}
77
78/// The generation of the connection in force right now.
79pub(super) struct State {
80    client: Arc<Client>,
81    /// Counts the connections made, from 1: what a daemon-side topic filter
82    /// belongs to, since a successor starts with none.
83    generation: u64,
84}
85
86/// The topics a connection's daemon-side filter is known to hold.
87///
88/// `Default` knows nothing, so the next subscription is sent.
89#[derive(Debug, Default)]
90pub(crate) struct Covered {
91    generation: u64,
92    topics: BTreeSet<String>,
93}
94
95/// Stops the supervisor when the handle goes away.
96///
97/// Without this, a dropped `ReconnectingClient` whose daemon is gone leaves
98/// a task reconnecting forever, holding the last generation's socket open
99/// with it. `abort` rather than a cooperative signal: the supervisor is
100/// parked on a connect attempt that would not see a flag.
101impl Drop for ReconnectingClient {
102    fn drop(&mut self) {
103        self.supervisor.abort();
104    }
105}
106
107/// The supervisor loop: wait for the current generation to die, then
108/// re-establish it, forever.
109///
110/// Ends only on a protocol refusal (see [`LinkState::Refused`]) or when the
111/// handle is dropped and this task is aborted.
112pub(super) async fn supervise(shared: Arc<Shared>) {
113    loop {
114        // held only for the death await, dropped before reconnecting so the
115        // dead socket goes away as soon as `install` replaces it
116        let generation = shared.client();
117        generation.closed().await;
118        drop(generation);
119        shared.set_link(LinkState::Reconnecting);
120
121        let mut delay = RECONNECT_MIN_DELAY;
122        loop {
123            match Client::connect_as(
124                &shared.socket,
125                shared.handshake_timeout,
126                shared.dog_name.as_deref(),
127            )
128            .await
129            {
130                Ok(fresh) => {
131                    shared.install(fresh);
132                    break;
133                }
134                Err(ConnectError::ProtocolMismatch {
135                    daemon_version,
136                    message,
137                    ..
138                }) => {
139                    shared.set_link(LinkState::Refused {
140                        daemon_version,
141                        message,
142                    });
143                    return;
144                }
145                // Everything else is a daemon that is not ready yet: the
146                // successor has not started accepting, or the socket is
147                // momentarily unbound. Those resolve on their own, so the
148                // only question is how often to ask.
149                Err(_transient) => {
150                    tokio::time::sleep(delay).await;
151                    delay = next_delay(delay);
152                }
153            }
154        }
155    }
156}
157
158impl Shared {
159    /// A read guard, treating a poisoned lock as ordinary data.
160    ///
161    /// Nothing inside a critical section here can panic: every one is a
162    /// clone or an assignment of a plain struct, so poisoning cannot
163    /// signal a torn value.
164    fn read(&self) -> RwLockReadGuard<'_, State> {
165        self.state.read().unwrap_or_else(PoisonError::into_inner)
166    }
167
168    /// A write guard. See [`Self::read`] for the poisoning argument.
169    fn write(&self) -> RwLockWriteGuard<'_, State> {
170        self.state.write().unwrap_or_else(PoisonError::into_inner)
171    }
172
173    /// The current generation, cloned out so the guard is dropped before
174    /// any caller awaits on it: no lock is ever held across an `await`.
175    fn client(&self) -> Arc<Client> {
176        Arc::clone(&self.read().client)
177    }
178
179    /// The current generation and its number, read together.
180    fn current(&self) -> (Arc<Client>, u64) {
181        let state = self.read();
182        (Arc::clone(&state.client), state.generation)
183    }
184
185    fn set_link(&self, link: LinkState) {
186        self.link.send_replace(link);
187    }
188
189    /// Swaps in a freshly handshaken generation, dropping the dead one.
190    ///
191    /// The generation lands before the link is announced, so a waiter woken
192    /// by [`LinkState::Connected`] finds the connection it was told about.
193    fn install(&self, client: Client) {
194        let mut state = self.write();
195        state.client = Arc::new(client);
196        state.generation += 1;
197        drop(state);
198        self.set_link(LinkState::Connected);
199    }
200}
201
202impl ReconnectingClient {
203    /// Connects to `socket`, performs the version handshake bounded by
204    /// [`HANDSHAKE_TIMEOUT`], and starts the supervisor that will
205    /// re-establish this connection whenever it dies.
206    ///
207    /// The FIRST connection is not supervised: a socket nobody is listening
208    /// on is a caller's error, not a handover, so this reports it rather
209    /// than retrying behind the caller's back.
210    ///
211    /// # Errors
212    ///
213    /// See [`Self::connect_with_timeout`].
214    pub async fn connect(socket: &Path) -> Result<Self, ConnectError> {
215        Self::connect_with_timeout(socket, HANDSHAKE_TIMEOUT).await
216    }
217
218    /// As [`Self::connect`], but with a caller-supplied handshake timeout,
219    /// used for the first connection and for every reconnect after it.
220    ///
221    /// # Errors
222    ///
223    /// - [`ConnectError::Connect`]: the initial `connect(2)` call failed.
224    /// - [`ConnectError::Wire`]: `Hello` failed to encode, or the reply failed to decode.
225    /// - [`ConnectError::Io`]: a framed read or write failed after connect.
226    /// - [`ConnectError::HandshakeClosed`]: the peer closed before a `HelloReply`.
227    /// - [`ConnectError::HandshakeTimeout`]: no `HelloReply` arrived within `timeout`.
228    /// - [`ConnectError::ProtocolMismatch`]: the daemon refused on protocol-version skew.
229    pub async fn connect_with_timeout(
230        socket: &Path,
231        timeout: Duration,
232    ) -> Result<Self, ConnectError> {
233        Self::connect_inner(socket, timeout, None).await
234    }
235
236    /// As [`Self::connect`], but announcing this client as the dog
237    /// registered under `name`, the value the daemon put in
238    /// `$SHEP_DOG_NAME` when it spawned this process.
239    ///
240    /// A dog should use this rather than [`Self::connect`]: the name makes
241    /// a refused handshake actionable, since a refusal never reaches a
242    /// request. Without it, the daemon still stops rather than spins on a
243    /// refusal, but cannot say which dog went stale or restart it.
244    ///
245    /// # Errors
246    ///
247    /// See [`Self::connect_with_timeout`].
248    pub async fn connect_as_dog(socket: &Path, name: &str) -> Result<Self, ConnectError> {
249        Self::connect_as_dog_with_timeout(socket, HANDSHAKE_TIMEOUT, name).await
250    }
251
252    /// As [`Self::connect_as_dog`], but with a caller-supplied handshake
253    /// timeout, used for the first connection and for every reconnect after
254    /// it.
255    ///
256    /// # Errors
257    ///
258    /// See [`Self::connect_with_timeout`].
259    pub async fn connect_as_dog_with_timeout(
260        socket: &Path,
261        timeout: Duration,
262        name: &str,
263    ) -> Result<Self, ConnectError> {
264        Self::connect_inner(socket, timeout, Some(name)).await
265    }
266
267    /// Connects as `identity` says: as its dog when a shepherd named this
268    /// process, and anonymously when none did.
269    ///
270    /// # Errors
271    ///
272    /// See [`Self::connect_with_timeout`].
273    pub async fn connect_as(socket: &Path, identity: &DogIdentity) -> Result<Self, ConnectError> {
274        Self::connect_inner(socket, HANDSHAKE_TIMEOUT, identity.handshake()).await
275    }
276
277    async fn connect_inner(
278        socket: &Path,
279        timeout: Duration,
280        dog_name: Option<&str>,
281    ) -> Result<Self, ConnectError> {
282        let client = Client::connect_as(socket, timeout, dog_name).await?;
283        Ok(Self::supervised(client, socket, timeout, dog_name))
284    }
285
286    /// As [`Self::connect_as`], with the first generation over `link`, a
287    /// connection the shepherd opened for this process.
288    ///
289    /// Every later generation dials `socket`, so a lost link is not the end
290    /// of the connection.
291    ///
292    /// # Errors
293    ///
294    /// See [`Client::connect_over`].
295    pub(crate) async fn connect_over_link(
296        link: shep_core::transport::ClientStream,
297        socket: &Path,
298        timeout: Duration,
299        dog_name: Option<&str>,
300    ) -> Result<Self, ConnectError> {
301        let client = Client::connect_over(link, socket, timeout, dog_name).await?;
302        Ok(Self::supervised(client, socket, timeout, dog_name))
303    }
304
305    fn supervised(
306        client: Client,
307        socket: &Path,
308        timeout: Duration,
309        dog_name: Option<&str>,
310    ) -> Self {
311        let shared = Arc::new(Shared {
312            socket: socket.to_path_buf(),
313            handshake_timeout: timeout,
314            dog_name: dog_name.map(str::to_owned),
315            link: watch::Sender::new(LinkState::Connected),
316            state: RwLock::new(State {
317                client: Arc::new(client),
318                generation: 1,
319            }),
320        });
321        let supervisor = tokio::spawn(supervise(Arc::clone(&shared)));
322        Self { shared, supervisor }
323    }
324
325    /// The name this client announces itself as a dog under, or `None` for
326    /// a caller that is not one.
327    #[must_use]
328    pub fn dog_name(&self) -> Option<&str> {
329        self.shared.dog_name.as_deref()
330    }
331
332    /// The handshake acknowledgement of the daemon this client is talking
333    /// to right now.
334    ///
335    /// Owned rather than borrowed, unlike [`Client::daemon`]: a reconnect can
336    /// replace the generation this ack came from at any moment. A cached ack
337    /// would describe the predecessor.
338    #[must_use]
339    pub fn daemon(&self) -> HelloAck {
340        self.shared.read().client.daemon().clone()
341    }
342
343    /// The path this client connects (and reconnects) through.
344    #[must_use]
345    pub fn socket(&self) -> &Path {
346        &self.shared.socket
347    }
348
349    /// What the supervisor is doing right now.
350    #[must_use]
351    pub fn link(&self) -> LinkState {
352        self.shared.link.borrow().clone()
353    }
354
355    /// Sends `body` with [`DEFAULT_DEADLINE`](crate::DEFAULT_DEADLINE) on
356    /// the current generation of the connection.
357    ///
358    /// # Errors
359    ///
360    /// See [`Self::request_with_deadline`].
361    pub async fn request(&self, body: Request) -> Result<Response, RequestError> {
362        self.request_with_deadline(body, None).await
363    }
364
365    /// Sends `body` with `deadline` on the current generation.
366    ///
367    /// Never retried: a request on the wire when the daemon was replaced
368    /// fails, and so does one issued while reconnecting. Only the
369    /// connection re-establishes; the caller decides whether resending is safe.
370    ///
371    /// # Errors
372    ///
373    /// - [`RequestError::Rpc`]: the daemon answered with a structured error.
374    /// - [`RequestError::Timeout`]: no reply within the request's budget.
375    /// - [`RequestError::Closed`]: the connection closed before a reply arrived, or already had.
376    /// - [`RequestError::Wire`]: `body` failed to encode.
377    pub async fn request_with_deadline(
378        &self,
379        body: Request,
380        deadline: Option<Duration>,
381    ) -> Result<Response, RequestError> {
382        self.shared
383            .client()
384            .request_with_deadline(body, deadline)
385            .await
386    }
387
388    /// Subscribes the current generation of the connection to `topics`.
389    ///
390    /// The returned stream belongs to one generation and is not re-armed
391    /// across a reconnect: it ends when that connection dies, like
392    /// [`Client::subscribe`]'s would. A consumer that wants events past a
393    /// handover subscribes again; re-arming here would silently swallow
394    /// the gap between the old connection dying and a new `Subscribe`.
395    ///
396    /// # Errors
397    ///
398    /// Same as [`Self::request`].
399    pub async fn subscribe(&self, topics: Vec<String>) -> Result<EventStream, RequestError> {
400        self.shared.client().subscribe(topics).await
401    }
402
403    /// As [`Self::subscribe`], except that a `Subscribe` is sent only when
404    /// `covered` does not already hold every topic on this generation.
405    ///
406    /// The shepherd answers a `Subscribe` by replacing the connection's
407    /// forwarder, and the event the old one holds in flight is lost. A
408    /// caller that keeps `topics` a union of what it needs and passes the
409    /// same `covered` each time asks only when the union grows.
410    ///
411    /// # Errors
412    ///
413    /// Same as [`Self::request`]. `covered` is forgotten, so the next call
414    /// asks again.
415    pub(crate) async fn subscribe_covered(
416        &self,
417        topics: Vec<String>,
418        covered: &mut Covered,
419    ) -> Result<EventStream, RequestError> {
420        let (client, generation) = self.shared.current();
421        let held = covered.generation == generation
422            && topics.iter().all(|topic| covered.topics.contains(topic));
423        let sent = if held {
424            client.events().await
425        } else {
426            client.subscribe(topics.clone()).await
427        };
428        match sent {
429            Ok(events) if held => Ok(events),
430            Ok(events) => {
431                *covered = Covered {
432                    generation,
433                    topics: topics.into_iter().collect(),
434                };
435                Ok(events)
436            }
437            Err(error) => {
438                *covered = Covered::default();
439                Err(error)
440            }
441        }
442    }
443
444    /// Waits for the supervisor to be on a connection again, for at most
445    /// `budget`, and returns at once when it already is.
446    ///
447    /// What a caller needing a fresh [`EventStream`] after a handover waits
448    /// on. A `Subscribe` issued against a dead generation fails at once with
449    /// [`RequestError::Closed`]. That says nothing about a successor on its way.
450    /// A returned `Ok` is where to try, never a promise: the report can still
451    /// say [`LinkState::Connected`] in the moment after a socket dies.
452    ///
453    /// # Errors
454    ///
455    /// - [`LinkLost::Refused`]: a successor refused on version skew, and the supervisor stopped.
456    /// - [`LinkLost::Budget`]: `budget` ran out with the supervisor still dialling.
457    pub async fn connected_within(&self, budget: Duration) -> Result<(), LinkLost> {
458        let started = Instant::now();
459        let mut link = self.shared.link.subscribe();
460        loop {
461            // Cloned out of the guard, which must not be held across an
462            // await.
463            match link.borrow_and_update().clone() {
464                LinkState::Connected => return Ok(()),
465                LinkState::Refused {
466                    daemon_version,
467                    message,
468                } => {
469                    return Err(LinkLost::Refused {
470                        daemon_version,
471                        message,
472                    });
473                }
474                _ => {}
475            }
476            let left = budget.saturating_sub(started.elapsed());
477            if left.is_zero() {
478                return Err(LinkLost::Budget {
479                    waited: started.elapsed(),
480                });
481            }
482            match tokio::time::timeout(left, link.changed()).await {
483                Ok(Ok(())) => {}
484                // Out of budget, or a sender that is gone so the state can
485                // never move again. The second is unreachable while `&self`
486                // holds that sender.
487                Ok(Err(_)) | Err(_) => {
488                    return Err(LinkLost::Budget {
489                        waited: started.elapsed(),
490                    });
491                }
492            }
493        }
494    }
495
496    // A `watch` receiver keeps only the latest value. A successor that
497    // connects and dies before this task next runs is never seen as
498    // `Connected`. Its outage joins the one before it into a single budget.
499    // A shepherd flapping that fast is one a dog cannot work with.
500    /// Waits until the link has been down for a whole `budget` without
501    /// coming back, or has been refused.
502    ///
503    /// The clock runs only while the link is down, so a connection that
504    /// drops and returns inside `budget` leaves the next one a full budget.
505    /// Never resolves while the link is up, which is what lets it sit in a
506    /// `select!` arm beside a caller's own work.
507    pub async fn link_lost(&self, budget: Duration) -> LinkLost {
508        let mut link = self.shared.link.subscribe();
509        loop {
510            // Parking here is the only await on the path a connected client
511            // takes. That is what keeps this future yielding. It also stops a
512            // live link spending the budget: `connected_within` answers at
513            // once while the link is up, so the outer loop would spin.
514            loop {
515                let up = matches!(*link.borrow_and_update(), LinkState::Connected);
516                if !up {
517                    break;
518                }
519                if link.changed().await.is_err() {
520                    // The state can never move again, so the link can never
521                    // be lost. Unreachable while `&self` holds the sender.
522                    core::future::pending::<()>().await;
523                }
524            }
525            match self.connected_within(budget).await {
526                // Back inside the budget: this was a handover, not a loss.
527                Ok(()) => {}
528                Err(lost) => return lost,
529            }
530        }
531    }
532}
533
534#[cfg(test)]
535mod tests {
536
537    use std::time::Duration;
538
539    // tokio's Instant, not std's: it moves with `tokio::time::pause`, and the
540    // budget below is measured against a `tokio::time::sleep` that does too.
541    use crate::client::{Client, RequestError};
542    use crate::connection::HANDSHAKE_TIMEOUT;
543    use shep_core::protocol::Request;
544
545    use super::super::testing::*;
546    use super::*;
547    use crate::testing::{Handshake, control_address, fake_daemon_across_handovers};
548    use shep_core::protocol::{RpcError, RpcErrorCode};
549
550    /// fails if a dog's name reaches the FIRST daemon and not its
551    /// successor. The refusal is the one that matters and it is the second
552    /// one here: a dog that named itself at boot and then reconnected
553    /// anonymously would leave the successor unable to say which dog it
554    /// just refused, which is precisely the case G8 exists for. A daemon's
555    /// predecessor is not around to be asked.
556    #[tokio::test]
557    async fn a_dogs_name_rides_every_handshake_including_the_refused_one() {
558        let dir = tempfile::tempdir().unwrap();
559        let path = control_address(dir.path());
560        let shepherds = fake_daemon_across_handovers(
561            &path,
562            vec![
563                Handshake::Accept(ack_from(11)),
564                Handshake::Refuse(RpcError {
565                    code: RpcErrorCode::ProtocolMismatch,
566                    message: "daemon speaks protocol 3, client sent 2".to_string(),
567                    daemon_version: Some("0.2.0".to_string()),
568                }),
569            ],
570        );
571        let client = ReconnectingClient::connect_as_dog(&path, "metrics")
572            .await
573            .unwrap();
574        assert_eq!(client.dog_name(), Some("metrics"));
575
576        shepherds.cut().await;
577        await_refusal(&client).await;
578
579        let named: Vec<Option<String>> = shepherds
580            .hellos()
581            .into_iter()
582            .map(|hello| hello.dog_name)
583            .collect();
584        assert_eq!(
585            named,
586            vec![Some("metrics".to_string()), Some("metrics".to_string())],
587            "every generation must be told which dog is talking to it"
588        );
589    }
590
591    #[tokio::test]
592    async fn an_identity_announces_its_handshake_name_or_none() {
593        let named = DogIdentity::named("rotate");
594        let anonymous = DogIdentity::from_env(&|_| None, "log-rotate");
595        for (identity, announced) in [(named, Some("rotate")), (anonymous, None)] {
596            let dir = tempfile::tempdir().unwrap();
597            let path = control_address(dir.path());
598            let hello = crate::testing::fake_daemon(&path, Ok(crate::testing::sample_ack())).await;
599
600            let client = ReconnectingClient::connect_as(&path, &identity)
601                .await
602                .unwrap();
603
604            let hello = tokio::time::timeout(Duration::from_secs(5), hello)
605                .await
606                .expect("the fake daemon read the hello")
607                .unwrap();
608            assert_eq!(hello.dog_name.as_deref(), announced);
609            assert_eq!(client.dog_name(), announced);
610        }
611    }
612
613    /// fails if a request that was in flight when the daemon was replaced
614    /// is re-sent to the successor: a re-issued `Stop` could stop a sheep
615    /// twice, and the client cannot tell a request the daemon never saw
616    /// from one it already acted on.
617    ///
618    /// The proof is positional: the successor's first envelope must be the
619    /// request issued after the reconnect, not the abandoned `Ping`.
620    #[tokio::test]
621    async fn an_in_flight_request_fails_and_is_never_re_sent_to_the_successor() {
622        let dir = tempfile::tempdir().unwrap();
623        let path = control_address(dir.path());
624        let shepherds = fake_daemon_across_handovers(
625            &path,
626            vec![
627                Handshake::Accept(ack_from(11)),
628                Handshake::Accept(ack_from(22)),
629            ],
630        );
631        let client = ReconnectingClient::connect(&path).await.unwrap();
632
633        // the predecessor reads this envelope and dies without answering it
634        shepherds.cut_on_next_request();
635        let lost = tokio::time::timeout(BOUND, client.request(Request::Ping))
636            .await
637            .expect("an abandoned request must fail, not hang");
638        assert_eq!(
639            lost,
640            Err(RequestError::Closed),
641            "an in-flight request must fail when the daemon is replaced"
642        );
643
644        await_reconnect(&client, &shepherds, 2).await;
645        let served = tokio::time::timeout(BOUND, client.request(Request::ListFlock))
646            .await
647            .expect("the request after a handover must not hang");
648        assert!(served.is_ok(), "after the handover: {served:?}");
649
650        let successor: Vec<Request> = shepherds
651            .envelopes()
652            .into_iter()
653            .filter(|(generation, _)| *generation == 2)
654            .map(|(_, envelope)| envelope.body)
655            .collect();
656        assert_eq!(
657            successor,
658            vec![Request::ListFlock],
659            "the successor must see only the request issued after the reconnect"
660        );
661    }
662
663    /// fails if dropping the handle leaves the supervisor reconnecting
664    /// forever against an address nobody will answer.
665    ///
666    /// Asserts a negative: the bound is the assertion, and the accept
667    /// count must stay flat within it.
668    #[tokio::test]
669    async fn dropping_the_handle_stops_the_supervisor() {
670        let dir = tempfile::tempdir().unwrap();
671        let path = control_address(dir.path());
672        let shepherds = fake_daemon_across_handovers(
673            &path,
674            vec![Handshake::Accept(ack_from(11)), Handshake::Drop],
675        );
676        let client = ReconnectingClient::connect(&path).await.unwrap();
677
678        shepherds.cut().await;
679        // Let the supervisor get as far as its first (dropped) reconnect,
680        // so it is provably still running at the moment the handle goes.
681        let reached = wait_until(BOUND, || shepherds.accepted() >= 2).await;
682        assert!(reached, "the supervisor never retried at all");
683
684        drop(client);
685
686        let grew = wait_until(NEGATIVE_WINDOW, || shepherds.accepted() >= 3).await;
687        assert!(
688            !grew,
689            "the supervisor kept reconnecting after its handle was dropped: {} accepts",
690            shepherds.accepted()
691        );
692    }
693
694    // The old connection is held until a new one has handshaken. A caller
695    // whose future loses a `select!` race still has the connection it
696    // started with. The fake serves one connection at a time. That holds the
697    // reconnect open long enough for the 1ms budget to cancel it.
698    #[tokio::test]
699    async fn a_cancelled_reconnect_leaves_the_handle_on_its_old_connection() {
700        let dir = tempfile::tempdir().unwrap();
701        let path = control_address(dir.path());
702        let shepherds = fake_daemon_across_handovers(
703            &path,
704            vec![
705                Handshake::Accept(ack_from(11)),
706                Handshake::Accept(ack_from(22)),
707            ],
708        );
709        let mut client = Client::connect(&path).await.unwrap();
710
711        let cancelled =
712            tokio::time::timeout(Duration::from_millis(1), client.reconnect_within(BOUND)).await;
713        assert!(
714            cancelled.is_err(),
715            "the reconnect must still have been in flight when it was cancelled"
716        );
717
718        assert_eq!(
719            client.daemon().pid,
720            11,
721            "a cancelled reconnect must not disturb the connection in hand"
722        );
723        let served = tokio::time::timeout(BOUND, client.request(Request::Ping))
724            .await
725            .expect("the old connection must still answer");
726        assert!(served.is_ok(), "after the cancelled reconnect: {served:?}");
727        assert_eq!(
728            shepherds.accepted(),
729            1,
730            "the fake serves one connection at a time, so the cancelled dial \
731                 never got past the backlog"
732        );
733    }
734
735    /// fails if a caller waiting out a handover is told the link came back
736    /// when it did not, or is left waiting after it did.
737    #[tokio::test]
738    async fn a_wait_for_the_link_returns_once_the_successor_is_up() {
739        let dir = tempfile::tempdir().unwrap();
740        let path = control_address(dir.path());
741        let shepherds = fake_daemon_across_handovers(
742            &path,
743            vec![
744                Handshake::Accept(ack_from(11)),
745                Handshake::Accept(ack_from(11)),
746            ],
747        );
748        let client = ReconnectingClient::connect(&path).await.unwrap();
749
750        subscribe_then_lose_it(&client, &shepherds).await;
751        let waited = tokio::time::timeout(BOUND, client.connected_within(BOUND))
752            .await
753            .expect("the wait must not outlive the bound");
754
755        assert_eq!(waited, Ok(()), "a successor did come up");
756        assert_eq!(client.link(), LinkState::Connected);
757    }
758
759    // Pins the trap rather than the convenience. A caller looping on this
760    // cannot use it as the loop's bound: a live link answers `Ok` without
761    // consulting the budget. `ClientEvents::resubscribe` in the CLI checks
762    // the budget itself for exactly this reason.
763    #[tokio::test]
764    async fn a_wait_on_a_live_link_answers_at_once_even_with_no_budget_left() {
765        let dir = tempfile::tempdir().unwrap();
766        let path = control_address(dir.path());
767        let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
768        let client = ReconnectingClient::connect(&path).await.unwrap();
769
770        let answered = tokio::time::timeout(BOUND, client.connected_within(Duration::ZERO))
771            .await
772            .expect("a live link must answer without waiting");
773
774        assert_eq!(answered, Ok(()));
775    }
776
777    /// fails if a dog whose shepherd is gone for good waits forever, which
778    /// is the lingering that lets it attach to an unrelated shepherd later.
779    #[tokio::test]
780    async fn a_wait_for_a_shepherd_that_never_comes_spends_its_budget_and_gives_up() {
781        let dir = tempfile::tempdir().unwrap();
782        let path = control_address(dir.path());
783        let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
784        let client = ReconnectingClient::connect(&path).await.unwrap();
785
786        subscribe_then_lose_it(&client, &shepherds).await;
787        // Gone for good, listener and all: nothing answers this address
788        // again, which is what a stopped shepherd leaves behind.
789        drop(shepherds);
790
791        let budget = Duration::from_millis(200);
792        let started = tokio::time::Instant::now();
793        let lost = tokio::time::timeout(BOUND, client.connected_within(budget))
794            .await
795            .expect("a spent budget must return, not hang");
796        let elapsed = started.elapsed();
797
798        assert!(
799            matches!(lost, Err(LinkLost::Budget { .. })),
800            "expected a spent budget, got {lost:?}"
801        );
802        assert!(
803            elapsed >= budget,
804            "gave up after {elapsed:?}, short of the {budget:?} it was given"
805        );
806    }
807
808    /// fails if a bark dog that cannot speak the protocol waits out its
809    /// whole budget before exiting: the daemon that refused is the party
810    /// that can fix it, so there is nothing to wait for.
811    #[tokio::test]
812    async fn a_wait_ends_on_a_refusal_rather_than_serving_out_the_budget() {
813        let dir = tempfile::tempdir().unwrap();
814        let path = control_address(dir.path());
815        let shepherds = fake_daemon_across_handovers(
816            &path,
817            vec![
818                Handshake::Accept(ack_from(11)),
819                Handshake::Refuse(RpcError {
820                    code: RpcErrorCode::ProtocolMismatch,
821                    message: "daemon speaks protocol 3, client speaks 2".into(),
822                    daemon_version: Some("0.9.9".into()),
823                }),
824            ],
825        );
826        let client = ReconnectingClient::connect(&path).await.unwrap();
827
828        subscribe_then_lose_it(&client, &shepherds).await;
829        // An hour, so serving the budget out could never look like passing.
830        let lost = tokio::time::timeout(BOUND, client.connected_within(Duration::from_secs(3600)))
831            .await
832            .expect("a refusal must end the wait long before its budget");
833
834        let Err(LinkLost::Refused {
835            daemon_version,
836            message,
837        }) = lost
838        else {
839            panic!("expected a refusal, got {lost:?}");
840        };
841        assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
842        assert!(message.contains("protocol 3"), "{message}");
843    }
844
845    /// fails if a dog watching for a lost shepherd fires while its shepherd
846    /// is right there, which would exit every dog on a healthy flock.
847    #[tokio::test]
848    async fn a_watch_for_a_lost_shepherd_stays_quiet_while_the_link_is_up() {
849        let dir = tempfile::tempdir().unwrap();
850        let path = control_address(dir.path());
851        let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
852        let client = ReconnectingClient::connect(&path).await.unwrap();
853
854        let fired =
855            tokio::time::timeout(NEGATIVE_WINDOW, client.link_lost(Duration::from_millis(1))).await;
856
857        assert!(
858            fired.is_err(),
859            "a connected client reported its shepherd lost: {fired:?}"
860        );
861    }
862
863    /// fails if a handover looks like a lost shepherd, which is the whole
864    /// distinction: the shepherd execs a successor on purpose, and every
865    /// dog is meant to cross that without exiting.
866    #[tokio::test]
867    async fn a_watch_for_a_lost_shepherd_rides_out_a_handover() {
868        let dir = tempfile::tempdir().unwrap();
869        let path = control_address(dir.path());
870        let shepherds = fake_daemon_across_handovers(
871            &path,
872            vec![
873                Handshake::Accept(ack_from(11)),
874                Handshake::Accept(ack_from(11)),
875            ],
876        );
877        let client = ReconnectingClient::connect(&path).await.unwrap();
878
879        shepherds.cut().await;
880        // Far longer than the reconnect ladder's first rung, so a successor
881        // that is coming has room to arrive.
882        let fired = tokio::time::timeout(NEGATIVE_WINDOW, client.link_lost(BOUND)).await;
883
884        assert!(
885            fired.is_err(),
886            "a handover was reported as a lost shepherd: {fired:?}"
887        );
888        assert_eq!(client.link(), LinkState::Connected);
889    }
890
891    /// fails if a dog a shepherd refuses sits out its whole budget before
892    /// noticing. `link_lost` reaches a refusal only by delegating to
893    /// `connected_within`, so the early exit is one call away from being
894    /// lost in a refactor, and nothing else here would catch it.
895    #[tokio::test]
896    async fn a_watch_for_a_lost_shepherd_ends_on_a_refusal_rather_than_a_budget() {
897        let dir = tempfile::tempdir().unwrap();
898        let path = control_address(dir.path());
899        let shepherds = fake_daemon_across_handovers(
900            &path,
901            vec![
902                Handshake::Accept(ack_from(11)),
903                Handshake::Refuse(RpcError {
904                    code: RpcErrorCode::ProtocolMismatch,
905                    message: "daemon speaks protocol 3, client speaks 2".into(),
906                    daemon_version: Some("0.9.9".into()),
907                }),
908            ],
909        );
910        let client = ReconnectingClient::connect(&path).await.unwrap();
911
912        subscribe_then_lose_it(&client, &shepherds).await;
913        // An hour, so serving the budget out could never look like passing.
914        let lost = tokio::time::timeout(BOUND, client.link_lost(Duration::from_secs(3600)))
915            .await
916            .expect("a refusal must end the watch long before its budget");
917
918        let LinkLost::Refused {
919            daemon_version,
920            message,
921        } = lost
922        else {
923            panic!("expected a refusal, got {lost:?}");
924        };
925        assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
926        assert!(message.contains("protocol 3"), "{message}");
927    }
928
929    /// fails if a dog whose shepherd stopped keeps waiting: the metrics dog
930    /// has no stream to end, so this watch is the only thing that tells it.
931    #[tokio::test]
932    async fn a_watch_for_a_lost_shepherd_fires_once_the_budget_is_spent() {
933        let dir = tempfile::tempdir().unwrap();
934        let path = control_address(dir.path());
935        let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
936        let client = ReconnectingClient::connect(&path).await.unwrap();
937
938        drop(shepherds);
939
940        let budget = Duration::from_millis(200);
941        let lost = tokio::time::timeout(BOUND, client.link_lost(budget))
942            .await
943            .expect("a spent budget must fire, not hang");
944
945        assert!(
946            matches!(lost, LinkLost::Budget { .. }),
947            "expected a spent budget, got {lost:?}"
948        );
949    }
950
951    /// fails if `Client`'s `Debug` starts printing its command channel, or
952    /// stops naming the dog it announces itself as. A derived impl would do
953    /// both, and the channel says nothing a reader can act on.
954    #[tokio::test]
955    async fn client_debug_names_the_socket_the_host_the_ack_and_the_dog() {
956        let dir = tempfile::tempdir().unwrap();
957        let path = control_address(dir.path());
958        let ack = ack_from(11);
959        let _shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack.clone())]);
960        let client = Client::connect_as(&path, HANDSHAKE_TIMEOUT, Some("metrics"))
961            .await
962            .unwrap();
963
964        assert_eq!(
965            format!("{client:?}"),
966            format!(
967                "Client {{ socket: {path:?}, host: None, ack: {ack:?}, dog_name: Some(\"metrics\"), .. }}"
968            )
969        );
970    }
971}