Skip to main content

shep_client/reconnect/
client.rs

1use super::*;
2use std::time::Duration;
3// tokio's Instant, not std's: it moves with `tokio::time::pause`, and the
4// budget below is measured against a `tokio::time::sleep` that does too.
5use crate::client::Client;
6use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
7use tokio::time::Instant;
8
9impl Client {
10    /// Re-establishes this connection, reporting which daemon answered.
11    ///
12    /// Shorthand for [`Self::reconnect_within`] with [`HANDSHAKE_TIMEOUT`].
13    ///
14    /// # Errors
15    ///
16    /// See [`Self::reconnect_within`].
17    pub async fn reconnect(&mut self) -> Result<Reconnected, ConnectError> {
18        self.reconnect_within(HANDSHAKE_TIMEOUT).await
19    }
20
21    /// Re-establishes this connection, retrying until `budget` is spent.
22    ///
23    /// Meant for a connection that has already ended, as [`Self::closed`]
24    /// reports. A successor still coming up is retried, from
25    /// [`RECONNECT_MIN_DELAY`] doubling to [`RECONNECT_MAX_DELAY`]; a refusal is
26    /// not. Safe to cancel: the old connection is held until a new one
27    /// handshakes. An [`EventStream`](crate::events::EventStream) belongs to the connection it was taken on.
28    ///
29    /// # Example
30    ///
31    /// ```no_run
32    /// use shep_client::{Client, Reconnected};
33    ///
34    /// # async fn watch(client: &mut Client, held: &mut Option<u32>)
35    /// # -> Result<(), Box<dyn core::error::Error>> {
36    /// client.closed().await;
37    /// if client.reconnect().await? == Reconnected::NewDaemon {
38    ///     // Ids are minted per daemon and never persisted, so this one now
39    ///     // names a different sheep, or none at all.
40    ///     *held = None;
41    /// }
42    /// # Ok(())
43    /// # }
44    /// # let _ = watch;
45    /// ```
46    /// # Errors
47    ///
48    /// - [`ConnectError::ProtocolMismatch`]: a daemon refused on version skew, and is not asked again.
49    /// - Anything else [`Self::connect_with_timeout`] reports, once `budget` leaves no room.
50    pub async fn reconnect_within(
51        &mut self,
52        budget: Duration,
53    ) -> Result<Reconnected, ConnectError> {
54        let started = Instant::now();
55        let predecessor = self.daemon().pid;
56        let mut delay = RECONNECT_MIN_DELAY;
57
58        loop {
59            let left = budget.saturating_sub(started.elapsed());
60            let attempt = Client::connect_dial(
61                self.dial(),
62                Some(left.min(HANDSHAKE_TIMEOUT)),
63                self.dog_name(),
64            )
65            .await;
66
67            let failed = match attempt {
68                Ok(fresh) => {
69                    let verdict = if fresh.daemon().pid == predecessor {
70                        Reconnected::SameDaemon
71                    } else {
72                        Reconnected::NewDaemon
73                    };
74                    self.replace_connection(fresh);
75                    return Ok(verdict);
76                }
77                Err(refusal @ ConnectError::ProtocolMismatch { .. }) => return Err(refusal),
78                // Everything else is a daemon that is not ready yet, which
79                // resolves on its own if the budget outlasts it.
80                Err(transient) => transient,
81            };
82
83            if budget.saturating_sub(started.elapsed()) <= delay {
84                return Err(failed);
85            }
86            tokio::time::sleep(delay).await;
87            delay = next_delay(delay);
88        }
89    }
90}
91
92#[cfg(test)]
93mod tests {
94
95    use std::time::Duration;
96
97    // tokio's Instant, not std's: it moves with `tokio::time::pause`, and the
98    // budget below is measured against a `tokio::time::sleep` that does too.
99    use crate::client::Client;
100    use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
101    use shep_core::protocol::{HelloAck, Request};
102
103    use super::super::testing::*;
104    use super::*;
105    use crate::testing::{Handshake, control_address, fake_daemon_across_handovers};
106    use shep_core::protocol::{PROTOCOL_VERSION, RpcError, RpcErrorCode};
107
108    /// fails if a dog is left holding a dead socket after its daemon is
109    /// replaced: only the listening socket crosses the exec, and an
110    /// accepted one dies with the image.
111    #[tokio::test]
112    async fn a_dropped_connection_is_re_established_and_the_next_request_is_served() {
113        let dir = tempfile::tempdir().unwrap();
114        let path = control_address(dir.path());
115        let shepherds = fake_daemon_across_handovers(
116            &path,
117            vec![
118                Handshake::Accept(ack_from(11)),
119                Handshake::Accept(ack_from(22)),
120            ],
121        );
122        let client = ReconnectingClient::connect(&path).await.unwrap();
123
124        let before = tokio::time::timeout(BOUND, client.request(Request::Ping))
125            .await
126            .expect("the first request must not hang");
127        assert!(before.is_ok(), "before the handover: {before:?}");
128
129        // The handover, exactly: the accepted connection dies under the
130        // client while the listener stays bound.
131        shepherds.cut().await;
132        await_reconnect(&client, &shepherds, 2).await;
133
134        let after = tokio::time::timeout(BOUND, client.request(Request::Ping))
135            .await
136            .expect("the request after a handover must not hang");
137        assert!(
138            after.is_ok(),
139            "a request after the handover must be served, got {after:?}"
140        );
141    }
142
143    /// fails if the ack keeps describing the predecessor after a
144    /// reconnect, which would make a dog publish `daemon_version` for a
145    /// daemon no longer running.
146    #[tokio::test]
147    async fn the_ack_follows_the_successor_rather_than_the_predecessor() {
148        let dir = tempfile::tempdir().unwrap();
149        let path = control_address(dir.path());
150        let shepherds = fake_daemon_across_handovers(
151            &path,
152            vec![
153                Handshake::Accept(ack_from(11)),
154                Handshake::Accept(ack_from(22)),
155            ],
156        );
157        let client = ReconnectingClient::connect(&path).await.unwrap();
158        assert_eq!(client.daemon().pid, 11);
159        assert_eq!(client.daemon().daemon_version, "0.0.11");
160
161        shepherds.cut().await;
162        await_reconnect(&client, &shepherds, 2).await;
163
164        assert_eq!(
165            client.daemon().daemon_version,
166            "0.0.22",
167            "the version must come from the daemon now answering"
168        );
169    }
170
171    /// fails if a client that is not a dog claims to be one. The name is
172    /// what lets a daemon restart a dog on a refused handshake, so a
173    /// `ReconnectingClient` built without one must stay anonymous rather
174    /// than inventing a name from its environment or its path.
175    #[tokio::test]
176    async fn a_client_that_is_not_a_dog_names_none() {
177        let dir = tempfile::tempdir().unwrap();
178        let path = control_address(dir.path());
179        let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
180        let client = ReconnectingClient::connect(&path).await.unwrap();
181        assert_eq!(client.dog_name(), None);
182
183        shepherds.cut().await;
184        await_reconnect(&client, &shepherds, 2).await;
185
186        assert!(
187            shepherds
188                .hellos()
189                .iter()
190                .all(|hello| hello.dog_name.is_none()),
191            "an unnamed client must stay unnamed across a reconnect: {:?}",
192            shepherds.hellos()
193        );
194    }
195
196    /// fails if a refused reconnect is retried. A successor that refuses on
197    /// protocol skew has said something no retry can change; the design's
198    /// G8 puts the fix on the daemon (restart that dog once, from disk) and
199    /// forbids the client spinning against it in the meantime.
200    #[tokio::test]
201    async fn a_refused_reconnect_stops_rather_than_spinning() {
202        let dir = tempfile::tempdir().unwrap();
203        let path = control_address(dir.path());
204        let shepherds = fake_daemon_across_handovers(
205            &path,
206            vec![
207                Handshake::Accept(ack_from(11)),
208                Handshake::Refuse(RpcError {
209                    code: RpcErrorCode::ProtocolMismatch,
210                    message: "daemon speaks protocol 3, client speaks 2".into(),
211                    daemon_version: Some("0.9.9".into()),
212                }),
213            ],
214        );
215        let client = ReconnectingClient::connect(&path).await.unwrap();
216
217        shepherds.cut().await;
218        let link = await_refusal(&client).await;
219
220        let LinkState::Refused {
221            daemon_version,
222            message,
223        } = link
224        else {
225            unreachable!("await_refusal only returns a refusal");
226        };
227        assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
228        assert!(message.contains("protocol 3"), "{message}");
229        assert_eq!(
230            shepherds.accepted(),
231            2,
232            "one initial connection plus exactly one refused reconnect"
233        );
234        // And it must STAY at two. Reaching `Refused` only proves the
235        // supervisor got there; a supervisor that recorded the refusal and
236        // then went round again would satisfy every assertion above and
237        // still be the spin G8 forbids, so the window is the assertion.
238        let spun = wait_until(NEGATIVE_WINDOW, || shepherds.accepted() >= 3).await;
239        assert!(
240            !spun,
241            "the supervisor retried a refusal: {} accepts",
242            shepherds.accepted()
243        );
244    }
245
246    /// fails if a reconnect gives up on a daemon that is merely not ready
247    /// yet. Across a real handover the listening socket stays bound while
248    /// the successor replays its blob, so a connect that completes and a
249    /// handshake that is not yet answered is the ordinary case, not a
250    /// failure.
251    #[tokio::test]
252    async fn a_reconnect_retries_past_a_successor_that_is_not_accepting_yet() {
253        let dir = tempfile::tempdir().unwrap();
254        let path = control_address(dir.path());
255        let shepherds = fake_daemon_across_handovers(
256            &path,
257            vec![
258                Handshake::Accept(ack_from(11)),
259                Handshake::Drop,
260                Handshake::Drop,
261                Handshake::Accept(ack_from(44)),
262            ],
263        );
264        let client = ReconnectingClient::connect(&path).await.unwrap();
265
266        shepherds.cut().await;
267        await_reconnect(&client, &shepherds, 4).await;
268
269        assert_eq!(
270            shepherds.accepted(),
271            4,
272            "two unanswered handshakes must be retried past, not given up on"
273        );
274        let served = tokio::time::timeout(BOUND, client.request(Request::Ping))
275            .await
276            .expect("the request after the retries must not hang");
277        assert!(served.is_ok(), "after the retries: {served:?}");
278    }
279
280    /// fails if a reconnect that reached the very daemon it was talking to
281    /// before reports a different one, which would have a caller throw away
282    /// ids that are still valid.
283    #[tokio::test]
284    async fn a_reconnect_to_the_same_daemon_reports_same_daemon() {
285        let dir = tempfile::tempdir().unwrap();
286        let path = control_address(dir.path());
287        let shepherds = fake_daemon_across_handovers(
288            &path,
289            vec![
290                Handshake::Accept(ack_from(11)),
291                Handshake::Accept(ack_from(11)),
292            ],
293        );
294        let mut client = Client::connect(&path).await.unwrap();
295
296        shepherds.cut().await;
297        let verdict = reconnect_ok(&mut client).await;
298
299        assert_eq!(verdict, Reconnected::SameDaemon);
300    }
301
302    /// fails if a reconnect that landed on a daemon started after the
303    /// predecessor died reports the same one. That daemon mints its ids
304    /// from a fresh space, so a caller acting on a held id would act on
305    /// whatever now happens to wear the number.
306    #[tokio::test]
307    async fn a_reconnect_to_a_daemon_with_another_pid_reports_new_daemon() {
308        let dir = tempfile::tempdir().unwrap();
309        let path = control_address(dir.path());
310        let shepherds = fake_daemon_across_handovers(
311            &path,
312            vec![
313                Handshake::Accept(ack_from(11)),
314                Handshake::Accept(ack_from(22)),
315            ],
316        );
317        let mut client = Client::connect(&path).await.unwrap();
318
319        shepherds.cut().await;
320        let verdict = reconnect_ok(&mut client).await;
321
322        assert_eq!(verdict, Reconnected::NewDaemon);
323    }
324
325    /// fails if a reconnect reports a verdict without actually swapping the
326    /// connection under the handle, which would leave every later request
327    /// going to a socket nobody is serving.
328    ///
329    /// The proof is positional: the successor's first envelope must be the
330    /// request issued after the reconnect.
331    #[tokio::test]
332    async fn a_reconnected_client_sends_its_next_request_to_the_successor() {
333        let dir = tempfile::tempdir().unwrap();
334        let path = control_address(dir.path());
335        let shepherds = fake_daemon_across_handovers(
336            &path,
337            vec![
338                Handshake::Accept(ack_from(11)),
339                Handshake::Accept(ack_from(22)),
340            ],
341        );
342        let mut client = Client::connect(&path).await.unwrap();
343
344        shepherds.cut().await;
345        // the verdict is asserted by its own cases above; this one is about
346        // what the reconnect did, not what it reported
347        let _ = reconnect_ok(&mut client).await;
348
349        let served = tokio::time::timeout(BOUND, client.request(Request::ListFlock))
350            .await
351            .expect("the request after a reconnect must not hang");
352        assert!(served.is_ok(), "after the reconnect: {served:?}");
353
354        let successor: Vec<Request> = shepherds
355            .envelopes()
356            .into_iter()
357            .filter(|(generation, _)| *generation == 2)
358            .map(|(_, envelope)| envelope.body)
359            .collect();
360        assert_eq!(successor, vec![Request::ListFlock]);
361    }
362
363    /// fails if the ack keeps describing the predecessor after a reconnect,
364    /// so a caller reading `daemon_version` would report a build that is no
365    /// longer running.
366    #[tokio::test]
367    async fn a_reconnect_updates_the_ack_to_the_daemon_now_answering() {
368        let dir = tempfile::tempdir().unwrap();
369        let path = control_address(dir.path());
370        let shepherds = fake_daemon_across_handovers(
371            &path,
372            vec![
373                Handshake::Accept(ack_from(11)),
374                Handshake::Accept(ack_from(22)),
375            ],
376        );
377        let mut client = Client::connect(&path).await.unwrap();
378        assert_eq!(client.daemon().daemon_version, "0.0.11");
379
380        shepherds.cut().await;
381        // the verdict is asserted by its own cases above; this one is about
382        // what the reconnect did, not what it reported
383        let _ = reconnect_ok(&mut client).await;
384
385        assert_eq!(client.daemon().pid, 22);
386        assert_eq!(client.daemon().daemon_version, "0.0.22");
387    }
388
389    /// fails if a reconnect gives up on a daemon that is merely not ready
390    /// yet. Across a handover the listening socket stays bound while the
391    /// successor replays its blob, so a connect that completes and a
392    /// handshake that is not yet answered is the ordinary case.
393    #[tokio::test]
394    async fn a_reconnect_retries_past_a_successor_still_coming_up() {
395        let dir = tempfile::tempdir().unwrap();
396        let path = control_address(dir.path());
397        let shepherds = fake_daemon_across_handovers(
398            &path,
399            vec![
400                Handshake::Accept(ack_from(11)),
401                Handshake::Drop,
402                Handshake::Drop,
403                Handshake::Accept(ack_from(44)),
404            ],
405        );
406        let mut client = Client::connect(&path).await.unwrap();
407
408        shepherds.cut().await;
409        let verdict = reconnect_ok(&mut client).await;
410
411        assert_eq!(verdict, Reconnected::NewDaemon);
412        assert_eq!(
413            shepherds.accepted(),
414            4,
415            "two unanswered handshakes must be retried past, not given up on"
416        );
417    }
418
419    /// fails if a refused reconnect is retried. A successor that refuses on
420    /// protocol skew has said something no retry can change, so the caller
421    /// gets the refusal rather than the budget being spent against it.
422    #[tokio::test]
423    async fn a_reconnect_returns_a_refusal_rather_than_retrying_it() {
424        let dir = tempfile::tempdir().unwrap();
425        let path = control_address(dir.path());
426        let shepherds = fake_daemon_across_handovers(
427            &path,
428            vec![
429                Handshake::Accept(ack_from(11)),
430                Handshake::Refuse(RpcError {
431                    code: RpcErrorCode::ProtocolMismatch,
432                    message: "daemon speaks protocol 3, client speaks 2".into(),
433                    daemon_version: Some("0.9.9".into()),
434                }),
435            ],
436        );
437        let mut client = Client::connect(&path).await.unwrap();
438
439        shepherds.cut().await;
440        let refused = tokio::time::timeout(BOUND, client.reconnect_within(BOUND))
441            .await
442            .expect("a refusal must be reported, not waited out")
443            .expect_err("the successor refused this handshake");
444
445        let ConnectError::ProtocolMismatch {
446            daemon_version,
447            message,
448            ..
449        } = refused
450        else {
451            panic!("a refusal must arrive as a mismatch, got {refused:?}");
452        };
453        assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
454        assert!(message.contains("protocol 3"), "{message}");
455        assert_eq!(
456            shepherds.accepted(),
457            2,
458            "one initial connection plus exactly one refused reconnect"
459        );
460    }
461
462    /// fails if a reconnect against a daemon that is genuinely gone waits
463    /// forever, or gives up on its first attempt without retrying at all.
464    ///
465    /// The budget is the forcing mechanism, and the elapsed time is the
466    /// assertion at both ends: at least one delay means it retried, and
467    /// returning at all means it stopped.
468    #[tokio::test]
469    async fn a_reconnect_gives_up_when_its_budget_is_spent() {
470        let dir = tempfile::tempdir().unwrap();
471        let path = control_address(dir.path());
472        let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
473        let mut client = Client::connect(&path).await.unwrap();
474
475        // A daemon that is gone for good, listener and all, rather than one
476        // replaced: nothing will answer this address again.
477        drop(shepherds);
478
479        let budget = Duration::from_millis(200);
480        let started = tokio::time::Instant::now();
481        let gave_up = tokio::time::timeout(BOUND, client.reconnect_within(budget))
482            .await
483            .expect("a spent budget must return, not hang");
484        let elapsed = started.elapsed();
485
486        assert!(
487            gave_up.is_err(),
488            "a daemon that is gone must not report a reconnect: {gave_up:?}"
489        );
490        assert!(
491            elapsed >= RECONNECT_MIN_DELAY,
492            "gave up without retrying once, after {elapsed:?}"
493        );
494    }
495
496    /// fails if a dog's name reaches the daemon it booted against and not
497    /// the one it reconnects to. The refusal that matters is the second
498    /// one, and a daemon cannot ask its predecessor which dog was talking.
499    #[tokio::test]
500    async fn a_dogs_name_rides_a_reconnect() {
501        let dir = tempfile::tempdir().unwrap();
502        let path = control_address(dir.path());
503        let shepherds = fake_daemon_across_handovers(
504            &path,
505            vec![
506                Handshake::Accept(ack_from(11)),
507                Handshake::Accept(ack_from(22)),
508            ],
509        );
510        let mut client = Client::connect_as(&path, HANDSHAKE_TIMEOUT, Some("metrics"))
511            .await
512            .unwrap();
513
514        shepherds.cut().await;
515        // the verdict is asserted by its own cases above; this one is about
516        // what the reconnect did, not what it reported
517        let _ = reconnect_ok(&mut client).await;
518
519        let named: Vec<Option<String>> = shepherds
520            .hellos()
521            .into_iter()
522            .map(|hello| hello.dog_name)
523            .collect();
524        assert_eq!(
525            named,
526            vec![Some("metrics".to_string()), Some("metrics".to_string())],
527            "every generation must be told which dog is talking to it"
528        );
529    }
530
531    /// fails if a client that is not a dog claims to be one after a
532    /// reconnect, which would have a daemon restart a dog nobody adopted.
533    #[tokio::test]
534    async fn a_client_that_is_not_a_dog_stays_anonymous_across_a_reconnect() {
535        let dir = tempfile::tempdir().unwrap();
536        let path = control_address(dir.path());
537        let shepherds = fake_daemon_across_handovers(
538            &path,
539            vec![
540                Handshake::Accept(ack_from(11)),
541                Handshake::Accept(ack_from(22)),
542            ],
543        );
544        let mut client = Client::connect(&path).await.unwrap();
545
546        shepherds.cut().await;
547        // the verdict is asserted by its own cases above; this one is about
548        // what the reconnect did, not what it reported
549        let _ = reconnect_ok(&mut client).await;
550
551        assert!(
552            shepherds
553                .hellos()
554                .iter()
555                .all(|hello| hello.dog_name.is_none()),
556            "an unnamed client must stay unnamed: {:?}",
557            shepherds.hellos()
558        );
559    }
560
561    // `shep daemon reload` onto a new build is still an execve. It keeps the
562    // pid and carries the id counter while the version changes. A version
563    // comparison would call that a new daemon and discard ids that are still
564    // good.
565    #[tokio::test]
566    async fn an_upgrade_handover_keeps_its_pid_and_reports_same_daemon() {
567        let dir = tempfile::tempdir().unwrap();
568        let path = control_address(dir.path());
569        let upgraded = |version: &str| HelloAck {
570            daemon_version: version.to_string(),
571            protocol: PROTOCOL_VERSION,
572            pid: 11,
573            min_supported: None,
574        };
575        let shepherds = fake_daemon_across_handovers(
576            &path,
577            vec![
578                Handshake::Accept(upgraded("0.8.0")),
579                Handshake::Accept(upgraded("0.9.0")),
580            ],
581        );
582        let mut client = Client::connect(&path).await.unwrap();
583
584        shepherds.cut().await;
585        let verdict = reconnect_ok(&mut client).await;
586
587        assert_eq!(verdict, Reconnected::SameDaemon);
588        assert_eq!(
589            client.daemon().daemon_version,
590            "0.9.0",
591            "the ack must still follow the build now answering"
592        );
593    }
594
595    /// fails if the verdict follows `daemon_version` rather than the pid, in
596    /// the other direction: a daemon stopped and started again on the SAME
597    /// build mints its ids from a fresh space, so a version comparison would
598    /// wave a caller through to reuse ids that now mean nothing.
599    #[tokio::test]
600    async fn a_restart_onto_the_same_build_reports_new_daemon() {
601        let dir = tempfile::tempdir().unwrap();
602        let path = control_address(dir.path());
603        let same_build = |pid: u32| HelloAck {
604            daemon_version: "0.9.0".to_string(),
605            protocol: PROTOCOL_VERSION,
606            pid,
607            min_supported: None,
608        };
609        let shepherds = fake_daemon_across_handovers(
610            &path,
611            vec![
612                Handshake::Accept(same_build(11)),
613                Handshake::Accept(same_build(22)),
614            ],
615        );
616        let mut client = Client::connect(&path).await.unwrap();
617
618        shepherds.cut().await;
619        let verdict = reconnect_ok(&mut client).await;
620
621        assert_eq!(verdict, Reconnected::NewDaemon);
622    }
623
624    /// fails if a reconnect of an ssh client reads `socket()` (empty) instead of
625    /// running the same ssh again.
626    #[cfg(unix)]
627    #[tokio::test]
628    async fn reconnect_over_ssh_redials_through_the_same_stand_in() {
629        use crate::ssh::SshOptions;
630        use crate::ssh::stand_in::{stand_in, write_reply};
631
632        let dir = tempfile::tempdir().unwrap();
633        write_reply(dir.path());
634        let program = stand_in(
635            dir.path(),
636            "echo run >> \"$(dirname \"$0\")/runs.txt\"\ncat \"$(dirname \"$0\")/reply.bin\"",
637        );
638        let mut client =
639            Client::connect_ssh_with("build-01", &SshOptions::default().program(program))
640                .await
641                .unwrap();
642        tokio::time::timeout(BOUND, client.closed())
643            .await
644            .expect("the stand-in exits after its reply, ending the connection");
645
646        let verdict = tokio::time::timeout(BOUND, client.reconnect_within(HANDSHAKE_TIMEOUT))
647            .await
648            .expect("the redial must not hang")
649            .unwrap();
650
651        assert_eq!(verdict, Reconnected::SameDaemon);
652        assert_eq!(client.host(), Some("build-01"));
653        let runs = std::fs::read_to_string(dir.path().join("runs.txt")).unwrap();
654        assert_eq!(runs.lines().count(), 2, "{runs:?}");
655    }
656}