1use super::*;
2use std::time::Duration;
3use crate::client::Client;
6use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
7use tokio::time::Instant;
8
9impl Client {
10 pub async fn reconnect(&mut self) -> Result<Reconnected, ConnectError> {
18 self.reconnect_within(HANDSHAKE_TIMEOUT).await
19 }
20
21 pub async fn reconnect_within(
51 &mut self,
52 budget: Duration,
53 ) -> Result<Reconnected, ConnectError> {
54 let started = Instant::now();
55 let predecessor = self.daemon().pid;
56 let mut delay = RECONNECT_MIN_DELAY;
57
58 loop {
59 let left = budget.saturating_sub(started.elapsed());
60 let attempt = Client::connect_dial(
61 self.dial(),
62 Some(left.min(HANDSHAKE_TIMEOUT)),
63 self.dog_name(),
64 )
65 .await;
66
67 let failed = match attempt {
68 Ok(fresh) => {
69 let verdict = if fresh.daemon().pid == predecessor {
70 Reconnected::SameDaemon
71 } else {
72 Reconnected::NewDaemon
73 };
74 self.replace_connection(fresh);
75 return Ok(verdict);
76 }
77 Err(refusal @ ConnectError::ProtocolMismatch { .. }) => return Err(refusal),
78 Err(transient) => transient,
81 };
82
83 if budget.saturating_sub(started.elapsed()) <= delay {
84 return Err(failed);
85 }
86 tokio::time::sleep(delay).await;
87 delay = next_delay(delay);
88 }
89 }
90}
91
92#[cfg(test)]
93mod tests {
94
95 use std::time::Duration;
96
97 use crate::client::Client;
100 use crate::connection::{ConnectError, HANDSHAKE_TIMEOUT};
101 use shep_core::protocol::{HelloAck, Request};
102
103 use super::super::testing::*;
104 use super::*;
105 use crate::testing::{Handshake, control_address, fake_daemon_across_handovers};
106 use shep_core::protocol::{PROTOCOL_VERSION, RpcError, RpcErrorCode};
107
108 #[tokio::test]
112 async fn a_dropped_connection_is_re_established_and_the_next_request_is_served() {
113 let dir = tempfile::tempdir().unwrap();
114 let path = control_address(dir.path());
115 let shepherds = fake_daemon_across_handovers(
116 &path,
117 vec![
118 Handshake::Accept(ack_from(11)),
119 Handshake::Accept(ack_from(22)),
120 ],
121 );
122 let client = ReconnectingClient::connect(&path).await.unwrap();
123
124 let before = tokio::time::timeout(BOUND, client.request(Request::Ping))
125 .await
126 .expect("the first request must not hang");
127 assert!(before.is_ok(), "before the handover: {before:?}");
128
129 shepherds.cut().await;
132 await_reconnect(&client, &shepherds, 2).await;
133
134 let after = tokio::time::timeout(BOUND, client.request(Request::Ping))
135 .await
136 .expect("the request after a handover must not hang");
137 assert!(
138 after.is_ok(),
139 "a request after the handover must be served, got {after:?}"
140 );
141 }
142
143 #[tokio::test]
147 async fn the_ack_follows_the_successor_rather_than_the_predecessor() {
148 let dir = tempfile::tempdir().unwrap();
149 let path = control_address(dir.path());
150 let shepherds = fake_daemon_across_handovers(
151 &path,
152 vec![
153 Handshake::Accept(ack_from(11)),
154 Handshake::Accept(ack_from(22)),
155 ],
156 );
157 let client = ReconnectingClient::connect(&path).await.unwrap();
158 assert_eq!(client.daemon().pid, 11);
159 assert_eq!(client.daemon().daemon_version, "0.0.11");
160
161 shepherds.cut().await;
162 await_reconnect(&client, &shepherds, 2).await;
163
164 assert_eq!(
165 client.daemon().daemon_version,
166 "0.0.22",
167 "the version must come from the daemon now answering"
168 );
169 }
170
171 #[tokio::test]
176 async fn a_client_that_is_not_a_dog_names_none() {
177 let dir = tempfile::tempdir().unwrap();
178 let path = control_address(dir.path());
179 let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
180 let client = ReconnectingClient::connect(&path).await.unwrap();
181 assert_eq!(client.dog_name(), None);
182
183 shepherds.cut().await;
184 await_reconnect(&client, &shepherds, 2).await;
185
186 assert!(
187 shepherds
188 .hellos()
189 .iter()
190 .all(|hello| hello.dog_name.is_none()),
191 "an unnamed client must stay unnamed across a reconnect: {:?}",
192 shepherds.hellos()
193 );
194 }
195
196 #[tokio::test]
201 async fn a_refused_reconnect_stops_rather_than_spinning() {
202 let dir = tempfile::tempdir().unwrap();
203 let path = control_address(dir.path());
204 let shepherds = fake_daemon_across_handovers(
205 &path,
206 vec![
207 Handshake::Accept(ack_from(11)),
208 Handshake::Refuse(RpcError {
209 code: RpcErrorCode::ProtocolMismatch,
210 message: "daemon speaks protocol 3, client speaks 2".into(),
211 daemon_version: Some("0.9.9".into()),
212 }),
213 ],
214 );
215 let client = ReconnectingClient::connect(&path).await.unwrap();
216
217 shepherds.cut().await;
218 let link = await_refusal(&client).await;
219
220 let LinkState::Refused {
221 daemon_version,
222 message,
223 } = link
224 else {
225 unreachable!("await_refusal only returns a refusal");
226 };
227 assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
228 assert!(message.contains("protocol 3"), "{message}");
229 assert_eq!(
230 shepherds.accepted(),
231 2,
232 "one initial connection plus exactly one refused reconnect"
233 );
234 let spun = wait_until(NEGATIVE_WINDOW, || shepherds.accepted() >= 3).await;
239 assert!(
240 !spun,
241 "the supervisor retried a refusal: {} accepts",
242 shepherds.accepted()
243 );
244 }
245
246 #[tokio::test]
252 async fn a_reconnect_retries_past_a_successor_that_is_not_accepting_yet() {
253 let dir = tempfile::tempdir().unwrap();
254 let path = control_address(dir.path());
255 let shepherds = fake_daemon_across_handovers(
256 &path,
257 vec![
258 Handshake::Accept(ack_from(11)),
259 Handshake::Drop,
260 Handshake::Drop,
261 Handshake::Accept(ack_from(44)),
262 ],
263 );
264 let client = ReconnectingClient::connect(&path).await.unwrap();
265
266 shepherds.cut().await;
267 await_reconnect(&client, &shepherds, 4).await;
268
269 assert_eq!(
270 shepherds.accepted(),
271 4,
272 "two unanswered handshakes must be retried past, not given up on"
273 );
274 let served = tokio::time::timeout(BOUND, client.request(Request::Ping))
275 .await
276 .expect("the request after the retries must not hang");
277 assert!(served.is_ok(), "after the retries: {served:?}");
278 }
279
280 #[tokio::test]
284 async fn a_reconnect_to_the_same_daemon_reports_same_daemon() {
285 let dir = tempfile::tempdir().unwrap();
286 let path = control_address(dir.path());
287 let shepherds = fake_daemon_across_handovers(
288 &path,
289 vec![
290 Handshake::Accept(ack_from(11)),
291 Handshake::Accept(ack_from(11)),
292 ],
293 );
294 let mut client = Client::connect(&path).await.unwrap();
295
296 shepherds.cut().await;
297 let verdict = reconnect_ok(&mut client).await;
298
299 assert_eq!(verdict, Reconnected::SameDaemon);
300 }
301
302 #[tokio::test]
307 async fn a_reconnect_to_a_daemon_with_another_pid_reports_new_daemon() {
308 let dir = tempfile::tempdir().unwrap();
309 let path = control_address(dir.path());
310 let shepherds = fake_daemon_across_handovers(
311 &path,
312 vec![
313 Handshake::Accept(ack_from(11)),
314 Handshake::Accept(ack_from(22)),
315 ],
316 );
317 let mut client = Client::connect(&path).await.unwrap();
318
319 shepherds.cut().await;
320 let verdict = reconnect_ok(&mut client).await;
321
322 assert_eq!(verdict, Reconnected::NewDaemon);
323 }
324
325 #[tokio::test]
332 async fn a_reconnected_client_sends_its_next_request_to_the_successor() {
333 let dir = tempfile::tempdir().unwrap();
334 let path = control_address(dir.path());
335 let shepherds = fake_daemon_across_handovers(
336 &path,
337 vec![
338 Handshake::Accept(ack_from(11)),
339 Handshake::Accept(ack_from(22)),
340 ],
341 );
342 let mut client = Client::connect(&path).await.unwrap();
343
344 shepherds.cut().await;
345 let _ = reconnect_ok(&mut client).await;
348
349 let served = tokio::time::timeout(BOUND, client.request(Request::ListFlock))
350 .await
351 .expect("the request after a reconnect must not hang");
352 assert!(served.is_ok(), "after the reconnect: {served:?}");
353
354 let successor: Vec<Request> = shepherds
355 .envelopes()
356 .into_iter()
357 .filter(|(generation, _)| *generation == 2)
358 .map(|(_, envelope)| envelope.body)
359 .collect();
360 assert_eq!(successor, vec![Request::ListFlock]);
361 }
362
363 #[tokio::test]
367 async fn a_reconnect_updates_the_ack_to_the_daemon_now_answering() {
368 let dir = tempfile::tempdir().unwrap();
369 let path = control_address(dir.path());
370 let shepherds = fake_daemon_across_handovers(
371 &path,
372 vec![
373 Handshake::Accept(ack_from(11)),
374 Handshake::Accept(ack_from(22)),
375 ],
376 );
377 let mut client = Client::connect(&path).await.unwrap();
378 assert_eq!(client.daemon().daemon_version, "0.0.11");
379
380 shepherds.cut().await;
381 let _ = reconnect_ok(&mut client).await;
384
385 assert_eq!(client.daemon().pid, 22);
386 assert_eq!(client.daemon().daemon_version, "0.0.22");
387 }
388
389 #[tokio::test]
394 async fn a_reconnect_retries_past_a_successor_still_coming_up() {
395 let dir = tempfile::tempdir().unwrap();
396 let path = control_address(dir.path());
397 let shepherds = fake_daemon_across_handovers(
398 &path,
399 vec![
400 Handshake::Accept(ack_from(11)),
401 Handshake::Drop,
402 Handshake::Drop,
403 Handshake::Accept(ack_from(44)),
404 ],
405 );
406 let mut client = Client::connect(&path).await.unwrap();
407
408 shepherds.cut().await;
409 let verdict = reconnect_ok(&mut client).await;
410
411 assert_eq!(verdict, Reconnected::NewDaemon);
412 assert_eq!(
413 shepherds.accepted(),
414 4,
415 "two unanswered handshakes must be retried past, not given up on"
416 );
417 }
418
419 #[tokio::test]
423 async fn a_reconnect_returns_a_refusal_rather_than_retrying_it() {
424 let dir = tempfile::tempdir().unwrap();
425 let path = control_address(dir.path());
426 let shepherds = fake_daemon_across_handovers(
427 &path,
428 vec![
429 Handshake::Accept(ack_from(11)),
430 Handshake::Refuse(RpcError {
431 code: RpcErrorCode::ProtocolMismatch,
432 message: "daemon speaks protocol 3, client speaks 2".into(),
433 daemon_version: Some("0.9.9".into()),
434 }),
435 ],
436 );
437 let mut client = Client::connect(&path).await.unwrap();
438
439 shepherds.cut().await;
440 let refused = tokio::time::timeout(BOUND, client.reconnect_within(BOUND))
441 .await
442 .expect("a refusal must be reported, not waited out")
443 .expect_err("the successor refused this handshake");
444
445 let ConnectError::ProtocolMismatch {
446 daemon_version,
447 message,
448 ..
449 } = refused
450 else {
451 panic!("a refusal must arrive as a mismatch, got {refused:?}");
452 };
453 assert_eq!(daemon_version.as_deref(), Some("0.9.9"));
454 assert!(message.contains("protocol 3"), "{message}");
455 assert_eq!(
456 shepherds.accepted(),
457 2,
458 "one initial connection plus exactly one refused reconnect"
459 );
460 }
461
462 #[tokio::test]
469 async fn a_reconnect_gives_up_when_its_budget_is_spent() {
470 let dir = tempfile::tempdir().unwrap();
471 let path = control_address(dir.path());
472 let shepherds = fake_daemon_across_handovers(&path, vec![Handshake::Accept(ack_from(11))]);
473 let mut client = Client::connect(&path).await.unwrap();
474
475 drop(shepherds);
478
479 let budget = Duration::from_millis(200);
480 let started = tokio::time::Instant::now();
481 let gave_up = tokio::time::timeout(BOUND, client.reconnect_within(budget))
482 .await
483 .expect("a spent budget must return, not hang");
484 let elapsed = started.elapsed();
485
486 assert!(
487 gave_up.is_err(),
488 "a daemon that is gone must not report a reconnect: {gave_up:?}"
489 );
490 assert!(
491 elapsed >= RECONNECT_MIN_DELAY,
492 "gave up without retrying once, after {elapsed:?}"
493 );
494 }
495
496 #[tokio::test]
500 async fn a_dogs_name_rides_a_reconnect() {
501 let dir = tempfile::tempdir().unwrap();
502 let path = control_address(dir.path());
503 let shepherds = fake_daemon_across_handovers(
504 &path,
505 vec![
506 Handshake::Accept(ack_from(11)),
507 Handshake::Accept(ack_from(22)),
508 ],
509 );
510 let mut client = Client::connect_as(&path, HANDSHAKE_TIMEOUT, Some("metrics"))
511 .await
512 .unwrap();
513
514 shepherds.cut().await;
515 let _ = reconnect_ok(&mut client).await;
518
519 let named: Vec<Option<String>> = shepherds
520 .hellos()
521 .into_iter()
522 .map(|hello| hello.dog_name)
523 .collect();
524 assert_eq!(
525 named,
526 vec![Some("metrics".to_string()), Some("metrics".to_string())],
527 "every generation must be told which dog is talking to it"
528 );
529 }
530
531 #[tokio::test]
534 async fn a_client_that_is_not_a_dog_stays_anonymous_across_a_reconnect() {
535 let dir = tempfile::tempdir().unwrap();
536 let path = control_address(dir.path());
537 let shepherds = fake_daemon_across_handovers(
538 &path,
539 vec![
540 Handshake::Accept(ack_from(11)),
541 Handshake::Accept(ack_from(22)),
542 ],
543 );
544 let mut client = Client::connect(&path).await.unwrap();
545
546 shepherds.cut().await;
547 let _ = reconnect_ok(&mut client).await;
550
551 assert!(
552 shepherds
553 .hellos()
554 .iter()
555 .all(|hello| hello.dog_name.is_none()),
556 "an unnamed client must stay unnamed: {:?}",
557 shepherds.hellos()
558 );
559 }
560
561 #[tokio::test]
566 async fn an_upgrade_handover_keeps_its_pid_and_reports_same_daemon() {
567 let dir = tempfile::tempdir().unwrap();
568 let path = control_address(dir.path());
569 let upgraded = |version: &str| HelloAck {
570 daemon_version: version.to_string(),
571 protocol: PROTOCOL_VERSION,
572 pid: 11,
573 min_supported: None,
574 };
575 let shepherds = fake_daemon_across_handovers(
576 &path,
577 vec![
578 Handshake::Accept(upgraded("0.8.0")),
579 Handshake::Accept(upgraded("0.9.0")),
580 ],
581 );
582 let mut client = Client::connect(&path).await.unwrap();
583
584 shepherds.cut().await;
585 let verdict = reconnect_ok(&mut client).await;
586
587 assert_eq!(verdict, Reconnected::SameDaemon);
588 assert_eq!(
589 client.daemon().daemon_version,
590 "0.9.0",
591 "the ack must still follow the build now answering"
592 );
593 }
594
595 #[tokio::test]
600 async fn a_restart_onto_the_same_build_reports_new_daemon() {
601 let dir = tempfile::tempdir().unwrap();
602 let path = control_address(dir.path());
603 let same_build = |pid: u32| HelloAck {
604 daemon_version: "0.9.0".to_string(),
605 protocol: PROTOCOL_VERSION,
606 pid,
607 min_supported: None,
608 };
609 let shepherds = fake_daemon_across_handovers(
610 &path,
611 vec![
612 Handshake::Accept(same_build(11)),
613 Handshake::Accept(same_build(22)),
614 ],
615 );
616 let mut client = Client::connect(&path).await.unwrap();
617
618 shepherds.cut().await;
619 let verdict = reconnect_ok(&mut client).await;
620
621 assert_eq!(verdict, Reconnected::NewDaemon);
622 }
623
624 #[cfg(unix)]
627 #[tokio::test]
628 async fn reconnect_over_ssh_redials_through_the_same_stand_in() {
629 use crate::ssh::SshOptions;
630 use crate::ssh::stand_in::{stand_in, write_reply};
631
632 let dir = tempfile::tempdir().unwrap();
633 write_reply(dir.path());
634 let program = stand_in(
635 dir.path(),
636 "echo run >> \"$(dirname \"$0\")/runs.txt\"\ncat \"$(dirname \"$0\")/reply.bin\"",
637 );
638 let mut client =
639 Client::connect_ssh_with("build-01", &SshOptions::default().program(program))
640 .await
641 .unwrap();
642 tokio::time::timeout(BOUND, client.closed())
643 .await
644 .expect("the stand-in exits after its reply, ending the connection");
645
646 let verdict = tokio::time::timeout(BOUND, client.reconnect_within(HANDSHAKE_TIMEOUT))
647 .await
648 .expect("the redial must not hang")
649 .unwrap();
650
651 assert_eq!(verdict, Reconnected::SameDaemon);
652 assert_eq!(client.host(), Some("build-01"));
653 let runs = std::fs::read_to_string(dir.path().join("runs.txt")).unwrap();
654 assert_eq!(runs.lines().count(), 2, "{runs:?}");
655 }
656}