code0_flow/flow_service/
retry.rs

1use tokio::time::{Duration, sleep};
2use tonic::transport::{Channel, Endpoint};
3
4const MAX_BACKOFF: u64 = 2000 * 60;
5const MAX_RETRIES: i8 = 10;
6
7// Will create a channel and retry if its not possible
8pub async fn create_channel_with_retry(
9    channel_name: &str,
10    url: String,
11    connect_timeout: Duration,
12    request_timeout: Duration,
13) -> Channel {
14    let mut backoff = 100;
15    let mut retries = 0;
16
17    loop {
18        let channel = match Endpoint::from_shared(url.clone()) {
19            Ok(c) => {
20                log::debug!("Creating a new endpoint for the: {} Service", channel_name);
21                c.connect_timeout(connect_timeout).timeout(request_timeout)
22            }
23            Err(err) => {
24                panic!(
25                    "Cannot create Endpoint for Service: `{}`. Reason: {:?}",
26                    channel_name, err
27                );
28            }
29        };
30
31        match channel.connect().await {
32            Ok(ch) => {
33                return ch;
34            }
35            Err(err) => {
36                log::warn!(
37                    "Retry connect to `{}` using url: `{}` failed: {:?}, retrying in {}ms",
38                    channel_name,
39                    url,
40                    err,
41                    backoff
42                );
43                sleep(Duration::from_millis(backoff)).await;
44
45                backoff = (backoff * 2).min(MAX_BACKOFF);
46                retries += 1;
47
48                if retries >= MAX_RETRIES {
49                    panic!("Reached max retries to url {}", url)
50                }
51            }
52        }
53    }
54}
code0_flow/flow_service/retry.rs

code0_flow/flow_service/
retry.rs
