commonware_deployer/ec2/
utils.rs

1//! Utility functions for interacting with EC2 instances
2
3use crate::ec2::Error;
4use tokio::{
5    process::Command,
6    time::{sleep, Duration},
7};
8use tracing::warn;
9
10/// Maximum number of SSH connection attempts before failing
11pub const MAX_SSH_ATTEMPTS: usize = 30;
12
13/// Maximum number of polling attempts for service status
14pub const MAX_POLL_ATTEMPTS: usize = 30;
15
16/// Interval between retries
17pub const RETRY_INTERVAL: Duration = Duration::from_secs(10);
18
19/// Protocol for deployer ingress
20pub const DEPLOYER_PROTOCOL: &str = "tcp";
21
22/// Minimum port for deployer ingress
23pub const DEPLOYER_MIN_PORT: i32 = 0;
24
25/// Maximum port for deployer ingress
26pub const DEPLOYER_MAX_PORT: i32 = 65535;
27
28/// Fetch the current machine's public IPv4 address
29pub async fn get_public_ip() -> Result<String, Error> {
30    // icanhazip.com is maintained by Cloudflare as of 6/6/2021 (https://major.io/p/a-new-future-for-icanhazip/)
31    let result = reqwest::get("https://ipv4.icanhazip.com")
32        .await?
33        .text()
34        .await?
35        .trim()
36        .to_string();
37    Ok(result)
38}
39
40/// Copies a local file to a remote instance via rsync with retries
41pub async fn rsync_file(
42    key_file: &str,
43    local_path: &str,
44    ip: &str,
45    remote_path: &str,
46) -> Result<(), Error> {
47    for _ in 0..MAX_SSH_ATTEMPTS {
48        let output = Command::new("rsync")
49            .arg("-az")
50            .arg("-e")
51            .arg(format!(
52                "ssh -i {key_file} -o IdentitiesOnly=yes -o ServerAliveInterval=600 -o StrictHostKeyChecking=no"
53            ))
54            .arg(local_path)
55            .arg(format!("ubuntu@{ip}:{remote_path}"))
56            .output()
57            .await?;
58        if output.status.success() {
59            return Ok(());
60        }
61        warn!(error = ?String::from_utf8_lossy(&output.stderr), "SCP failed");
62        sleep(RETRY_INTERVAL).await;
63    }
64    Err(Error::ScpFailed)
65}
66
67/// Executes a command on a remote instance via SSH with retries
68pub async fn ssh_execute(key_file: &str, ip: &str, command: &str) -> Result<(), Error> {
69    for _ in 0..MAX_SSH_ATTEMPTS {
70        let output = Command::new("ssh")
71            .arg("-i")
72            .arg(key_file)
73            .arg("-o")
74            .arg("IdentitiesOnly=yes")
75            .arg("-o")
76            .arg("ServerAliveInterval=600")
77            .arg("-o")
78            .arg("StrictHostKeyChecking=no")
79            .arg(format!("ubuntu@{ip}"))
80            .arg(command)
81            .output()
82            .await?;
83        if output.status.success() {
84            return Ok(());
85        }
86        warn!(error = ?String::from_utf8_lossy(&output.stderr), "SSH failed");
87        sleep(RETRY_INTERVAL).await;
88    }
89    Err(Error::SshFailed)
90}
91
92/// Polls the status of a systemd service on a remote instance until active
93pub async fn poll_service_active(key_file: &str, ip: &str, service: &str) -> Result<(), Error> {
94    for _ in 0..MAX_POLL_ATTEMPTS {
95        let output = Command::new("ssh")
96            .arg("-i")
97            .arg(key_file)
98            .arg("-o")
99            .arg("IdentitiesOnly=yes")
100            .arg("-o")
101            .arg("ServerAliveInterval=600")
102            .arg("-o")
103            .arg("StrictHostKeyChecking=no")
104            .arg(format!("ubuntu@{ip}"))
105            .arg(format!("systemctl is-active {service}"))
106            .output()
107            .await?;
108        let parsed = String::from_utf8_lossy(&output.stdout);
109        let parsed = parsed.trim();
110        if parsed == "active" {
111            return Ok(());
112        }
113        if service == "binary" && parsed == "failed" {
114            warn!(service, "service failed to start (check logs and update)");
115            return Ok(());
116        }
117        warn!(error = ?String::from_utf8_lossy(&output.stderr), service, "active status check failed");
118        sleep(RETRY_INTERVAL).await;
119    }
120    Err(Error::ServiceTimeout(ip.to_string(), service.to_string()))
121}
122
123/// Polls the status of a systemd service on a remote instance until it becomes inactive
124pub async fn poll_service_inactive(key_file: &str, ip: &str, service: &str) -> Result<(), Error> {
125    for _ in 0..MAX_POLL_ATTEMPTS {
126        let output = Command::new("ssh")
127            .arg("-i")
128            .arg(key_file)
129            .arg("-o")
130            .arg("IdentitiesOnly=yes")
131            .arg("-o")
132            .arg("ServerAliveInterval=600")
133            .arg("-o")
134            .arg("StrictHostKeyChecking=no")
135            .arg(format!("ubuntu@{ip}"))
136            .arg(format!("systemctl is-active {service}"))
137            .output()
138            .await?;
139        let parsed = String::from_utf8_lossy(&output.stdout);
140        let parsed = parsed.trim();
141        if parsed == "inactive" {
142            return Ok(());
143        }
144        if service == "binary" && parsed == "failed" {
145            warn!(service, "service was never active");
146            return Ok(());
147        }
148        warn!(error = ?String::from_utf8_lossy(&output.stderr), service, "inactive status check failed");
149        sleep(RETRY_INTERVAL).await;
150    }
151    Err(Error::ServiceTimeout(ip.to_string(), service.to_string()))
152}
153
154/// Enables BBR on a remote instance by copying and applying sysctl settings.
155pub async fn enable_bbr(key_file: &str, ip: &str, bbr_conf_local_path: &str) -> Result<(), Error> {
156    rsync_file(
157        key_file,
158        bbr_conf_local_path,
159        ip,
160        "/home/ubuntu/99-bbr.conf",
161    )
162    .await?;
163    ssh_execute(
164        key_file,
165        ip,
166        "sudo mv /home/ubuntu/99-bbr.conf /etc/sysctl.d/99-bbr.conf",
167    )
168    .await?;
169    ssh_execute(key_file, ip, "sudo sysctl -p /etc/sysctl.d/99-bbr.conf").await?;
170    Ok(())
171}
172
173/// Converts an IP address to a CIDR block
174pub fn exact_cidr(ip: &str) -> String {
175    format!("{ip}/32")
176}
commonware_deployer/ec2/utils.rs

commonware_deployer/ec2/
utils.rs
