commonware_deployer/ec2/
create.rs

1//! `create` subcommand for `ec2`
2
3use crate::ec2::{
4    aws::*, deployer_directory, services::*, utils::*, Config, Error, Host, Hosts, InstanceConfig,
5    CREATED_FILE_NAME, LOGS_PORT, MONITORING_NAME, MONITORING_REGION, PROFILES_PORT, TRACES_PORT,
6};
7use futures::future::try_join_all;
8use std::{
9    collections::{BTreeSet, HashMap, HashSet},
10    fs::File,
11    net::IpAddr,
12    path::PathBuf,
13};
14use tokio::process::Command;
15use tracing::info;
16
17/// Represents a deployed instance with its configuration and public IP
18#[derive(Clone)]
19pub struct Deployment {
20    pub instance: InstanceConfig,
21    pub id: String,
22    pub ip: String,
23}
24
25/// Represents AWS resources created in a specific region
26pub struct RegionResources {
27    pub vpc_id: String,
28    pub vpc_cidr: String,
29    pub route_table_id: String,
30    pub subnet_id: String,
31    pub binary_sg_id: Option<String>,
32    pub monitoring_sg_id: Option<String>,
33}
34
35/// Sets up EC2 instances, deploys files, and configures monitoring and logging
36pub async fn create(config: &PathBuf) -> Result<(), Error> {
37    // Load configuration from YAML file
38    let config: Config = {
39        let config_file = File::open(config)?;
40        serde_yaml::from_reader(config_file)?
41    };
42    let tag = &config.tag;
43    info!(tag = tag.as_str(), "loaded configuration");
44
45    // Create a temporary directory for local files
46    let tag_directory = deployer_directory(tag);
47    if tag_directory.exists() {
48        return Err(Error::CreationAttempted);
49    }
50    std::fs::create_dir_all(&tag_directory)?;
51    info!(path = ?tag_directory, "created tag directory");
52
53    // Ensure no instance is duplicated or named MONITORING_NAME
54    let mut instance_names = HashSet::new();
55    for instance in &config.instances {
56        if instance_names.contains(&instance.name) || instance.name == MONITORING_NAME {
57            return Err(Error::InvalidInstanceName(instance.name.clone()));
58        }
59        instance_names.insert(instance.name.clone());
60    }
61
62    // Get public IP address of the deployer
63    let deployer_ip = get_public_ip().await?;
64    info!(ip = deployer_ip.as_str(), "recovered public IP");
65
66    // Generate SSH key pair
67    let key_name = format!("deployer-{tag}");
68    let private_key_path = tag_directory.join(format!("id_rsa_{tag}"));
69    let public_key_path = tag_directory.join(format!("id_rsa_{tag}.pub"));
70    let output = Command::new("ssh-keygen")
71        .arg("-t")
72        .arg("rsa")
73        .arg("-b")
74        .arg("4096")
75        .arg("-f")
76        .arg(private_key_path.to_str().unwrap())
77        .arg("-N")
78        .arg("")
79        .output()
80        .await?;
81    if !output.status.success() {
82        return Err(Error::KeygenFailed);
83    }
84    let public_key = std::fs::read_to_string(&public_key_path)?;
85    let private_key = private_key_path.to_str().unwrap();
86
87    // Determine unique regions
88    let mut regions: BTreeSet<String> = config.instances.iter().map(|i| i.region.clone()).collect();
89    regions.insert(MONITORING_REGION.to_string());
90
91    // Determine instance types by region
92    let mut instance_types_by_region: HashMap<String, HashSet<String>> = HashMap::new();
93    for instance in &config.instances {
94        instance_types_by_region
95            .entry(instance.region.clone())
96            .or_default()
97            .insert(instance.instance_type.clone());
98    }
99    instance_types_by_region
100        .entry(MONITORING_REGION.to_string())
101        .or_default()
102        .insert(config.monitoring.instance_type.clone());
103
104    // Initialize resources for each region
105    info!(?regions, "initializing resources");
106    let mut vpc_cidrs = HashMap::new();
107    let mut subnet_cidrs = HashMap::new();
108    let mut ec2_clients = HashMap::new();
109    let mut region_resources = HashMap::new();
110    for (idx, region) in regions.iter().enumerate() {
111        // Create client for region
112        let ec2_client = create_ec2_client(Region::new(region.clone())).await;
113        ec2_clients.insert(region.clone(), ec2_client);
114        info!(region = region.as_str(), "created EC2 client");
115
116        // Assert all instance types are ARM-based
117        let instance_types: Vec<String> =
118            instance_types_by_region[region].iter().cloned().collect();
119        assert_arm64_support(&ec2_clients[region], &instance_types).await?;
120
121        // Find availability zone that supports all instance types
122        let az = find_availability_zone(&ec2_clients[region], &instance_types).await?;
123        info!(
124            az = az.as_str(),
125            region = region.as_str(),
126            "selected availability zone"
127        );
128
129        // Create VPC, IGW, route table, subnet, security groups, and key pair
130        let vpc_cidr = format!("10.{idx}.0.0/16");
131        vpc_cidrs.insert(region.clone(), vpc_cidr.clone());
132        let vpc_id = create_vpc(&ec2_clients[region], &vpc_cidr, tag).await?;
133        info!(
134            vpc = vpc_id.as_str(),
135            region = region.as_str(),
136            "created VPC"
137        );
138        let igw_id = create_and_attach_igw(&ec2_clients[region], &vpc_id, tag).await?;
139        info!(
140            igw = igw_id.as_str(),
141            vpc = vpc_id.as_str(),
142            region = region.as_str(),
143            "created and attached IGW"
144        );
145        let route_table_id =
146            create_route_table(&ec2_clients[region], &vpc_id, &igw_id, tag).await?;
147        info!(
148            route_table = route_table_id.as_str(),
149            vpc = vpc_id.as_str(),
150            region = region.as_str(),
151            "created route table"
152        );
153        let subnet_cidr = format!("10.{idx}.1.0/24");
154        subnet_cidrs.insert(region.clone(), subnet_cidr.clone());
155        let subnet_id = create_subnet(
156            &ec2_clients[region],
157            &vpc_id,
158            &route_table_id,
159            &subnet_cidr,
160            &az,
161            tag,
162        )
163        .await?;
164        info!(
165            subnet = subnet_id.as_str(),
166            vpc = vpc_id.as_str(),
167            region = region.as_str(),
168            "created subnet"
169        );
170
171        // Create monitoring security group in monitoring region
172        let monitoring_sg_id = if *region == MONITORING_REGION {
173            let sg_id =
174                create_security_group_monitoring(&ec2_clients[region], &vpc_id, &deployer_ip, tag)
175                    .await?;
176            info!(
177                sg = sg_id.as_str(),
178                vpc = vpc_id.as_str(),
179                region = region.as_str(),
180                "created monitoring security group"
181            );
182            Some(sg_id)
183        } else {
184            None
185        };
186
187        // Import key pair
188        import_key_pair(&ec2_clients[region], &key_name, &public_key).await?;
189        info!(
190            key = key_name.as_str(),
191            region = region.as_str(),
192            "imported key pair"
193        );
194
195        // Store resources for region
196        info!(
197            vpc = vpc_id.as_str(),
198            subnet = subnet_id.as_str(),
199            subnet_cidr = subnet_cidr.as_str(),
200            region = region.as_str(),
201            "initialized resources"
202        );
203        region_resources.insert(
204            region.clone(),
205            RegionResources {
206                vpc_id,
207                vpc_cidr: vpc_cidr.clone(),
208                route_table_id,
209                subnet_id,
210                binary_sg_id: None,
211                monitoring_sg_id,
212            },
213        );
214    }
215    info!(?regions, "initialized resources");
216
217    // Setup VPC peering connections
218    info!("initializing VPC peering connections");
219    let monitoring_region = MONITORING_REGION.to_string();
220    let monitoring_resources = region_resources.get(&monitoring_region).unwrap();
221    let monitoring_vpc_id = &monitoring_resources.vpc_id;
222    let monitoring_cidr = &monitoring_resources.vpc_cidr;
223    let binary_regions: HashSet<String> =
224        config.instances.iter().map(|i| i.region.clone()).collect();
225    for region in &regions {
226        if region != &monitoring_region && binary_regions.contains(region) {
227            let binary_resources = region_resources.get(region).unwrap();
228            let binary_vpc_id = &binary_resources.vpc_id;
229            let binary_cidr = &binary_resources.vpc_cidr;
230            let peer_id = create_vpc_peering_connection(
231                &ec2_clients[&monitoring_region],
232                monitoring_vpc_id,
233                binary_vpc_id,
234                region,
235                tag,
236            )
237            .await?;
238            info!(
239                peer = peer_id.as_str(),
240                monitoring = monitoring_vpc_id.as_str(),
241                binary = binary_vpc_id.as_str(),
242                region = region.as_str(),
243                "created VPC peering connection"
244            );
245            wait_for_vpc_peering_connection(&ec2_clients[region], &peer_id).await?;
246            info!(
247                peer = peer_id.as_str(),
248                region = region.as_str(),
249                "VPC peering connection is available"
250            );
251            accept_vpc_peering_connection(&ec2_clients[region], &peer_id).await?;
252            info!(
253                peer = peer_id.as_str(),
254                region = region.as_str(),
255                "accepted VPC peering connection"
256            );
257            add_route(
258                &ec2_clients[&monitoring_region],
259                &monitoring_resources.route_table_id,
260                binary_cidr,
261                &peer_id,
262            )
263            .await?;
264            add_route(
265                &ec2_clients[region],
266                &binary_resources.route_table_id,
267                monitoring_cidr,
268                &peer_id,
269            )
270            .await?;
271            info!(
272                peer = peer_id.as_str(),
273                monitoring = monitoring_vpc_id.as_str(),
274                binary = binary_vpc_id.as_str(),
275                region = region.as_str(),
276                "added routes for VPC peering connection"
277            );
278        }
279    }
280    info!("initialized VPC peering connections");
281
282    // Launch monitoring instance
283    info!("launching monitoring instance");
284    let monitoring_instance_id;
285    let monitoring_ip;
286    let monitoring_private_ip;
287    let monitoring_sg_id;
288    {
289        let monitoring_ec2_client = &ec2_clients[&monitoring_region];
290        let ami_id = find_latest_ami(monitoring_ec2_client).await?;
291        let monitoring_instance_type = InstanceType::try_parse(&config.monitoring.instance_type)
292            .expect("Invalid instance type");
293        let monitoring_storage_class =
294            VolumeType::try_parse(&config.monitoring.storage_class).expect("Invalid storage class");
295        monitoring_sg_id = monitoring_resources
296            .monitoring_sg_id
297            .as_ref()
298            .unwrap()
299            .clone();
300        monitoring_instance_id = launch_instances(
301            monitoring_ec2_client,
302            &ami_id,
303            monitoring_instance_type,
304            config.monitoring.storage_size,
305            monitoring_storage_class,
306            &key_name,
307            &monitoring_resources.subnet_id,
308            &monitoring_sg_id,
309            1,
310            MONITORING_NAME,
311            tag,
312        )
313        .await?[0]
314            .clone();
315        monitoring_ip =
316            wait_for_instances_running(monitoring_ec2_client, &[monitoring_instance_id.clone()])
317                .await?[0]
318                .clone();
319        monitoring_private_ip =
320            get_private_ip(monitoring_ec2_client, &monitoring_instance_id).await?;
321    }
322    info!(ip = monitoring_ip.as_str(), "launched monitoring instance");
323
324    // Create binary security groups
325    info!("creating security groups");
326    for (region, resources) in region_resources.iter_mut() {
327        let binary_sg_id = create_security_group_binary(
328            &ec2_clients[region],
329            &resources.vpc_id,
330            &deployer_ip,
331            &monitoring_ip,
332            tag,
333            &config.ports,
334        )
335        .await?;
336        info!(
337            sg = binary_sg_id.as_str(),
338            vpc = resources.vpc_id.as_str(),
339            region = region.as_str(),
340            "created binary security group"
341        );
342        resources.binary_sg_id = Some(binary_sg_id);
343    }
344    info!("created security groups");
345
346    // Launch binary instances
347    info!("launching binary instances");
348    let mut launch_futures = Vec::new();
349    for instance in &config.instances {
350        let key_name = key_name.clone();
351        let region = instance.region.clone();
352        let resources = region_resources.get(&region).unwrap();
353        let ec2_client = ec2_clients.get(&region).unwrap();
354        let ami_id = find_latest_ami(ec2_client).await?;
355        let instance_type =
356            InstanceType::try_parse(&instance.instance_type).expect("Invalid instance type");
357        let storage_class =
358            VolumeType::try_parse(&instance.storage_class).expect("Invalid storage class");
359        let binary_sg_id = resources.binary_sg_id.as_ref().unwrap();
360        let tag = tag.clone();
361        let future = async move {
362            let instance_id = launch_instances(
363                ec2_client,
364                &ami_id,
365                instance_type,
366                instance.storage_size,
367                storage_class,
368                &key_name,
369                &resources.subnet_id,
370                binary_sg_id,
371                1,
372                &instance.name,
373                &tag,
374            )
375            .await?[0]
376                .clone();
377            let ip =
378                wait_for_instances_running(ec2_client, &[instance_id.clone()]).await?[0].clone();
379            info!(
380                ip = ip.as_str(),
381                instance = instance.name.as_str(),
382                "launched instance"
383            );
384            Ok::<Deployment, Error>(Deployment {
385                instance: instance.clone(),
386                id: instance_id,
387                ip,
388            })
389        };
390        launch_futures.push(future);
391    }
392    let deployments: Vec<Deployment> = try_join_all(launch_futures).await?;
393    info!("launched binary instances");
394
395    // Write systemd service files
396    let prometheus_service_path = tag_directory.join("prometheus.service");
397    std::fs::write(&prometheus_service_path, PROMETHEUS_SERVICE)?;
398    let loki_service_path = tag_directory.join("loki.service");
399    std::fs::write(&loki_service_path, LOKI_SERVICE)?;
400    let pyroscope_service_path = tag_directory.join("pyroscope.service");
401    std::fs::write(&pyroscope_service_path, PYROSCOPE_SERVICE)?;
402    let tempo_service_path = tag_directory.join("tempo.service");
403    std::fs::write(&tempo_service_path, TEMPO_SERVICE)?;
404    let promtail_service_path = tag_directory.join("promtail.service");
405    std::fs::write(&promtail_service_path, PROMTAIL_SERVICE)?;
406    let node_exporter_service_path = tag_directory.join("node_exporter.service");
407    std::fs::write(&node_exporter_service_path, NODE_EXPORTER_SERVICE)?;
408    let pyroscope_agent_service_path = tag_directory.join("pyroscope-agent.service");
409    std::fs::write(&pyroscope_agent_service_path, PYROSCOPE_AGENT_SERVICE)?;
410    let pyroscope_agent_timer_path = tag_directory.join("pyroscope-agent.timer");
411    std::fs::write(&pyroscope_agent_timer_path, PYROSCOPE_AGENT_TIMER)?;
412    let binary_service_path = tag_directory.join("binary.service");
413    std::fs::write(&binary_service_path, BINARY_SERVICE)?;
414    let memleak_agent_service_path = tag_directory.join("memleak-agent.service");
415    std::fs::write(&memleak_agent_service_path, MEMLEAK_AGENT_SERVICE)?;
416    let memleak_agent_script_path = tag_directory.join("memleak-agent.sh");
417    std::fs::write(&memleak_agent_script_path, MEMLEAK_AGENT_SCRIPT)?;
418
419    // Write logrotate configuration file
420    let logrotate_conf_path = tag_directory.join("logrotate.conf");
421    std::fs::write(&logrotate_conf_path, LOGROTATE_CONF)?;
422
423    // Add BBR configuration file
424    let bbr_conf_path = tag_directory.join("99-bbr.conf");
425    std::fs::write(&bbr_conf_path, BBR_CONF)?;
426
427    // Configure monitoring instance
428    info!("configuring monitoring instance");
429    wait_for_instances_ready(&ec2_clients[&monitoring_region], &[monitoring_instance_id]).await?;
430    let instances: Vec<(&str, &str, &str)> = deployments
431        .iter()
432        .map(|d| {
433            (
434                d.instance.name.as_str(),
435                d.ip.as_str(),
436                d.instance.region.as_str(),
437            )
438        })
439        .collect();
440    let prom_config = generate_prometheus_config(&instances);
441    let prom_path = tag_directory.join("prometheus.yml");
442    std::fs::write(&prom_path, prom_config)?;
443    let datasources_path = tag_directory.join("datasources.yml");
444    std::fs::write(&datasources_path, DATASOURCES_YML)?;
445    let all_yaml_path = tag_directory.join("all.yml");
446    std::fs::write(&all_yaml_path, ALL_YML)?;
447    let loki_config_path = tag_directory.join("loki.yml");
448    std::fs::write(&loki_config_path, LOKI_CONFIG)?;
449    let pyroscope_config_path = tag_directory.join("pyroscope.yml");
450    std::fs::write(&pyroscope_config_path, PYROSCOPE_CONFIG)?;
451    let tempo_yml_path = tag_directory.join("tempo.yml");
452    std::fs::write(&tempo_yml_path, TEMPO_CONFIG)?;
453    rsync_file(
454        private_key,
455        prom_path.to_str().unwrap(),
456        &monitoring_ip,
457        "/home/ubuntu/prometheus.yml",
458    )
459    .await?;
460    rsync_file(
461        private_key,
462        datasources_path.to_str().unwrap(),
463        &monitoring_ip,
464        "/home/ubuntu/datasources.yml",
465    )
466    .await?;
467    rsync_file(
468        private_key,
469        all_yaml_path.to_str().unwrap(),
470        &monitoring_ip,
471        "/home/ubuntu/all.yml",
472    )
473    .await?;
474    rsync_file(
475        private_key,
476        &config.monitoring.dashboard,
477        &monitoring_ip,
478        "/home/ubuntu/dashboard.json",
479    )
480    .await?;
481    rsync_file(
482        private_key,
483        prometheus_service_path.to_str().unwrap(),
484        &monitoring_ip,
485        "/home/ubuntu/prometheus.service",
486    )
487    .await?;
488    rsync_file(
489        private_key,
490        loki_config_path.to_str().unwrap(),
491        &monitoring_ip,
492        "/home/ubuntu/loki.yml",
493    )
494    .await?;
495    rsync_file(
496        private_key,
497        loki_service_path.to_str().unwrap(),
498        &monitoring_ip,
499        "/home/ubuntu/loki.service",
500    )
501    .await?;
502    rsync_file(
503        private_key,
504        node_exporter_service_path.to_str().unwrap(),
505        &monitoring_ip,
506        "/home/ubuntu/node_exporter.service",
507    )
508    .await?;
509    rsync_file(
510        private_key,
511        pyroscope_config_path.to_str().unwrap(),
512        &monitoring_ip,
513        "/home/ubuntu/pyroscope.yml",
514    )
515    .await?;
516    rsync_file(
517        private_key,
518        pyroscope_service_path.to_str().unwrap(),
519        &monitoring_ip,
520        "/home/ubuntu/pyroscope.service",
521    )
522    .await?;
523    rsync_file(
524        private_key,
525        tempo_yml_path.to_str().unwrap(),
526        &monitoring_ip,
527        "/home/ubuntu/tempo.yml",
528    )
529    .await?;
530    rsync_file(
531        private_key,
532        tempo_service_path.to_str().unwrap(),
533        &monitoring_ip,
534        "/home/ubuntu/tempo.service",
535    )
536    .await?;
537    enable_bbr(private_key, &monitoring_ip, bbr_conf_path.to_str().unwrap()).await?;
538    ssh_execute(
539        private_key,
540        &monitoring_ip,
541        &setup_node_exporter_cmd(NODE_EXPORTER_VERSION),
542    )
543    .await?;
544    poll_service_active(private_key, &monitoring_ip, "node_exporter").await?;
545    ssh_execute(
546        private_key,
547        &monitoring_ip,
548        &install_monitoring_cmd(
549            PROMETHEUS_VERSION,
550            GRAFANA_VERSION,
551            LOKI_VERSION,
552            PYROSCOPE_VERSION,
553            TEMPO_VERSION,
554        ),
555    )
556    .await?;
557    poll_service_active(private_key, &monitoring_ip, "prometheus").await?;
558    poll_service_active(private_key, &monitoring_ip, "loki").await?;
559    poll_service_active(private_key, &monitoring_ip, "pyroscope").await?;
560    poll_service_active(private_key, &monitoring_ip, "tempo").await?;
561    poll_service_active(private_key, &monitoring_ip, "grafana-server").await?;
562    info!("configured monitoring instance");
563
564    // Generate hosts.yaml
565    let hosts = Hosts {
566        monitoring: monitoring_private_ip.clone().parse::<IpAddr>().unwrap(),
567        hosts: deployments
568            .iter()
569            .map(|d| Host {
570                name: d.instance.name.clone(),
571                region: d.instance.region.clone(),
572                ip: d.ip.clone().parse::<IpAddr>().unwrap(),
573            })
574            .collect(),
575    };
576    let hosts_yaml = serde_yaml::to_string(&hosts)?;
577    let hosts_path = tag_directory.join("hosts.yaml");
578    std::fs::write(&hosts_path, hosts_yaml)?;
579
580    // Configure binary instances
581    info!("configuring binary instances");
582    let mut start_futures = Vec::new();
583    for deployment in &deployments {
584        let tag_directory = tag_directory.clone();
585        let instance = deployment.instance.clone();
586        wait_for_instances_ready(&ec2_clients[&instance.region], &[deployment.id.clone()]).await?;
587        let ip = deployment.ip.clone();
588        let monitoring_private_ip = monitoring_private_ip.clone();
589        let hosts_path = hosts_path.clone();
590        let logrotate_conf_path = logrotate_conf_path.clone();
591        let bbr_conf_path = bbr_conf_path.clone();
592        let promtail_service_path = promtail_service_path.clone();
593        let node_exporter_service_path = node_exporter_service_path.clone();
594        let binary_service_path = binary_service_path.clone();
595        let pyroscope_agent_service_path = pyroscope_agent_service_path.clone();
596        let pyroscope_agent_timer_path = pyroscope_agent_timer_path.clone();
597        let memleak_agent_service_path = memleak_agent_service_path.clone();
598        let memleak_agent_script_path = memleak_agent_script_path.clone();
599        let future = async move {
600            rsync_file(private_key, &instance.binary, &ip, "/home/ubuntu/binary").await?;
601            rsync_file(
602                private_key,
603                &instance.config,
604                &ip,
605                "/home/ubuntu/config.conf",
606            )
607            .await?;
608            rsync_file(
609                private_key,
610                hosts_path.to_str().unwrap(),
611                &ip,
612                "/home/ubuntu/hosts.yaml",
613            )
614            .await?;
615            let promtail_config_path =
616                tag_directory.join(format!("promtail_{}.yml", instance.name));
617            std::fs::write(
618                &promtail_config_path,
619                promtail_config(
620                    &monitoring_private_ip,
621                    &instance.name,
622                    ip.as_str(),
623                    instance.region.as_str(),
624                ),
625            )?;
626            rsync_file(
627                private_key,
628                promtail_config_path.to_str().unwrap(),
629                &ip,
630                "/home/ubuntu/promtail.yml",
631            )
632            .await?;
633            rsync_file(
634                private_key,
635                promtail_service_path.to_str().unwrap(),
636                &ip,
637                "/home/ubuntu/promtail.service",
638            )
639            .await?;
640            rsync_file(
641                private_key,
642                node_exporter_service_path.to_str().unwrap(),
643                &ip,
644                "/home/ubuntu/node_exporter.service",
645            )
646            .await?;
647            rsync_file(
648                private_key,
649                binary_service_path.to_str().unwrap(),
650                &ip,
651                "/home/ubuntu/binary.service",
652            )
653            .await?;
654            rsync_file(
655                private_key,
656                logrotate_conf_path.to_str().unwrap(),
657                &ip,
658                "/home/ubuntu/logrotate.conf",
659            )
660            .await?;
661            rsync_file(
662                private_key,
663                pyroscope_agent_service_path.to_str().unwrap(),
664                &ip,
665                "/home/ubuntu/pyroscope-agent.service",
666            )
667            .await?;
668            let pyroscope_agent_script_path =
669                tag_directory.join(format!("pyroscope-agent_{}.sh", instance.name));
670            std::fs::write(
671                &pyroscope_agent_script_path,
672                generate_pyroscope_script(
673                    &monitoring_private_ip,
674                    &instance.name,
675                    &ip,
676                    &instance.region,
677                ),
678            )?;
679            rsync_file(
680                private_key,
681                pyroscope_agent_script_path.to_str().unwrap(),
682                &ip,
683                "/home/ubuntu/pyroscope-agent.sh",
684            )
685            .await?;
686            rsync_file(
687                private_key,
688                pyroscope_agent_timer_path.to_str().unwrap(),
689                &ip,
690                "/home/ubuntu/pyroscope-agent.timer",
691            )
692            .await?;
693            rsync_file(
694                private_key,
695                memleak_agent_service_path.to_str().unwrap(),
696                &ip,
697                "/home/ubuntu/memleak-agent.service",
698            )
699            .await?;
700            rsync_file(
701                private_key,
702                memleak_agent_script_path.to_str().unwrap(),
703                &ip,
704                "/home/ubuntu/memleak-agent.sh",
705            )
706            .await?;
707            enable_bbr(private_key, &ip, bbr_conf_path.to_str().unwrap()).await?;
708            ssh_execute(private_key, &ip, &setup_promtail_cmd(PROMTAIL_VERSION)).await?;
709            poll_service_active(private_key, &ip, "promtail").await?;
710            ssh_execute(
711                private_key,
712                &ip,
713                &setup_node_exporter_cmd(NODE_EXPORTER_VERSION),
714            )
715            .await?;
716            poll_service_active(private_key, &ip, "node_exporter").await?;
717            ssh_execute(private_key, &ip, &install_binary_cmd(instance.profiling)).await?;
718            poll_service_active(private_key, &ip, "binary").await?;
719            info!(
720                ip = ip.as_str(),
721                instance = instance.name.as_str(),
722                "configured instance"
723            );
724            Ok::<String, Error>(ip)
725        };
726        start_futures.push(future);
727    }
728    let all_binary_ips = try_join_all(start_futures).await?;
729    info!("configured binary instances");
730
731    // Update monitoring security group to restrict Loki port (3100)
732    info!("updating monitoring security group to allow traffic from binary instances");
733    let monitoring_ec2_client = &ec2_clients[&monitoring_region];
734    if binary_regions.contains(&monitoring_region) {
735        let binary_sg_id = region_resources[&monitoring_region]
736            .binary_sg_id
737            .clone()
738            .unwrap();
739        monitoring_ec2_client
740            .authorize_security_group_ingress()
741            .group_id(&monitoring_sg_id)
742            .ip_permissions(
743                IpPermission::builder()
744                    .ip_protocol("tcp")
745                    .from_port(LOGS_PORT as i32)
746                    .to_port(LOGS_PORT as i32)
747                    .user_id_group_pairs(
748                        UserIdGroupPair::builder()
749                            .group_id(binary_sg_id.clone())
750                            .build(),
751                    )
752                    .build(),
753            )
754            .ip_permissions(
755                IpPermission::builder()
756                    .ip_protocol("tcp")
757                    .from_port(PROFILES_PORT as i32)
758                    .to_port(PROFILES_PORT as i32)
759                    .user_id_group_pairs(
760                        UserIdGroupPair::builder()
761                            .group_id(binary_sg_id.clone())
762                            .build(),
763                    )
764                    .build(),
765            )
766            .ip_permissions(
767                IpPermission::builder()
768                    .ip_protocol("tcp")
769                    .from_port(TRACES_PORT as i32)
770                    .to_port(TRACES_PORT as i32)
771                    .user_id_group_pairs(
772                        UserIdGroupPair::builder()
773                            .group_id(binary_sg_id.clone())
774                            .build(),
775                    )
776                    .build(),
777            )
778            .send()
779            .await
780            .map_err(|err| err.into_service_error())?;
781        info!(
782            monitoring = monitoring_sg_id.as_str(),
783            binary = binary_sg_id.as_str(),
784            region = monitoring_region.as_str(),
785            "linked monitoring and binary security groups in monitoring region"
786        );
787    }
788    for region in &regions {
789        if region != &monitoring_region && binary_regions.contains(region) {
790            let binary_cidr = &region_resources[region].vpc_cidr;
791            monitoring_ec2_client
792                .authorize_security_group_ingress()
793                .group_id(&monitoring_sg_id)
794                .ip_permissions(
795                    IpPermission::builder()
796                        .ip_protocol("tcp")
797                        .from_port(LOGS_PORT as i32)
798                        .to_port(LOGS_PORT as i32)
799                        .ip_ranges(IpRange::builder().cidr_ip(binary_cidr).build())
800                        .build(),
801                )
802                .ip_permissions(
803                    IpPermission::builder()
804                        .ip_protocol("tcp")
805                        .from_port(PROFILES_PORT as i32)
806                        .to_port(PROFILES_PORT as i32)
807                        .ip_ranges(IpRange::builder().cidr_ip(binary_cidr).build())
808                        .build(),
809                )
810                .ip_permissions(
811                    IpPermission::builder()
812                        .ip_protocol("tcp")
813                        .from_port(TRACES_PORT as i32)
814                        .to_port(TRACES_PORT as i32)
815                        .ip_ranges(IpRange::builder().cidr_ip(binary_cidr).build())
816                        .build(),
817                )
818                .send()
819                .await
820                .map_err(|err| err.into_service_error())?;
821            info!(
822                monitoring = monitoring_sg_id.as_str(),
823                binary = binary_cidr.as_str(),
824                region = region.as_str(),
825                "opened monitoring part to traffic from binary VPC"
826            );
827        }
828    }
829    info!("updated monitoring security group");
830
831    // Mark deployment as complete
832    File::create(tag_directory.join(CREATED_FILE_NAME))?;
833    info!(
834        monitoring = monitoring_ip.as_str(),
835        binary = ?all_binary_ips,
836        "deployment complete"
837    );
838    Ok(())
839}
commonware_deployer/ec2/create.rs

commonware_deployer/ec2/
create.rs
