sn_testnet_deploy/ansible/
provisioning.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use super::{
8    extra_vars::ExtraVarsDocBuilder,
9    inventory::{
10        generate_full_cone_private_node_static_environment_inventory,
11        generate_symmetric_private_node_static_environment_inventory,
12    },
13    AnsibleInventoryType, AnsiblePlaybook, AnsibleRunner,
14};
15use crate::{
16    ansible::inventory::{
17        generate_custom_environment_inventory,
18        generate_full_cone_nat_gateway_static_environment_inventory,
19    },
20    bootstrap::BootstrapOptions,
21    clients::ClientsDeployOptions,
22    deploy::DeployOptions,
23    error::{Error, Result},
24    funding::FundingOptions,
25    inventory::{DeploymentNodeRegistries, VirtualMachine},
26    print_duration, run_external_command, BinaryOption, CloudProvider, EvmNetwork, LogFormat,
27    NodeType, SshClient, UpgradeOptions,
28};
29use ant_service_management::NodeRegistry;
30use evmlib::common::U256;
31use log::{debug, error, trace};
32use semver::Version;
33use serde::{Deserialize, Serialize};
34use std::{
35    collections::HashMap,
36    net::IpAddr,
37    path::PathBuf,
38    time::{Duration, Instant},
39};
40use walkdir::WalkDir;
41
42use crate::ansible::extra_vars;
43
44pub const DEFAULT_BETA_ENCRYPTION_KEY: &str =
45    "49113d2083f57a976076adbe85decb75115820de1e6e74b47e0429338cef124a";
46
47#[derive(Clone, Serialize, Deserialize)]
48pub struct ProvisionOptions {
49    /// The safe version is also in the binary option, but only for an initial deployment.
50    /// For the upscale, it needs to be provided explicitly, because currently it is not
51    /// recorded in the inventory.
52    pub ant_version: Option<String>,
53    pub binary_option: BinaryOption,
54    pub chunk_size: Option<u64>,
55    pub client_env_variables: Option<Vec<(String, String)>>,
56    pub delayed_verifier_batch_size: Option<u16>,
57    pub delayed_verifier_quorum_value: Option<String>,
58    pub enable_delayed_verifier: bool,
59    pub enable_random_verifier: bool,
60    pub enable_performance_verifier: bool,
61    pub enable_telegraf: bool,
62    pub enable_uploaders: bool,
63    pub evm_data_payments_address: Option<String>,
64    pub evm_network: EvmNetwork,
65    pub evm_payment_token_address: Option<String>,
66    pub evm_rpc_url: Option<String>,
67    pub expected_hash: Option<String>,
68    pub expected_size: Option<u64>,
69    pub file_address: Option<String>,
70    pub full_cone_private_node_count: u16,
71    pub funding_wallet_secret_key: Option<String>,
72    pub gas_amount: Option<U256>,
73    pub interval: Option<Duration>,
74    pub log_format: Option<LogFormat>,
75    pub max_archived_log_files: u16,
76    pub max_log_files: u16,
77    pub max_uploads: Option<u32>,
78    pub name: String,
79    pub network_id: Option<u8>,
80    pub network_dashboard_branch: Option<String>,
81    pub node_count: u16,
82    pub node_env_variables: Option<Vec<(String, String)>>,
83    pub output_inventory_dir_path: PathBuf,
84    pub peer_cache_node_count: u16,
85    pub performance_verifier_batch_size: Option<u16>,
86    pub public_rpc: bool,
87    pub random_verifier_batch_size: Option<u16>,
88    pub rewards_address: Option<String>,
89    pub sleep_duration: Option<u16>,
90    pub symmetric_private_node_count: u16,
91    pub token_amount: Option<U256>,
92    pub upload_size: Option<u16>,
93    pub upload_interval: Option<u16>,
94    pub uploaders_count: Option<u16>,
95    pub wallet_secret_keys: Option<Vec<String>>,
96}
97
98/// These are obtained by running the inventory playbook
99#[derive(Clone, Debug)]
100pub struct PrivateNodeProvisionInventory {
101    pub full_cone_nat_gateway_vms: Vec<VirtualMachine>,
102    pub full_cone_private_node_vms: Vec<VirtualMachine>,
103    pub symmetric_nat_gateway_vms: Vec<VirtualMachine>,
104    pub symmetric_private_node_vms: Vec<VirtualMachine>,
105}
106
107impl PrivateNodeProvisionInventory {
108    pub fn new(
109        provisioner: &AnsibleProvisioner,
110        full_cone_private_node_vm_count: Option<u16>,
111        symmetric_private_node_vm_count: Option<u16>,
112    ) -> Result<Self> {
113        // All the environment types set private_node_vm count to >0 if not specified.
114        let should_provision_full_cone_private_nodes = full_cone_private_node_vm_count
115            .map(|count| count > 0)
116            .unwrap_or(true);
117        let should_provision_symmetric_private_nodes = symmetric_private_node_vm_count
118            .map(|count| count > 0)
119            .unwrap_or(true);
120
121        let mut inventory = Self {
122            full_cone_nat_gateway_vms: Default::default(),
123            full_cone_private_node_vms: Default::default(),
124            symmetric_nat_gateway_vms: Default::default(),
125            symmetric_private_node_vms: Default::default(),
126        };
127
128        if should_provision_full_cone_private_nodes {
129            let full_cone_private_node_vms = provisioner
130                .ansible_runner
131                .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)
132                .inspect_err(|err| {
133                    println!("Failed to obtain the inventory of Full Cone private node: {err:?}");
134                })?;
135
136            let full_cone_nat_gateway_inventory = provisioner
137                .ansible_runner
138                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)
139                .inspect_err(|err| {
140                    println!("Failed to get Full Cone NAT Gateway inventory {err:?}");
141                })?;
142
143            if full_cone_nat_gateway_inventory.len() != full_cone_private_node_vms.len() {
144                println!("The number of Full Cone private nodes does not match the number of Full Cone NAT Gateway VMs");
145                return Err(Error::VmCountMismatch(
146                    Some(AnsibleInventoryType::FullConePrivateNodes),
147                    Some(AnsibleInventoryType::FullConeNatGateway),
148                ));
149            }
150
151            inventory.full_cone_private_node_vms = full_cone_private_node_vms;
152            inventory.full_cone_nat_gateway_vms = full_cone_nat_gateway_inventory;
153        }
154
155        if should_provision_symmetric_private_nodes {
156            let symmetric_private_node_vms = provisioner
157                .ansible_runner
158                .get_inventory(AnsibleInventoryType::SymmetricPrivateNodes, true)
159                .inspect_err(|err| {
160                    println!("Failed to obtain the inventory of Symmetric private node: {err:?}");
161                })?;
162
163            let symmetric_nat_gateway_inventory = provisioner
164                .ansible_runner
165                .get_inventory(AnsibleInventoryType::SymmetricNatGateway, true)
166                .inspect_err(|err| {
167                    println!("Failed to get Symmetric NAT Gateway inventory {err:?}");
168                })?;
169
170            if symmetric_nat_gateway_inventory.len() != symmetric_private_node_vms.len() {
171                println!("The number of Symmetric private nodes does not match the number of Symmetric NAT Gateway VMs");
172                return Err(Error::VmCountMismatch(
173                    Some(AnsibleInventoryType::SymmetricPrivateNodes),
174                    Some(AnsibleInventoryType::SymmetricNatGateway),
175                ));
176            }
177
178            inventory.symmetric_private_node_vms = symmetric_private_node_vms;
179            inventory.symmetric_nat_gateway_vms = symmetric_nat_gateway_inventory;
180        }
181
182        Ok(inventory)
183    }
184
185    pub fn should_provision_full_cone_private_nodes(&self) -> bool {
186        !self.full_cone_private_node_vms.is_empty()
187    }
188
189    pub fn should_provision_symmetric_private_nodes(&self) -> bool {
190        !self.symmetric_private_node_vms.is_empty()
191    }
192
193    pub fn symmetric_private_node_and_gateway_map(
194        &self,
195    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
196        Self::match_private_node_vm_and_gateway_vm(
197            &self.symmetric_private_node_vms,
198            &self.symmetric_nat_gateway_vms,
199        )
200    }
201
202    pub fn full_cone_private_node_and_gateway_map(
203        &self,
204    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
205        Self::match_private_node_vm_and_gateway_vm(
206            &self.full_cone_private_node_vms,
207            &self.full_cone_nat_gateway_vms,
208        )
209    }
210
211    pub fn match_private_node_vm_and_gateway_vm(
212        private_node_vms: &[VirtualMachine],
213        nat_gateway_vms: &[VirtualMachine],
214    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
215        if private_node_vms.len() != nat_gateway_vms.len() {
216            println!(
217            "The number of private node VMs ({}) does not match the number of NAT Gateway VMs ({})",
218            private_node_vms.len(),
219            nat_gateway_vms.len()
220        );
221            error!("The number of private node VMs does not match the number of NAT Gateway VMs: Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}");
222            return Err(Error::VmCountMismatch(None, None));
223        }
224
225        let mut map = HashMap::new();
226        for private_vm in private_node_vms {
227            let nat_gateway = nat_gateway_vms
228                .iter()
229                .find(|vm| {
230                    let private_node_name = private_vm.name.split('-').next_back().unwrap();
231                    let nat_gateway_name = vm.name.split('-').next_back().unwrap();
232                    private_node_name == nat_gateway_name
233                })
234                .ok_or_else(|| {
235                    println!(
236                        "Failed to find a matching NAT Gateway for private node: {}",
237                        private_vm.name
238                    );
239                    error!("Failed to find a matching NAT Gateway for private node: {}. Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}", private_vm.name);
240                    Error::VmCountMismatch(None, None)
241                })?;
242
243            let _ = map.insert(private_vm.clone(), nat_gateway.clone());
244        }
245
246        Ok(map)
247    }
248}
249
250impl From<BootstrapOptions> for ProvisionOptions {
251    fn from(bootstrap_options: BootstrapOptions) -> Self {
252        ProvisionOptions {
253            ant_version: None,
254            binary_option: bootstrap_options.binary_option,
255            chunk_size: bootstrap_options.chunk_size,
256            client_env_variables: None,
257            delayed_verifier_batch_size: None,
258            delayed_verifier_quorum_value: None,
259            enable_delayed_verifier: false,
260            enable_random_verifier: false,
261            enable_performance_verifier: false,
262            enable_telegraf: true,
263            enable_uploaders: false,
264            evm_data_payments_address: bootstrap_options.evm_data_payments_address,
265            evm_network: bootstrap_options.evm_network,
266            evm_payment_token_address: bootstrap_options.evm_payment_token_address,
267            evm_rpc_url: bootstrap_options.evm_rpc_url,
268            expected_hash: None,
269            expected_size: None,
270            file_address: None,
271            full_cone_private_node_count: bootstrap_options.full_cone_private_node_count,
272            funding_wallet_secret_key: None,
273            gas_amount: None,
274            interval: Some(bootstrap_options.interval),
275            log_format: bootstrap_options.log_format,
276            max_archived_log_files: bootstrap_options.max_archived_log_files,
277            max_log_files: bootstrap_options.max_log_files,
278            max_uploads: None,
279            name: bootstrap_options.name,
280            network_id: Some(bootstrap_options.network_id),
281            network_dashboard_branch: None,
282            node_count: bootstrap_options.node_count,
283            node_env_variables: bootstrap_options.node_env_variables,
284            output_inventory_dir_path: bootstrap_options.output_inventory_dir_path,
285            peer_cache_node_count: 0,
286            performance_verifier_batch_size: None,
287            public_rpc: false,
288            random_verifier_batch_size: None,
289            rewards_address: Some(bootstrap_options.rewards_address),
290            sleep_duration: None,
291            symmetric_private_node_count: bootstrap_options.symmetric_private_node_count,
292            token_amount: None,
293            upload_size: None,
294            upload_interval: None,
295            uploaders_count: None,
296            wallet_secret_keys: None,
297        }
298    }
299}
300
301impl From<DeployOptions> for ProvisionOptions {
302    fn from(deploy_options: DeployOptions) -> Self {
303        ProvisionOptions {
304            ant_version: None,
305            binary_option: deploy_options.binary_option,
306            chunk_size: deploy_options.chunk_size,
307            client_env_variables: deploy_options.client_env_variables,
308            delayed_verifier_batch_size: None,
309            delayed_verifier_quorum_value: None,
310            enable_delayed_verifier: deploy_options.enable_delayed_verifier,
311            enable_performance_verifier: deploy_options.enable_performance_verifier,
312            enable_random_verifier: deploy_options.enable_random_verifier,
313            enable_telegraf: deploy_options.enable_telegraf,
314            enable_uploaders: true,
315            node_env_variables: deploy_options.node_env_variables,
316            evm_data_payments_address: deploy_options.evm_data_payments_address,
317            evm_network: deploy_options.evm_network,
318            evm_payment_token_address: deploy_options.evm_payment_token_address,
319            evm_rpc_url: deploy_options.evm_rpc_url,
320            expected_hash: None,
321            expected_size: None,
322            file_address: None,
323            full_cone_private_node_count: deploy_options.full_cone_private_node_count,
324            funding_wallet_secret_key: deploy_options.funding_wallet_secret_key,
325            gas_amount: deploy_options.initial_gas,
326            interval: Some(deploy_options.interval),
327            log_format: deploy_options.log_format,
328            max_archived_log_files: deploy_options.max_archived_log_files,
329            max_log_files: deploy_options.max_log_files,
330            max_uploads: None,
331            name: deploy_options.name,
332            network_id: Some(deploy_options.network_id),
333            network_dashboard_branch: deploy_options.network_dashboard_branch,
334            node_count: deploy_options.node_count,
335            output_inventory_dir_path: deploy_options.output_inventory_dir_path,
336            peer_cache_node_count: deploy_options.peer_cache_node_count,
337            performance_verifier_batch_size: None,
338            public_rpc: deploy_options.public_rpc,
339            random_verifier_batch_size: None,
340            rewards_address: Some(deploy_options.rewards_address),
341            sleep_duration: None,
342            symmetric_private_node_count: deploy_options.symmetric_private_node_count,
343            token_amount: deploy_options.initial_tokens,
344            upload_size: None,
345            upload_interval: Some(deploy_options.upload_interval),
346            uploaders_count: Some(deploy_options.uploaders_count),
347            wallet_secret_keys: None,
348        }
349    }
350}
351
352impl From<ClientsDeployOptions> for ProvisionOptions {
353    fn from(client_options: ClientsDeployOptions) -> Self {
354        Self {
355            ant_version: None,
356            binary_option: client_options.binary_option,
357            chunk_size: client_options.chunk_size,
358            client_env_variables: client_options.client_env_variables,
359            delayed_verifier_batch_size: client_options.delayed_verifier_batch_size,
360            delayed_verifier_quorum_value: client_options.delayed_verifier_quorum_value,
361            enable_delayed_verifier: client_options.enable_delayed_verifier,
362            enable_random_verifier: client_options.enable_random_verifier,
363            enable_performance_verifier: client_options.enable_performance_verifier,
364            enable_telegraf: client_options.enable_telegraf,
365            enable_uploaders: client_options.enable_uploaders,
366            evm_data_payments_address: client_options.evm_details.data_payments_address,
367            evm_network: client_options.evm_details.network,
368            evm_payment_token_address: client_options.evm_details.payment_token_address,
369            evm_rpc_url: client_options.evm_details.rpc_url,
370            expected_hash: client_options.expected_hash,
371            expected_size: client_options.expected_size,
372            file_address: client_options.file_address,
373            full_cone_private_node_count: 0,
374            funding_wallet_secret_key: client_options.funding_wallet_secret_key,
375            gas_amount: client_options.initial_gas,
376            interval: None,
377            log_format: None,
378            max_archived_log_files: client_options.max_archived_log_files,
379            max_log_files: client_options.max_log_files,
380            max_uploads: client_options.max_uploads,
381            name: client_options.name,
382            network_id: client_options.network_id,
383            network_dashboard_branch: None,
384            node_count: 0,
385            node_env_variables: None,
386            output_inventory_dir_path: client_options.output_inventory_dir_path,
387            peer_cache_node_count: 0,
388            performance_verifier_batch_size: client_options.performance_verifier_batch_size,
389            public_rpc: false,
390            random_verifier_batch_size: client_options.random_verifier_batch_size,
391            rewards_address: None,
392            sleep_duration: client_options.sleep_duration,
393            symmetric_private_node_count: 0,
394            token_amount: client_options.initial_tokens,
395            upload_size: client_options.upload_size,
396            upload_interval: None,
397            uploaders_count: Some(client_options.uploaders_count),
398            wallet_secret_keys: client_options.wallet_secret_keys,
399        }
400    }
401}
402
403#[derive(Clone)]
404pub struct AnsibleProvisioner {
405    pub ansible_runner: AnsibleRunner,
406    pub cloud_provider: CloudProvider,
407    pub ssh_client: SshClient,
408}
409
410impl AnsibleProvisioner {
411    pub fn new(
412        ansible_runner: AnsibleRunner,
413        cloud_provider: CloudProvider,
414        ssh_client: SshClient,
415    ) -> Self {
416        Self {
417            ansible_runner,
418            cloud_provider,
419            ssh_client,
420        }
421    }
422
423    pub fn build_autonomi_binaries(
424        &self,
425        options: &ProvisionOptions,
426        binaries_to_build: Option<Vec<String>>,
427    ) -> Result<()> {
428        let start = Instant::now();
429        println!("Obtaining IP address for build VM...");
430        let build_inventory = self
431            .ansible_runner
432            .get_inventory(AnsibleInventoryType::Build, true)?;
433        let build_ip = build_inventory[0].public_ip_addr;
434        self.ssh_client
435            .wait_for_ssh_availability(&build_ip, &self.cloud_provider.get_ssh_user())?;
436
437        println!("Running ansible against build VM...");
438        let base_extra_vars = extra_vars::build_binaries_extra_vars_doc(options)?;
439
440        let extra_vars = if let Some(binaries) = binaries_to_build {
441            let mut build_ant = false;
442            let mut build_antnode = false;
443            let mut build_antctl = false;
444            let mut build_antctld = false;
445
446            for binary in &binaries {
447                match binary.as_str() {
448                    "ant" => build_ant = true,
449                    "antnode" => build_antnode = true,
450                    "antctl" => build_antctl = true,
451                    "antctld" => build_antctld = true,
452                    _ => return Err(Error::InvalidBinaryName(binary.clone())),
453                }
454            }
455
456            let mut json_value: serde_json::Value = serde_json::from_str(&base_extra_vars)?;
457            if let serde_json::Value::Object(ref mut map) = json_value {
458                map.insert("build_ant".to_string(), serde_json::Value::Bool(build_ant));
459                map.insert(
460                    "build_antnode".to_string(),
461                    serde_json::Value::Bool(build_antnode),
462                );
463                map.insert(
464                    "build_antctl".to_string(),
465                    serde_json::Value::Bool(build_antctl),
466                );
467                map.insert(
468                    "build_antctld".to_string(),
469                    serde_json::Value::Bool(build_antctld),
470                );
471            }
472            json_value.to_string()
473        } else {
474            base_extra_vars
475        };
476
477        self.ansible_runner.run_playbook(
478            AnsiblePlaybook::Build,
479            AnsibleInventoryType::Build,
480            Some(extra_vars),
481        )?;
482        print_duration(start.elapsed());
483        Ok(())
484    }
485
486    pub fn cleanup_node_logs(&self, setup_cron: bool) -> Result<()> {
487        for node_inv_type in AnsibleInventoryType::iter_node_type() {
488            self.ansible_runner.run_playbook(
489                AnsiblePlaybook::CleanupLogs,
490                node_inv_type,
491                Some(format!("{{ \"setup_cron\": \"{setup_cron}\" }}")),
492            )?;
493        }
494
495        Ok(())
496    }
497
498    pub fn copy_logs(&self, name: &str, resources_only: bool) -> Result<()> {
499        for node_inv_type in AnsibleInventoryType::iter_node_type() {
500            self.ansible_runner.run_playbook(
501                AnsiblePlaybook::CopyLogs,
502                node_inv_type,
503                Some(format!(
504                    "{{ \"env_name\": \"{name}\", \"resources_only\" : \"{resources_only}\" }}"
505                )),
506            )?;
507        }
508        Ok(())
509    }
510
511    pub fn get_all_node_inventory(&self) -> Result<Vec<VirtualMachine>> {
512        let mut all_node_inventory = Vec::new();
513        for node_inv_type in AnsibleInventoryType::iter_node_type() {
514            all_node_inventory.extend(self.ansible_runner.get_inventory(node_inv_type, false)?);
515        }
516
517        Ok(all_node_inventory)
518    }
519
520    pub fn get_symmetric_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
521        self.ansible_runner
522            .get_inventory(AnsibleInventoryType::SymmetricNatGateway, false)
523    }
524
525    pub fn get_full_cone_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
526        self.ansible_runner
527            .get_inventory(AnsibleInventoryType::FullConeNatGateway, false)
528    }
529
530    pub fn get_client_inventory(&self) -> Result<Vec<VirtualMachine>> {
531        self.ansible_runner
532            .get_inventory(AnsibleInventoryType::Clients, false)
533    }
534
535    pub fn get_node_registries(
536        &self,
537        inventory_type: &AnsibleInventoryType,
538    ) -> Result<DeploymentNodeRegistries> {
539        debug!("Fetching node manager inventory for {inventory_type:?}");
540        let temp_dir_path = tempfile::tempdir()?.into_path();
541        let temp_dir_json = serde_json::to_string(&temp_dir_path)?;
542
543        self.ansible_runner.run_playbook(
544            AnsiblePlaybook::AntCtlInventory,
545            *inventory_type,
546            Some(format!("{{ \"dest\": {temp_dir_json} }}")),
547        )?;
548
549        let node_registry_paths = WalkDir::new(temp_dir_path)
550            .into_iter()
551            .flatten()
552            .filter_map(|entry| {
553                if entry.file_type().is_file()
554                    && entry.path().extension().is_some_and(|ext| ext == "json")
555                {
556                    // tempdir/<testnet_name>-node/var/safenode-manager/node_registry.json
557                    let mut vm_name = entry.path().to_path_buf();
558                    trace!("Found file with json extension: {vm_name:?}");
559                    vm_name.pop();
560                    vm_name.pop();
561                    vm_name.pop();
562                    // Extract the <testnet_name>-node string
563                    trace!("Extracting the vm name from the path");
564                    let vm_name = vm_name.file_name()?.to_str()?;
565                    trace!("Extracted vm name from path: {vm_name}");
566                    Some((vm_name.to_string(), entry.path().to_path_buf()))
567                } else {
568                    None
569                }
570            })
571            .collect::<Vec<(String, PathBuf)>>();
572
573        let mut node_registries = Vec::new();
574        let mut failed_vms = Vec::new();
575        for (vm_name, file_path) in node_registry_paths {
576            match NodeRegistry::load(&file_path) {
577                Ok(node_registry) => node_registries.push((vm_name.clone(), node_registry)),
578                Err(_) => failed_vms.push(vm_name.clone()),
579            }
580        }
581
582        let deployment_registries = DeploymentNodeRegistries {
583            inventory_type: *inventory_type,
584            retrieved_registries: node_registries,
585            failed_vms,
586        };
587        Ok(deployment_registries)
588    }
589
590    pub fn provision_evm_nodes(&self, options: &ProvisionOptions) -> Result<()> {
591        let start = Instant::now();
592        println!("Obtaining IP address for EVM nodes...");
593        let evm_node_inventory = self
594            .ansible_runner
595            .get_inventory(AnsibleInventoryType::EvmNodes, true)?;
596        let evm_node_ip = evm_node_inventory[0].public_ip_addr;
597        self.ssh_client
598            .wait_for_ssh_availability(&evm_node_ip, &self.cloud_provider.get_ssh_user())?;
599
600        println!("Running ansible against EVM nodes...");
601        self.ansible_runner.run_playbook(
602            AnsiblePlaybook::EvmNodes,
603            AnsibleInventoryType::EvmNodes,
604            Some(extra_vars::build_evm_nodes_extra_vars_doc(
605                &options.name,
606                &self.cloud_provider,
607                &options.binary_option,
608            )),
609        )?;
610        print_duration(start.elapsed());
611        Ok(())
612    }
613
614    pub fn provision_genesis_node(&self, options: &ProvisionOptions) -> Result<()> {
615        let start = Instant::now();
616        let genesis_inventory = self
617            .ansible_runner
618            .get_inventory(AnsibleInventoryType::Genesis, true)?;
619        let genesis_ip = genesis_inventory[0].public_ip_addr;
620        self.ssh_client
621            .wait_for_ssh_availability(&genesis_ip, &self.cloud_provider.get_ssh_user())?;
622        self.ansible_runner.run_playbook(
623            AnsiblePlaybook::Genesis,
624            AnsibleInventoryType::Genesis,
625            Some(extra_vars::build_node_extra_vars_doc(
626                &self.cloud_provider.to_string(),
627                options,
628                NodeType::Genesis,
629                None,
630                None,
631                1,
632                options.evm_network.clone(),
633                false,
634            )?),
635        )?;
636
637        print_duration(start.elapsed());
638
639        Ok(())
640    }
641
642    pub fn provision_full_cone(
643        &self,
644        options: &ProvisionOptions,
645        initial_contact_peer: Option<String>,
646        initial_network_contacts_url: Option<String>,
647        private_node_inventory: PrivateNodeProvisionInventory,
648        new_full_cone_nat_gateway_new_vms_for_upscale: Option<Vec<VirtualMachine>>,
649    ) -> Result<()> {
650        // Step 1 of Full Cone NAT Gateway
651        let start = Instant::now();
652        self.print_ansible_run_banner("Provision Full Cone NAT Gateway - Step 1");
653
654        for vm in new_full_cone_nat_gateway_new_vms_for_upscale
655            .as_ref()
656            .unwrap_or(&private_node_inventory.full_cone_nat_gateway_vms)
657            .iter()
658        {
659            println!(
660                "Checking SSH availability for Full Cone NAT Gateway: {}",
661                vm.public_ip_addr
662            );
663            self.ssh_client
664                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
665                .map_err(|e| {
666                    println!("Failed to establish SSH connection to Full Cone NAT Gateway: {e}");
667                    e
668                })?;
669        }
670
671        let mut modified_private_node_inventory = private_node_inventory.clone();
672
673        // If we are upscaling, then we cannot access the gateway VMs which are already deployed.
674        if let Some(new_full_cone_nat_gateway_new_vms_for_upscale) =
675            &new_full_cone_nat_gateway_new_vms_for_upscale
676        {
677            debug!("Removing existing full cone NAT Gateway and private node VMs from the inventory. Old inventory: {modified_private_node_inventory:?}");
678            let mut names_to_keep = Vec::new();
679
680            for vm in new_full_cone_nat_gateway_new_vms_for_upscale.iter() {
681                let nat_gateway_name = vm.name.split('-').next_back().unwrap();
682                names_to_keep.push(nat_gateway_name);
683            }
684
685            modified_private_node_inventory
686                .full_cone_nat_gateway_vms
687                .retain(|vm| {
688                    let nat_gateway_name = vm.name.split('-').next_back().unwrap();
689                    names_to_keep.contains(&nat_gateway_name)
690                });
691            modified_private_node_inventory
692                .full_cone_private_node_vms
693                .retain(|vm| {
694                    let nat_gateway_name = vm.name.split('-').next_back().unwrap();
695                    names_to_keep.contains(&nat_gateway_name)
696                });
697            debug!("New inventory after removing existing full cone NAT Gateway and private node VMs: {modified_private_node_inventory:?}");
698        }
699
700        if modified_private_node_inventory
701            .full_cone_nat_gateway_vms
702            .is_empty()
703        {
704            error!("There are no full cone NAT Gateway VMs available to upscale");
705            return Ok(());
706        }
707
708        let private_node_ip_map = modified_private_node_inventory
709            .full_cone_private_node_and_gateway_map()?
710            .into_iter()
711            .map(|(k, v)| {
712                let gateway_name = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
713                    debug!("Upscaling, using public IP address for gateway name");
714                    v.public_ip_addr.to_string()
715                } else {
716                    v.name.clone()
717                };
718                (gateway_name, k.private_ip_addr)
719            })
720            .collect::<HashMap<String, IpAddr>>();
721
722        if private_node_ip_map.is_empty() {
723            println!("There are no full cone private node VM available to be routed through the full cone NAT Gateway");
724            return Err(Error::EmptyInventory(
725                AnsibleInventoryType::FullConePrivateNodes,
726            ));
727        }
728
729        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
730            &options.name,
731            private_node_ip_map.clone(),
732            "step1",
733        );
734        debug!("Provisioning Full Cone NAT Gateway - Step 1 with vars: {vars}");
735        let gateway_inventory = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
736            debug!("Upscaling, using static inventory for full cone nat gateway.");
737            generate_full_cone_nat_gateway_static_environment_inventory(
738                &modified_private_node_inventory.full_cone_nat_gateway_vms,
739                &options.name,
740                &options.output_inventory_dir_path,
741            )?;
742
743            AnsibleInventoryType::FullConeNatGatewayStatic
744        } else {
745            AnsibleInventoryType::FullConeNatGateway
746        };
747        self.ansible_runner.run_playbook(
748            AnsiblePlaybook::StaticFullConeNatGateway,
749            gateway_inventory,
750            Some(vars),
751        )?;
752
753        // setup private node config
754        self.print_ansible_run_banner("Provisioning Full Cone Private Node Config");
755
756        generate_full_cone_private_node_static_environment_inventory(
757            &options.name,
758            &options.output_inventory_dir_path,
759            &private_node_inventory.full_cone_private_node_vms,
760            &private_node_inventory.full_cone_nat_gateway_vms,
761            &self.ssh_client.private_key_path,
762        )
763        .inspect_err(|err| {
764            error!("Failed to generate full cone private node static inv with err: {err:?}")
765        })?;
766
767        // For a new deployment, it's quite probable that SSH is available, because this part occurs
768        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
769        // check that SSH is available before proceeding.
770        println!("Obtaining IP addresses for nodes...");
771        let inventory = self
772            .ansible_runner
773            .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)?;
774
775        println!("Waiting for SSH availability on Symmetric Private nodes...");
776        for vm in inventory.iter() {
777            println!(
778                "Checking SSH availability for {}: {}",
779                vm.name, vm.public_ip_addr
780            );
781            self.ssh_client
782                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
783                .map_err(|e| {
784                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
785                    e
786                })?;
787        }
788
789        println!("SSH is available on all nodes. Proceeding with provisioning...");
790
791        self.ansible_runner.run_playbook(
792            AnsiblePlaybook::PrivateNodeConfig,
793            AnsibleInventoryType::FullConePrivateNodes,
794            Some(
795                extra_vars::build_full_cone_private_node_config_extra_vars_docs(
796                    &private_node_inventory,
797                )?,
798            ),
799        )?;
800
801        // Step 2 of Full Cone NAT Gateway
802
803        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
804            &options.name,
805            private_node_ip_map,
806            "step2",
807        );
808
809        self.print_ansible_run_banner("Provisioning Full Cone NAT Gateway - Step 2");
810        debug!("Provisioning Full Cone NAT Gateway - Step 2 with vars: {vars}");
811        self.ansible_runner.run_playbook(
812            AnsiblePlaybook::StaticFullConeNatGateway,
813            gateway_inventory,
814            Some(vars),
815        )?;
816
817        // provision the nodes
818
819        let home_dir = std::env::var("HOME").inspect_err(|err| {
820            println!("Failed to get home directory with error: {err:?}",);
821        })?;
822        let known_hosts_path = format!("{home_dir}/.ssh/known_hosts");
823        debug!("Cleaning up known hosts file at {known_hosts_path} ");
824        run_external_command(
825            PathBuf::from("rm"),
826            std::env::current_dir()?,
827            vec![known_hosts_path],
828            false,
829            false,
830        )?;
831
832        self.print_ansible_run_banner("Provision Full Cone Private Nodes");
833
834        self.ssh_client.set_full_cone_nat_routed_vms(
835            &private_node_inventory.full_cone_private_node_vms,
836            &private_node_inventory.full_cone_nat_gateway_vms,
837        )?;
838
839        self.provision_nodes(
840            options,
841            initial_contact_peer,
842            initial_network_contacts_url,
843            NodeType::FullConePrivateNode,
844        )?;
845
846        print_duration(start.elapsed());
847        Ok(())
848    }
849    pub fn provision_symmetric_nat_gateway(
850        &self,
851        options: &ProvisionOptions,
852        private_node_inventory: &PrivateNodeProvisionInventory,
853    ) -> Result<()> {
854        let start = Instant::now();
855        for vm in &private_node_inventory.symmetric_nat_gateway_vms {
856            println!(
857                "Checking SSH availability for Symmetric NAT Gateway: {}",
858                vm.public_ip_addr
859            );
860            self.ssh_client
861                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
862                .map_err(|e| {
863                    println!("Failed to establish SSH connection to Symmetric NAT Gateway: {e}");
864                    e
865                })?;
866        }
867
868        let private_node_ip_map = private_node_inventory
869            .symmetric_private_node_and_gateway_map()?
870            .into_iter()
871            .map(|(k, v)| (v.name.clone(), k.private_ip_addr))
872            .collect::<HashMap<String, IpAddr>>();
873
874        if private_node_ip_map.is_empty() {
875            println!("There are no Symmetric private node VM available to be routed through the Symmetric NAT Gateway");
876            return Err(Error::EmptyInventory(
877                AnsibleInventoryType::SymmetricPrivateNodes,
878            ));
879        }
880
881        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
882            &options.name,
883            private_node_ip_map,
884            "symmetric",
885        );
886        debug!("Provisioning Symmetric NAT Gateway with vars: {vars}");
887        self.ansible_runner.run_playbook(
888            AnsiblePlaybook::SymmetricNatGateway,
889            AnsibleInventoryType::SymmetricNatGateway,
890            Some(vars),
891        )?;
892
893        print_duration(start.elapsed());
894        Ok(())
895    }
896
897    pub fn provision_nodes(
898        &self,
899        options: &ProvisionOptions,
900        initial_contact_peer: Option<String>,
901        initial_network_contacts_url: Option<String>,
902        node_type: NodeType,
903    ) -> Result<()> {
904        let start = Instant::now();
905        let mut write_older_cache_files = false;
906        let (inventory_type, node_count) = match &node_type {
907            NodeType::FullConePrivateNode => (
908                node_type.to_ansible_inventory_type(),
909                options.full_cone_private_node_count,
910            ),
911            // use provision_genesis_node fn
912            NodeType::Generic => (node_type.to_ansible_inventory_type(), options.node_count),
913            NodeType::Genesis => return Err(Error::InvalidNodeType(node_type)),
914            NodeType::PeerCache => {
915                write_older_cache_files = true;
916                (
917                    node_type.to_ansible_inventory_type(),
918                    options.peer_cache_node_count,
919                )
920            }
921            NodeType::SymmetricPrivateNode => (
922                node_type.to_ansible_inventory_type(),
923                options.symmetric_private_node_count,
924            ),
925        };
926
927        // For a new deployment, it's quite probable that SSH is available, because this part occurs
928        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
929        // check that SSH is available before proceeding.
930        println!("Obtaining IP addresses for nodes...");
931        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
932
933        println!("Waiting for SSH availability on {node_type:?} nodes...");
934        for vm in inventory.iter() {
935            println!(
936                "Checking SSH availability for {}: {}",
937                vm.name, vm.public_ip_addr
938            );
939            self.ssh_client
940                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
941                .map_err(|e| {
942                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
943                    e
944                })?;
945        }
946
947        println!("SSH is available on all nodes. Proceeding with provisioning...");
948
949        let playbook = match node_type {
950            NodeType::Generic => AnsiblePlaybook::Nodes,
951            NodeType::PeerCache => AnsiblePlaybook::PeerCacheNodes,
952            NodeType::FullConePrivateNode => AnsiblePlaybook::Nodes,
953            NodeType::SymmetricPrivateNode => AnsiblePlaybook::Nodes,
954            _ => return Err(Error::InvalidNodeType(node_type.clone())),
955        };
956        self.ansible_runner.run_playbook(
957            playbook,
958            inventory_type,
959            Some(extra_vars::build_node_extra_vars_doc(
960                &self.cloud_provider.to_string(),
961                options,
962                node_type.clone(),
963                initial_contact_peer,
964                initial_network_contacts_url,
965                node_count,
966                options.evm_network.clone(),
967                write_older_cache_files,
968            )?),
969        )?;
970
971        print_duration(start.elapsed());
972        Ok(())
973    }
974
975    pub fn provision_symmetric_private_nodes(
976        &self,
977        options: &mut ProvisionOptions,
978        initial_contact_peer: Option<String>,
979        initial_network_contacts_url: Option<String>,
980        private_node_inventory: &PrivateNodeProvisionInventory,
981    ) -> Result<()> {
982        let start = Instant::now();
983        self.print_ansible_run_banner("Provision Symmetric Private Node Config");
984
985        generate_symmetric_private_node_static_environment_inventory(
986            &options.name,
987            &options.output_inventory_dir_path,
988            &private_node_inventory.symmetric_private_node_vms,
989            &private_node_inventory.symmetric_nat_gateway_vms,
990            &self.ssh_client.private_key_path,
991        )
992        .inspect_err(|err| {
993            error!("Failed to generate symmetric private node static inv with err: {err:?}")
994        })?;
995
996        self.ssh_client.set_symmetric_nat_routed_vms(
997            &private_node_inventory.symmetric_private_node_vms,
998            &private_node_inventory.symmetric_nat_gateway_vms,
999        )?;
1000
1001        let inventory_type = AnsibleInventoryType::SymmetricPrivateNodes;
1002
1003        // For a new deployment, it's quite probable that SSH is available, because this part occurs
1004        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
1005        // check that SSH is available before proceeding.
1006        println!("Obtaining IP addresses for nodes...");
1007        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
1008
1009        println!("Waiting for SSH availability on Symmetric Private nodes...");
1010        for vm in inventory.iter() {
1011            println!(
1012                "Checking SSH availability for {}: {}",
1013                vm.name, vm.public_ip_addr
1014            );
1015            self.ssh_client
1016                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
1017                .map_err(|e| {
1018                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
1019                    e
1020                })?;
1021        }
1022
1023        println!("SSH is available on all nodes. Proceeding with provisioning...");
1024
1025        self.ansible_runner.run_playbook(
1026            AnsiblePlaybook::PrivateNodeConfig,
1027            inventory_type,
1028            Some(
1029                extra_vars::build_symmetric_private_node_config_extra_vars_doc(
1030                    private_node_inventory,
1031                )?,
1032            ),
1033        )?;
1034
1035        println!("Provisioned Symmetric Private Node Config");
1036        print_duration(start.elapsed());
1037
1038        self.provision_nodes(
1039            options,
1040            initial_contact_peer,
1041            initial_network_contacts_url,
1042            NodeType::SymmetricPrivateNode,
1043        )?;
1044
1045        Ok(())
1046    }
1047
1048    pub async fn provision_downloaders(
1049        &self,
1050        options: &ProvisionOptions,
1051        genesis_multiaddr: Option<String>,
1052        genesis_network_contacts_url: Option<String>,
1053    ) -> Result<()> {
1054        let start = Instant::now();
1055
1056        println!("Running ansible against Client machine to start the downloader script.");
1057        debug!("Running ansible against Client machine to start the downloader script.");
1058
1059        self.ansible_runner.run_playbook(
1060            AnsiblePlaybook::Downloaders,
1061            AnsibleInventoryType::Clients,
1062            Some(extra_vars::build_downloaders_extra_vars_doc(
1063                &self.cloud_provider.to_string(),
1064                options,
1065                genesis_multiaddr,
1066                genesis_network_contacts_url,
1067            )?),
1068        )?;
1069        print_duration(start.elapsed());
1070        Ok(())
1071    }
1072
1073    pub async fn provision_static_downloaders(
1074        &self,
1075        options: &ProvisionOptions,
1076        genesis_multiaddr: Option<String>,
1077        genesis_network_contacts_url: Option<String>,
1078    ) -> Result<()> {
1079        let start = Instant::now();
1080
1081        println!("Running ansible against client machine to start the static downloaders.");
1082        debug!("Running ansible against client machine to start the static downloaders.");
1083
1084        self.ansible_runner.run_playbook(
1085            AnsiblePlaybook::StaticDownloaders,
1086            AnsibleInventoryType::Clients,
1087            Some(extra_vars::build_downloaders_extra_vars_doc(
1088                &self.cloud_provider.to_string(),
1089                options,
1090                genesis_multiaddr,
1091                genesis_network_contacts_url,
1092            )?),
1093        )?;
1094        print_duration(start.elapsed());
1095        Ok(())
1096    }
1097
1098    pub async fn provision_clients(
1099        &self,
1100        options: &ProvisionOptions,
1101        genesis_multiaddr: Option<String>,
1102        genesis_network_contacts_url: Option<String>,
1103    ) -> Result<()> {
1104        let start = Instant::now();
1105
1106        let sk_map = if let Some(wallet_keys) = &options.wallet_secret_keys {
1107            self.prepare_pre_funded_wallets(wallet_keys).await?
1108        } else {
1109            self.deposit_funds_to_clients(&FundingOptions {
1110                evm_data_payments_address: options.evm_data_payments_address.clone(),
1111                evm_network: options.evm_network.clone(),
1112                evm_payment_token_address: options.evm_payment_token_address.clone(),
1113                evm_rpc_url: options.evm_rpc_url.clone(),
1114                funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
1115                gas_amount: options.gas_amount,
1116                token_amount: options.token_amount,
1117                uploaders_count: options.uploaders_count,
1118            })
1119            .await?
1120        };
1121
1122        self.ansible_runner.run_playbook(
1123            AnsiblePlaybook::Uploaders,
1124            AnsibleInventoryType::Clients,
1125            Some(extra_vars::build_clients_extra_vars_doc(
1126                &self.cloud_provider.to_string(),
1127                options,
1128                genesis_multiaddr,
1129                genesis_network_contacts_url,
1130                &sk_map,
1131            )?),
1132        )?;
1133        print_duration(start.elapsed());
1134        Ok(())
1135    }
1136
1137    pub fn start_nodes(
1138        &self,
1139        environment_name: &str,
1140        interval: Duration,
1141        node_type: Option<NodeType>,
1142        custom_inventory: Option<Vec<VirtualMachine>>,
1143    ) -> Result<()> {
1144        let mut extra_vars = ExtraVarsDocBuilder::default();
1145        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1146
1147        if let Some(node_type) = node_type {
1148            println!("Running the start nodes playbook for {node_type:?} nodes");
1149            self.ansible_runner.run_playbook(
1150                AnsiblePlaybook::StartNodes,
1151                node_type.to_ansible_inventory_type(),
1152                Some(extra_vars.build()),
1153            )?;
1154            return Ok(());
1155        }
1156
1157        if let Some(custom_inventory) = custom_inventory {
1158            println!("Running the start nodes playbook with a custom inventory");
1159            generate_custom_environment_inventory(
1160                &custom_inventory,
1161                environment_name,
1162                &self.ansible_runner.working_directory_path.join("inventory"),
1163            )?;
1164            self.ansible_runner.run_playbook(
1165                AnsiblePlaybook::StartNodes,
1166                AnsibleInventoryType::Custom,
1167                Some(extra_vars.build()),
1168            )?;
1169            return Ok(());
1170        }
1171
1172        println!("Running the start nodes playbook for all node types");
1173        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1174            self.ansible_runner.run_playbook(
1175                AnsiblePlaybook::StartNodes,
1176                node_inv_type,
1177                Some(extra_vars.build()),
1178            )?;
1179        }
1180        Ok(())
1181    }
1182
1183    pub fn status(&self) -> Result<()> {
1184        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1185            self.ansible_runner
1186                .run_playbook(AnsiblePlaybook::Status, node_inv_type, None)?;
1187        }
1188        Ok(())
1189    }
1190
1191    pub fn start_telegraf(
1192        &self,
1193        environment_name: &str,
1194        node_type: Option<NodeType>,
1195        custom_inventory: Option<Vec<VirtualMachine>>,
1196    ) -> Result<()> {
1197        if let Some(node_type) = node_type {
1198            println!("Running the start telegraf playbook for {node_type:?} nodes");
1199            self.ansible_runner.run_playbook(
1200                AnsiblePlaybook::StartTelegraf,
1201                node_type.to_ansible_inventory_type(),
1202                None,
1203            )?;
1204            return Ok(());
1205        }
1206
1207        if let Some(custom_inventory) = custom_inventory {
1208            println!("Running the start telegraf playbook with a custom inventory");
1209            generate_custom_environment_inventory(
1210                &custom_inventory,
1211                environment_name,
1212                &self.ansible_runner.working_directory_path.join("inventory"),
1213            )?;
1214            self.ansible_runner.run_playbook(
1215                AnsiblePlaybook::StartTelegraf,
1216                AnsibleInventoryType::Custom,
1217                None,
1218            )?;
1219            return Ok(());
1220        }
1221
1222        println!("Running the start telegraf playbook for all node types");
1223        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1224            self.ansible_runner.run_playbook(
1225                AnsiblePlaybook::StartTelegraf,
1226                node_inv_type,
1227                None,
1228            )?;
1229        }
1230
1231        Ok(())
1232    }
1233
1234    pub fn stop_nodes(
1235        &self,
1236        environment_name: &str,
1237        interval: Duration,
1238        node_type: Option<NodeType>,
1239        custom_inventory: Option<Vec<VirtualMachine>>,
1240        delay: Option<u64>,
1241        service_names: Option<Vec<String>>,
1242    ) -> Result<()> {
1243        let mut extra_vars = ExtraVarsDocBuilder::default();
1244        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1245        if let Some(delay) = delay {
1246            extra_vars.add_variable("delay", &delay.to_string());
1247        }
1248        if let Some(service_names) = service_names {
1249            extra_vars.add_list_variable("service_names", service_names);
1250        }
1251        let extra_vars = extra_vars.build();
1252
1253        if let Some(node_type) = node_type {
1254            println!("Running the stop nodes playbook for {node_type:?} nodes");
1255            self.ansible_runner.run_playbook(
1256                AnsiblePlaybook::StopNodes,
1257                node_type.to_ansible_inventory_type(),
1258                Some(extra_vars),
1259            )?;
1260            return Ok(());
1261        }
1262
1263        if let Some(custom_inventory) = custom_inventory {
1264            println!("Running the stop nodes playbook with a custom inventory");
1265            generate_custom_environment_inventory(
1266                &custom_inventory,
1267                environment_name,
1268                &self.ansible_runner.working_directory_path.join("inventory"),
1269            )?;
1270            self.ansible_runner.run_playbook(
1271                AnsiblePlaybook::StopNodes,
1272                AnsibleInventoryType::Custom,
1273                Some(extra_vars),
1274            )?;
1275            return Ok(());
1276        }
1277
1278        println!("Running the stop nodes playbook for all node types");
1279        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1280            self.ansible_runner.run_playbook(
1281                AnsiblePlaybook::StopNodes,
1282                node_inv_type,
1283                Some(extra_vars.clone()),
1284            )?;
1285        }
1286
1287        Ok(())
1288    }
1289
1290    pub fn stop_telegraf(
1291        &self,
1292        environment_name: &str,
1293        node_type: Option<NodeType>,
1294        custom_inventory: Option<Vec<VirtualMachine>>,
1295    ) -> Result<()> {
1296        if let Some(node_type) = node_type {
1297            println!("Running the stop telegraf playbook for {node_type:?} nodes");
1298            self.ansible_runner.run_playbook(
1299                AnsiblePlaybook::StopTelegraf,
1300                node_type.to_ansible_inventory_type(),
1301                None,
1302            )?;
1303            return Ok(());
1304        }
1305
1306        if let Some(custom_inventory) = custom_inventory {
1307            println!("Running the stop telegraf playbook with a custom inventory");
1308            generate_custom_environment_inventory(
1309                &custom_inventory,
1310                environment_name,
1311                &self.ansible_runner.working_directory_path.join("inventory"),
1312            )?;
1313            self.ansible_runner.run_playbook(
1314                AnsiblePlaybook::StopTelegraf,
1315                AnsibleInventoryType::Custom,
1316                None,
1317            )?;
1318            return Ok(());
1319        }
1320
1321        println!("Running the stop telegraf playbook for all node types");
1322        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1323            self.ansible_runner
1324                .run_playbook(AnsiblePlaybook::StopTelegraf, node_inv_type, None)?;
1325        }
1326
1327        Ok(())
1328    }
1329
1330    pub fn upgrade_node_telegraf(&self, name: &str) -> Result<()> {
1331        self.ansible_runner.run_playbook(
1332            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1333            AnsibleInventoryType::PeerCacheNodes,
1334            Some(extra_vars::build_node_telegraf_upgrade(
1335                name,
1336                &NodeType::PeerCache,
1337            )?),
1338        )?;
1339        self.ansible_runner.run_playbook(
1340            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1341            AnsibleInventoryType::Nodes,
1342            Some(extra_vars::build_node_telegraf_upgrade(
1343                name,
1344                &NodeType::Generic,
1345            )?),
1346        )?;
1347
1348        self.ansible_runner.run_playbook(
1349            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1350            AnsibleInventoryType::SymmetricPrivateNodes,
1351            Some(extra_vars::build_node_telegraf_upgrade(
1352                name,
1353                &NodeType::SymmetricPrivateNode,
1354            )?),
1355        )?;
1356
1357        self.ansible_runner.run_playbook(
1358            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1359            AnsibleInventoryType::FullConePrivateNodes,
1360            Some(extra_vars::build_node_telegraf_upgrade(
1361                name,
1362                &NodeType::FullConePrivateNode,
1363            )?),
1364        )?;
1365        Ok(())
1366    }
1367
1368    pub fn upgrade_client_telegraf(&self, name: &str) -> Result<()> {
1369        self.ansible_runner.run_playbook(
1370            AnsiblePlaybook::UpgradeClientTelegrafConfig,
1371            AnsibleInventoryType::Clients,
1372            Some(extra_vars::build_client_telegraf_upgrade(name)?),
1373        )?;
1374        Ok(())
1375    }
1376
1377    pub fn upgrade_nodes(&self, options: &UpgradeOptions) -> Result<()> {
1378        if let Some(custom_inventory) = &options.custom_inventory {
1379            println!("Running the UpgradeNodes with a custom inventory");
1380            generate_custom_environment_inventory(
1381                custom_inventory,
1382                &options.name,
1383                &self.ansible_runner.working_directory_path.join("inventory"),
1384            )?;
1385            match self.ansible_runner.run_playbook(
1386                AnsiblePlaybook::UpgradeNodes,
1387                AnsibleInventoryType::Custom,
1388                Some(options.get_ansible_vars()),
1389            ) {
1390                Ok(()) => println!("All nodes were successfully upgraded"),
1391                Err(_) => {
1392                    println!("WARNING: some nodes may not have been upgraded or restarted");
1393                }
1394            }
1395            return Ok(());
1396        }
1397
1398        if let Some(node_type) = &options.node_type {
1399            println!("Running the UpgradeNodes playbook for {node_type:?} nodes");
1400            match self.ansible_runner.run_playbook(
1401                AnsiblePlaybook::UpgradeNodes,
1402                node_type.to_ansible_inventory_type(),
1403                Some(options.get_ansible_vars()),
1404            ) {
1405                Ok(()) => println!("All {node_type:?} nodes were successfully upgraded"),
1406                Err(_) => {
1407                    println!(
1408                        "WARNING: some {node_type:?} nodes may not have been upgraded or restarted"
1409                    );
1410                }
1411            }
1412            return Ok(());
1413        }
1414
1415        println!("Running the UpgradeNodes playbook for all node types");
1416
1417        match self.ansible_runner.run_playbook(
1418            AnsiblePlaybook::UpgradeNodes,
1419            AnsibleInventoryType::PeerCacheNodes,
1420            Some(options.get_ansible_vars()),
1421        ) {
1422            Ok(()) => println!("All Peer Cache nodes were successfully upgraded"),
1423            Err(_) => {
1424                println!("WARNING: some Peer Cacche nodes may not have been upgraded or restarted");
1425            }
1426        }
1427        match self.ansible_runner.run_playbook(
1428            AnsiblePlaybook::UpgradeNodes,
1429            AnsibleInventoryType::Nodes,
1430            Some(options.get_ansible_vars()),
1431        ) {
1432            Ok(()) => println!("All generic nodes were successfully upgraded"),
1433            Err(_) => {
1434                println!("WARNING: some nodes may not have been upgraded or restarted");
1435            }
1436        }
1437        match self.ansible_runner.run_playbook(
1438            AnsiblePlaybook::UpgradeNodes,
1439            AnsibleInventoryType::SymmetricPrivateNodes,
1440            Some(options.get_ansible_vars()),
1441        ) {
1442            Ok(()) => println!("All private nodes were successfully upgraded"),
1443            Err(_) => {
1444                println!("WARNING: some nodes may not have been upgraded or restarted");
1445            }
1446        }
1447        // Don't use AnsibleInventoryType::iter_node_type() here, because the genesis node should be upgraded last
1448        match self.ansible_runner.run_playbook(
1449            AnsiblePlaybook::UpgradeNodes,
1450            AnsibleInventoryType::Genesis,
1451            Some(options.get_ansible_vars()),
1452        ) {
1453            Ok(()) => println!("The genesis nodes was successfully upgraded"),
1454            Err(_) => {
1455                println!("WARNING: the genesis node may not have been upgraded or restarted");
1456            }
1457        }
1458        Ok(())
1459    }
1460
1461    pub fn upgrade_antctl(
1462        &self,
1463        environment_name: &str,
1464        version: &Version,
1465        node_type: Option<NodeType>,
1466        custom_inventory: Option<Vec<VirtualMachine>>,
1467    ) -> Result<()> {
1468        let mut extra_vars = ExtraVarsDocBuilder::default();
1469        extra_vars.add_variable("version", &version.to_string());
1470
1471        if let Some(node_type) = node_type {
1472            println!("Running the upgrade safenode-manager playbook for {node_type:?} nodes");
1473            self.ansible_runner.run_playbook(
1474                AnsiblePlaybook::UpgradeAntctl,
1475                node_type.to_ansible_inventory_type(),
1476                Some(extra_vars.build()),
1477            )?;
1478            return Ok(());
1479        }
1480
1481        if let Some(custom_inventory) = custom_inventory {
1482            println!("Running the upgrade safenode-manager playbook with a custom inventory");
1483            generate_custom_environment_inventory(
1484                &custom_inventory,
1485                environment_name,
1486                &self.ansible_runner.working_directory_path.join("inventory"),
1487            )?;
1488            self.ansible_runner.run_playbook(
1489                AnsiblePlaybook::UpgradeAntctl,
1490                AnsibleInventoryType::Custom,
1491                Some(extra_vars.build()),
1492            )?;
1493            return Ok(());
1494        }
1495
1496        println!("Running the upgrade safenode-manager playbook for all node types");
1497        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1498            self.ansible_runner.run_playbook(
1499                AnsiblePlaybook::UpgradeAntctl,
1500                node_inv_type,
1501                Some(extra_vars.build()),
1502            )?;
1503        }
1504
1505        Ok(())
1506    }
1507
1508    pub fn upgrade_nginx_config(
1509        &self,
1510        environment_name: &str,
1511        custom_inventory: Option<Vec<VirtualMachine>>,
1512    ) -> Result<()> {
1513        if let Some(custom_inventory) = custom_inventory {
1514            println!("Running the upgrade nginx config playbook with a custom inventory");
1515            generate_custom_environment_inventory(
1516                &custom_inventory,
1517                environment_name,
1518                &self.ansible_runner.working_directory_path.join("inventory"),
1519            )?;
1520            self.ansible_runner.run_playbook(
1521                AnsiblePlaybook::UpgradeNginx,
1522                AnsibleInventoryType::Custom,
1523                None,
1524            )?;
1525            return Ok(());
1526        }
1527
1528        println!("Running the upgrade nginx config playbook for peer cache nodes");
1529        self.ansible_runner.run_playbook(
1530            AnsiblePlaybook::UpgradeNginx,
1531            AnsibleInventoryType::PeerCacheNodes,
1532            None,
1533        )?;
1534        Ok(())
1535    }
1536
1537    pub fn upgrade_geoip_telegraf(&self, name: &str) -> Result<()> {
1538        self.ansible_runner.run_playbook(
1539            AnsiblePlaybook::UpgradeGeoIpTelegrafConfig,
1540            AnsibleInventoryType::PeerCacheNodes,
1541            Some(extra_vars::build_node_telegraf_upgrade(
1542                name,
1543                &NodeType::PeerCache,
1544            )?),
1545        )?;
1546        Ok(())
1547    }
1548
1549    pub fn print_ansible_run_banner(&self, s: &str) {
1550        let ansible_run_msg = "Ansible Run: ";
1551        let line = "=".repeat(s.len() + ansible_run_msg.len());
1552        println!("{line}\n{ansible_run_msg}{s}\n{line}");
1553    }
1554}
sn_testnet_deploy/ansible/provisioning.rs

sn_testnet_deploy/ansible/
provisioning.rs
