sn_testnet_deploy/ansible/
provisioning.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use super::{
8    extra_vars::ExtraVarsDocBuilder,
9    inventory::{
10        generate_full_cone_private_node_static_environment_inventory,
11        generate_symmetric_private_node_static_environment_inventory,
12    },
13    AnsibleInventoryType, AnsiblePlaybook, AnsibleRunner,
14};
15use crate::{
16    ansible::inventory::{
17        generate_custom_environment_inventory,
18        generate_full_cone_nat_gateway_static_environment_inventory,
19    },
20    bootstrap::BootstrapOptions,
21    clients::ClientsDeployOptions,
22    deploy::DeployOptions,
23    error::{Error, Result},
24    funding::FundingOptions,
25    inventory::{DeploymentNodeRegistries, VirtualMachine},
26    print_duration, run_external_command, BinaryOption, CloudProvider, EvmNetwork, LogFormat,
27    NodeType, SshClient, UpgradeOptions,
28};
29use ant_service_management::NodeRegistry;
30use evmlib::common::U256;
31use log::{debug, error, trace};
32use semver::Version;
33use serde::{Deserialize, Serialize};
34use std::{
35    collections::HashMap,
36    net::IpAddr,
37    path::PathBuf,
38    time::{Duration, Instant},
39};
40use walkdir::WalkDir;
41
42use crate::ansible::extra_vars;
43
44pub const DEFAULT_BETA_ENCRYPTION_KEY: &str =
45    "49113d2083f57a976076adbe85decb75115820de1e6e74b47e0429338cef124a";
46
47#[derive(Clone, Serialize, Deserialize)]
48pub struct ProvisionOptions {
49    /// The safe version is also in the binary option, but only for an initial deployment.
50    /// For the upscale, it needs to be provided explicitly, because currently it is not
51    /// recorded in the inventory.
52    pub ant_version: Option<String>,
53    pub binary_option: BinaryOption,
54    pub chunk_size: Option<u64>,
55    pub client_env_variables: Option<Vec<(String, String)>>,
56    pub delayed_verifier_batch_size: Option<u16>,
57    pub delayed_verifier_quorum_value: Option<String>,
58    pub enable_delayed_verifier: bool,
59    pub enable_random_verifier: bool,
60    pub enable_performance_verifier: bool,
61    pub enable_telegraf: bool,
62    pub enable_uploaders: bool,
63    pub evm_data_payments_address: Option<String>,
64    pub evm_network: EvmNetwork,
65    pub evm_payment_token_address: Option<String>,
66    pub evm_rpc_url: Option<String>,
67    pub expected_hash: Option<String>,
68    pub expected_size: Option<u64>,
69    pub file_address: Option<String>,
70    pub full_cone_private_node_count: u16,
71    pub funding_wallet_secret_key: Option<String>,
72    pub gas_amount: Option<U256>,
73    pub interval: Option<Duration>,
74    pub log_format: Option<LogFormat>,
75    pub max_archived_log_files: u16,
76    pub max_log_files: u16,
77    pub max_uploads: Option<u32>,
78    pub name: String,
79    pub network_id: Option<u8>,
80    pub network_dashboard_branch: Option<String>,
81    pub node_count: u16,
82    pub node_env_variables: Option<Vec<(String, String)>>,
83    pub output_inventory_dir_path: PathBuf,
84    pub peer_cache_node_count: u16,
85    pub performance_verifier_batch_size: Option<u16>,
86    pub public_rpc: bool,
87    pub random_verifier_batch_size: Option<u16>,
88    pub rewards_address: Option<String>,
89    pub sleep_duration: Option<u16>,
90    pub symmetric_private_node_count: u16,
91    pub token_amount: Option<U256>,
92    pub upload_batch_size: Option<u16>,
93    pub upload_size: Option<u16>,
94    pub upload_interval: Option<u16>,
95    pub uploaders_count: Option<u16>,
96    pub upnp_private_node_count: u16,
97    pub wallet_secret_keys: Option<Vec<String>>,
98}
99
100/// These are obtained by running the inventory playbook
101#[derive(Clone, Debug)]
102pub struct PrivateNodeProvisionInventory {
103    pub full_cone_nat_gateway_vms: Vec<VirtualMachine>,
104    pub full_cone_private_node_vms: Vec<VirtualMachine>,
105    pub symmetric_nat_gateway_vms: Vec<VirtualMachine>,
106    pub symmetric_private_node_vms: Vec<VirtualMachine>,
107}
108
109impl PrivateNodeProvisionInventory {
110    pub fn new(
111        provisioner: &AnsibleProvisioner,
112        full_cone_private_node_vm_count: Option<u16>,
113        symmetric_private_node_vm_count: Option<u16>,
114    ) -> Result<Self> {
115        // All the environment types set private_node_vm count to >0 if not specified.
116        let should_provision_full_cone_private_nodes = full_cone_private_node_vm_count
117            .map(|count| count > 0)
118            .unwrap_or(true);
119        let should_provision_symmetric_private_nodes = symmetric_private_node_vm_count
120            .map(|count| count > 0)
121            .unwrap_or(true);
122
123        let mut inventory = Self {
124            full_cone_nat_gateway_vms: Default::default(),
125            full_cone_private_node_vms: Default::default(),
126            symmetric_nat_gateway_vms: Default::default(),
127            symmetric_private_node_vms: Default::default(),
128        };
129
130        if should_provision_full_cone_private_nodes {
131            let full_cone_private_node_vms = provisioner
132                .ansible_runner
133                .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)
134                .inspect_err(|err| {
135                    println!("Failed to obtain the inventory of Full Cone private node: {err:?}");
136                })?;
137
138            let full_cone_nat_gateway_inventory = provisioner
139                .ansible_runner
140                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)
141                .inspect_err(|err| {
142                    println!("Failed to get Full Cone NAT Gateway inventory {err:?}");
143                })?;
144
145            if full_cone_nat_gateway_inventory.len() != full_cone_private_node_vms.len() {
146                println!("The number of Full Cone private nodes does not match the number of Full Cone NAT Gateway VMs");
147                return Err(Error::VmCountMismatch(
148                    Some(AnsibleInventoryType::FullConePrivateNodes),
149                    Some(AnsibleInventoryType::FullConeNatGateway),
150                ));
151            }
152
153            inventory.full_cone_private_node_vms = full_cone_private_node_vms;
154            inventory.full_cone_nat_gateway_vms = full_cone_nat_gateway_inventory;
155        }
156
157        if should_provision_symmetric_private_nodes {
158            let symmetric_private_node_vms = provisioner
159                .ansible_runner
160                .get_inventory(AnsibleInventoryType::SymmetricPrivateNodes, true)
161                .inspect_err(|err| {
162                    println!("Failed to obtain the inventory of Symmetric private node: {err:?}");
163                })?;
164
165            let symmetric_nat_gateway_inventory = provisioner
166                .ansible_runner
167                .get_inventory(AnsibleInventoryType::SymmetricNatGateway, true)
168                .inspect_err(|err| {
169                    println!("Failed to get Symmetric NAT Gateway inventory {err:?}");
170                })?;
171
172            if symmetric_nat_gateway_inventory.len() != symmetric_private_node_vms.len() {
173                println!("The number of Symmetric private nodes does not match the number of Symmetric NAT Gateway VMs");
174                return Err(Error::VmCountMismatch(
175                    Some(AnsibleInventoryType::SymmetricPrivateNodes),
176                    Some(AnsibleInventoryType::SymmetricNatGateway),
177                ));
178            }
179
180            inventory.symmetric_private_node_vms = symmetric_private_node_vms;
181            inventory.symmetric_nat_gateway_vms = symmetric_nat_gateway_inventory;
182        }
183
184        Ok(inventory)
185    }
186
187    pub fn should_provision_full_cone_private_nodes(&self) -> bool {
188        !self.full_cone_private_node_vms.is_empty()
189    }
190
191    pub fn should_provision_symmetric_private_nodes(&self) -> bool {
192        !self.symmetric_private_node_vms.is_empty()
193    }
194
195    pub fn symmetric_private_node_and_gateway_map(
196        &self,
197    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
198        Self::match_private_node_vm_and_gateway_vm(
199            &self.symmetric_private_node_vms,
200            &self.symmetric_nat_gateway_vms,
201        )
202    }
203
204    pub fn full_cone_private_node_and_gateway_map(
205        &self,
206    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
207        Self::match_private_node_vm_and_gateway_vm(
208            &self.full_cone_private_node_vms,
209            &self.full_cone_nat_gateway_vms,
210        )
211    }
212
213    pub fn match_private_node_vm_and_gateway_vm(
214        private_node_vms: &[VirtualMachine],
215        nat_gateway_vms: &[VirtualMachine],
216    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
217        if private_node_vms.len() != nat_gateway_vms.len() {
218            println!(
219            "The number of private node VMs ({}) does not match the number of NAT Gateway VMs ({})",
220            private_node_vms.len(),
221            nat_gateway_vms.len()
222        );
223            error!("The number of private node VMs does not match the number of NAT Gateway VMs: Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}");
224            return Err(Error::VmCountMismatch(None, None));
225        }
226
227        let mut map = HashMap::new();
228        for private_vm in private_node_vms {
229            let nat_gateway = nat_gateway_vms
230                .iter()
231                .find(|vm| {
232                    let private_node_name = private_vm.name.split('-').next_back().unwrap();
233                    let nat_gateway_name = vm.name.split('-').next_back().unwrap();
234                    private_node_name == nat_gateway_name
235                })
236                .ok_or_else(|| {
237                    println!(
238                        "Failed to find a matching NAT Gateway for private node: {}",
239                        private_vm.name
240                    );
241                    error!("Failed to find a matching NAT Gateway for private node: {}. Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}", private_vm.name);
242                    Error::VmCountMismatch(None, None)
243                })?;
244
245            let _ = map.insert(private_vm.clone(), nat_gateway.clone());
246        }
247
248        Ok(map)
249    }
250}
251
252impl From<BootstrapOptions> for ProvisionOptions {
253    fn from(bootstrap_options: BootstrapOptions) -> Self {
254        ProvisionOptions {
255            ant_version: None,
256            binary_option: bootstrap_options.binary_option,
257            chunk_size: bootstrap_options.chunk_size,
258            client_env_variables: None,
259            delayed_verifier_batch_size: None,
260            delayed_verifier_quorum_value: None,
261            enable_delayed_verifier: false,
262            enable_random_verifier: false,
263            enable_performance_verifier: false,
264            enable_telegraf: true,
265            enable_uploaders: false,
266            evm_data_payments_address: bootstrap_options.evm_data_payments_address,
267            evm_network: bootstrap_options.evm_network,
268            evm_payment_token_address: bootstrap_options.evm_payment_token_address,
269            evm_rpc_url: bootstrap_options.evm_rpc_url,
270            expected_hash: None,
271            expected_size: None,
272            file_address: None,
273            full_cone_private_node_count: bootstrap_options.full_cone_private_node_count,
274            funding_wallet_secret_key: None,
275            gas_amount: None,
276            interval: Some(bootstrap_options.interval),
277            log_format: bootstrap_options.log_format,
278            max_archived_log_files: bootstrap_options.max_archived_log_files,
279            max_log_files: bootstrap_options.max_log_files,
280            max_uploads: None,
281            name: bootstrap_options.name,
282            network_id: Some(bootstrap_options.network_id),
283            network_dashboard_branch: None,
284            node_count: bootstrap_options.node_count,
285            node_env_variables: bootstrap_options.node_env_variables,
286            output_inventory_dir_path: bootstrap_options.output_inventory_dir_path,
287            peer_cache_node_count: 0,
288            performance_verifier_batch_size: None,
289            public_rpc: false,
290            random_verifier_batch_size: None,
291            rewards_address: Some(bootstrap_options.rewards_address),
292            sleep_duration: None,
293            symmetric_private_node_count: bootstrap_options.symmetric_private_node_count,
294            token_amount: None,
295            upload_batch_size: None,
296            upload_size: None,
297            upload_interval: None,
298            uploaders_count: None,
299            upnp_private_node_count: bootstrap_options.upnp_private_node_count,
300            wallet_secret_keys: None,
301        }
302    }
303}
304
305impl From<DeployOptions> for ProvisionOptions {
306    fn from(deploy_options: DeployOptions) -> Self {
307        ProvisionOptions {
308            ant_version: None,
309            binary_option: deploy_options.binary_option,
310            chunk_size: deploy_options.chunk_size,
311            client_env_variables: deploy_options.client_env_variables,
312            delayed_verifier_batch_size: None,
313            delayed_verifier_quorum_value: None,
314            enable_delayed_verifier: deploy_options.enable_delayed_verifier,
315            enable_performance_verifier: deploy_options.enable_performance_verifier,
316            enable_random_verifier: deploy_options.enable_random_verifier,
317            enable_telegraf: deploy_options.enable_telegraf,
318            enable_uploaders: true,
319            node_env_variables: deploy_options.node_env_variables,
320            evm_data_payments_address: deploy_options.evm_data_payments_address,
321            evm_network: deploy_options.evm_network,
322            evm_payment_token_address: deploy_options.evm_payment_token_address,
323            evm_rpc_url: deploy_options.evm_rpc_url,
324            expected_hash: None,
325            expected_size: None,
326            file_address: None,
327            full_cone_private_node_count: deploy_options.full_cone_private_node_count,
328            funding_wallet_secret_key: deploy_options.funding_wallet_secret_key,
329            gas_amount: deploy_options.initial_gas,
330            interval: Some(deploy_options.interval),
331            log_format: deploy_options.log_format,
332            max_archived_log_files: deploy_options.max_archived_log_files,
333            max_log_files: deploy_options.max_log_files,
334            max_uploads: None,
335            name: deploy_options.name,
336            network_id: Some(deploy_options.network_id),
337            network_dashboard_branch: deploy_options.network_dashboard_branch,
338            node_count: deploy_options.node_count,
339            output_inventory_dir_path: deploy_options.output_inventory_dir_path,
340            peer_cache_node_count: deploy_options.peer_cache_node_count,
341            performance_verifier_batch_size: None,
342            public_rpc: deploy_options.public_rpc,
343            random_verifier_batch_size: None,
344            rewards_address: Some(deploy_options.rewards_address),
345            sleep_duration: None,
346            symmetric_private_node_count: deploy_options.symmetric_private_node_count,
347            token_amount: deploy_options.initial_tokens,
348            upload_batch_size: None,
349            upload_size: Some(deploy_options.upload_size),
350            upload_interval: Some(deploy_options.upload_interval),
351            uploaders_count: Some(deploy_options.uploaders_count),
352            upnp_private_node_count: deploy_options.upnp_private_node_count,
353            wallet_secret_keys: None,
354        }
355    }
356}
357
358impl From<ClientsDeployOptions> for ProvisionOptions {
359    fn from(client_options: ClientsDeployOptions) -> Self {
360        Self {
361            ant_version: None,
362            binary_option: client_options.binary_option,
363            chunk_size: client_options.chunk_size,
364            client_env_variables: client_options.client_env_variables,
365            delayed_verifier_batch_size: client_options.delayed_verifier_batch_size,
366            delayed_verifier_quorum_value: client_options.delayed_verifier_quorum_value,
367            enable_delayed_verifier: client_options.enable_delayed_verifier,
368            enable_random_verifier: client_options.enable_random_verifier,
369            enable_performance_verifier: client_options.enable_performance_verifier,
370            enable_telegraf: client_options.enable_telegraf,
371            enable_uploaders: client_options.enable_uploaders,
372            evm_data_payments_address: client_options.evm_details.data_payments_address,
373            evm_network: client_options.evm_details.network,
374            evm_payment_token_address: client_options.evm_details.payment_token_address,
375            evm_rpc_url: client_options.evm_details.rpc_url,
376            expected_hash: client_options.expected_hash,
377            expected_size: client_options.expected_size,
378            file_address: client_options.file_address,
379            full_cone_private_node_count: 0,
380            funding_wallet_secret_key: client_options.funding_wallet_secret_key,
381            gas_amount: client_options.initial_gas,
382            interval: None,
383            log_format: None,
384            max_archived_log_files: client_options.max_archived_log_files,
385            max_log_files: client_options.max_log_files,
386            max_uploads: client_options.max_uploads,
387            name: client_options.name,
388            network_id: client_options.network_id,
389            network_dashboard_branch: None,
390            node_count: 0,
391            node_env_variables: None,
392            output_inventory_dir_path: client_options.output_inventory_dir_path,
393            peer_cache_node_count: 0,
394            performance_verifier_batch_size: client_options.performance_verifier_batch_size,
395            public_rpc: false,
396            random_verifier_batch_size: client_options.random_verifier_batch_size,
397            rewards_address: None,
398            sleep_duration: client_options.sleep_duration,
399            symmetric_private_node_count: 0,
400            token_amount: client_options.initial_tokens,
401            upload_batch_size: client_options.upload_batch_size,
402            upload_size: client_options.upload_size,
403            upload_interval: None,
404            uploaders_count: Some(client_options.uploaders_count),
405            upnp_private_node_count: 0,
406            wallet_secret_keys: client_options.wallet_secret_keys,
407        }
408    }
409}
410
411#[derive(Clone)]
412pub struct AnsibleProvisioner {
413    pub ansible_runner: AnsibleRunner,
414    pub cloud_provider: CloudProvider,
415    pub ssh_client: SshClient,
416}
417
418impl AnsibleProvisioner {
419    pub fn new(
420        ansible_runner: AnsibleRunner,
421        cloud_provider: CloudProvider,
422        ssh_client: SshClient,
423    ) -> Self {
424        Self {
425            ansible_runner,
426            cloud_provider,
427            ssh_client,
428        }
429    }
430
431    pub fn build_autonomi_binaries(
432        &self,
433        options: &ProvisionOptions,
434        binaries_to_build: Option<Vec<String>>,
435    ) -> Result<()> {
436        let start = Instant::now();
437        println!("Obtaining IP address for build VM...");
438        let build_inventory = self
439            .ansible_runner
440            .get_inventory(AnsibleInventoryType::Build, true)?;
441        let build_ip = build_inventory[0].public_ip_addr;
442        self.ssh_client
443            .wait_for_ssh_availability(&build_ip, &self.cloud_provider.get_ssh_user())?;
444
445        println!("Running ansible against build VM...");
446        let base_extra_vars = extra_vars::build_binaries_extra_vars_doc(options)?;
447
448        let extra_vars = if let Some(binaries) = binaries_to_build {
449            let mut build_ant = false;
450            let mut build_antnode = false;
451            let mut build_antctl = false;
452            let mut build_antctld = false;
453
454            for binary in &binaries {
455                match binary.as_str() {
456                    "ant" => build_ant = true,
457                    "antnode" => build_antnode = true,
458                    "antctl" => build_antctl = true,
459                    "antctld" => build_antctld = true,
460                    _ => return Err(Error::InvalidBinaryName(binary.clone())),
461                }
462            }
463
464            let mut json_value: serde_json::Value = serde_json::from_str(&base_extra_vars)?;
465            if let serde_json::Value::Object(ref mut map) = json_value {
466                map.insert("build_ant".to_string(), serde_json::Value::Bool(build_ant));
467                map.insert(
468                    "build_antnode".to_string(),
469                    serde_json::Value::Bool(build_antnode),
470                );
471                map.insert(
472                    "build_antctl".to_string(),
473                    serde_json::Value::Bool(build_antctl),
474                );
475                map.insert(
476                    "build_antctld".to_string(),
477                    serde_json::Value::Bool(build_antctld),
478                );
479            }
480            json_value.to_string()
481        } else {
482            base_extra_vars
483        };
484
485        self.ansible_runner.run_playbook(
486            AnsiblePlaybook::Build,
487            AnsibleInventoryType::Build,
488            Some(extra_vars),
489        )?;
490        print_duration(start.elapsed());
491        Ok(())
492    }
493
494    pub fn cleanup_node_logs(&self, setup_cron: bool) -> Result<()> {
495        for node_inv_type in AnsibleInventoryType::iter_node_type() {
496            self.ansible_runner.run_playbook(
497                AnsiblePlaybook::CleanupLogs,
498                node_inv_type,
499                Some(format!("{{ \"setup_cron\": \"{setup_cron}\" }}")),
500            )?;
501        }
502
503        Ok(())
504    }
505
506    pub fn copy_logs(&self, name: &str, resources_only: bool) -> Result<()> {
507        for node_inv_type in AnsibleInventoryType::iter_node_type() {
508            self.ansible_runner.run_playbook(
509                AnsiblePlaybook::CopyLogs,
510                node_inv_type,
511                Some(format!(
512                    "{{ \"env_name\": \"{name}\", \"resources_only\" : \"{resources_only}\" }}"
513                )),
514            )?;
515        }
516        Ok(())
517    }
518
519    pub fn get_all_node_inventory(&self) -> Result<Vec<VirtualMachine>> {
520        let mut all_node_inventory = Vec::new();
521        for node_inv_type in AnsibleInventoryType::iter_node_type() {
522            all_node_inventory.extend(self.ansible_runner.get_inventory(node_inv_type, false)?);
523        }
524
525        Ok(all_node_inventory)
526    }
527
528    pub fn get_symmetric_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
529        self.ansible_runner
530            .get_inventory(AnsibleInventoryType::SymmetricNatGateway, false)
531    }
532
533    pub fn get_full_cone_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
534        self.ansible_runner
535            .get_inventory(AnsibleInventoryType::FullConeNatGateway, false)
536    }
537
538    pub fn get_client_inventory(&self) -> Result<Vec<VirtualMachine>> {
539        self.ansible_runner
540            .get_inventory(AnsibleInventoryType::Clients, false)
541    }
542
543    pub fn get_node_registries(
544        &self,
545        inventory_type: &AnsibleInventoryType,
546    ) -> Result<DeploymentNodeRegistries> {
547        debug!("Fetching node manager inventory for {inventory_type:?}");
548        let temp_dir_path = tempfile::tempdir()?.into_path();
549        let temp_dir_json = serde_json::to_string(&temp_dir_path)?;
550
551        self.ansible_runner.run_playbook(
552            AnsiblePlaybook::AntCtlInventory,
553            *inventory_type,
554            Some(format!("{{ \"dest\": {temp_dir_json} }}")),
555        )?;
556
557        let node_registry_paths = WalkDir::new(temp_dir_path)
558            .into_iter()
559            .flatten()
560            .filter_map(|entry| {
561                if entry.file_type().is_file()
562                    && entry.path().extension().is_some_and(|ext| ext == "json")
563                {
564                    // tempdir/<testnet_name>-node/var/safenode-manager/node_registry.json
565                    let mut vm_name = entry.path().to_path_buf();
566                    trace!("Found file with json extension: {vm_name:?}");
567                    vm_name.pop();
568                    vm_name.pop();
569                    vm_name.pop();
570                    // Extract the <testnet_name>-node string
571                    trace!("Extracting the vm name from the path");
572                    let vm_name = vm_name.file_name()?.to_str()?;
573                    trace!("Extracted vm name from path: {vm_name}");
574                    Some((vm_name.to_string(), entry.path().to_path_buf()))
575                } else {
576                    None
577                }
578            })
579            .collect::<Vec<(String, PathBuf)>>();
580
581        let mut node_registries = Vec::new();
582        let mut failed_vms = Vec::new();
583        for (vm_name, file_path) in node_registry_paths {
584            match NodeRegistry::load(&file_path) {
585                Ok(node_registry) => node_registries.push((vm_name.clone(), node_registry)),
586                Err(_) => failed_vms.push(vm_name.clone()),
587            }
588        }
589
590        let deployment_registries = DeploymentNodeRegistries {
591            inventory_type: *inventory_type,
592            retrieved_registries: node_registries,
593            failed_vms,
594        };
595        Ok(deployment_registries)
596    }
597
598    pub fn provision_evm_nodes(&self, options: &ProvisionOptions) -> Result<()> {
599        let start = Instant::now();
600        println!("Obtaining IP address for EVM nodes...");
601        let evm_node_inventory = self
602            .ansible_runner
603            .get_inventory(AnsibleInventoryType::EvmNodes, true)?;
604        let evm_node_ip = evm_node_inventory[0].public_ip_addr;
605        self.ssh_client
606            .wait_for_ssh_availability(&evm_node_ip, &self.cloud_provider.get_ssh_user())?;
607
608        println!("Running ansible against EVM nodes...");
609        self.ansible_runner.run_playbook(
610            AnsiblePlaybook::EvmNodes,
611            AnsibleInventoryType::EvmNodes,
612            Some(extra_vars::build_evm_nodes_extra_vars_doc(
613                &options.name,
614                &self.cloud_provider,
615                &options.binary_option,
616            )),
617        )?;
618        print_duration(start.elapsed());
619        Ok(())
620    }
621
622    pub fn provision_genesis_node(&self, options: &ProvisionOptions) -> Result<()> {
623        let start = Instant::now();
624        let genesis_inventory = self
625            .ansible_runner
626            .get_inventory(AnsibleInventoryType::Genesis, true)?;
627        let genesis_ip = genesis_inventory[0].public_ip_addr;
628        self.ssh_client
629            .wait_for_ssh_availability(&genesis_ip, &self.cloud_provider.get_ssh_user())?;
630        self.ansible_runner.run_playbook(
631            AnsiblePlaybook::Genesis,
632            AnsibleInventoryType::Genesis,
633            Some(extra_vars::build_node_extra_vars_doc(
634                &self.cloud_provider.to_string(),
635                options,
636                NodeType::Genesis,
637                None,
638                None,
639                1,
640                options.evm_network.clone(),
641                false,
642            )?),
643        )?;
644
645        print_duration(start.elapsed());
646
647        Ok(())
648    }
649
650    pub fn provision_full_cone(
651        &self,
652        options: &ProvisionOptions,
653        initial_contact_peer: Option<String>,
654        initial_network_contacts_url: Option<String>,
655        private_node_inventory: PrivateNodeProvisionInventory,
656        new_full_cone_nat_gateway_new_vms_for_upscale: Option<Vec<VirtualMachine>>,
657    ) -> Result<()> {
658        // Step 1 of Full Cone NAT Gateway
659        let start = Instant::now();
660        self.print_ansible_run_banner("Provision Full Cone NAT Gateway - Step 1");
661
662        for vm in new_full_cone_nat_gateway_new_vms_for_upscale
663            .as_ref()
664            .unwrap_or(&private_node_inventory.full_cone_nat_gateway_vms)
665            .iter()
666        {
667            println!(
668                "Checking SSH availability for Full Cone NAT Gateway: {}",
669                vm.public_ip_addr
670            );
671            self.ssh_client
672                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
673                .map_err(|e| {
674                    println!("Failed to establish SSH connection to Full Cone NAT Gateway: {e}");
675                    e
676                })?;
677        }
678
679        let mut modified_private_node_inventory = private_node_inventory.clone();
680
681        // If we are upscaling, then we cannot access the gateway VMs which are already deployed.
682        if let Some(new_full_cone_nat_gateway_new_vms_for_upscale) =
683            &new_full_cone_nat_gateway_new_vms_for_upscale
684        {
685            debug!("Removing existing full cone NAT Gateway and private node VMs from the inventory. Old inventory: {modified_private_node_inventory:?}");
686            let mut names_to_keep = Vec::new();
687
688            for vm in new_full_cone_nat_gateway_new_vms_for_upscale.iter() {
689                let nat_gateway_name = vm.name.split('-').next_back().unwrap();
690                names_to_keep.push(nat_gateway_name);
691            }
692
693            modified_private_node_inventory
694                .full_cone_nat_gateway_vms
695                .retain(|vm| {
696                    let nat_gateway_name = vm.name.split('-').next_back().unwrap();
697                    names_to_keep.contains(&nat_gateway_name)
698                });
699            modified_private_node_inventory
700                .full_cone_private_node_vms
701                .retain(|vm| {
702                    let nat_gateway_name = vm.name.split('-').next_back().unwrap();
703                    names_to_keep.contains(&nat_gateway_name)
704                });
705            debug!("New inventory after removing existing full cone NAT Gateway and private node VMs: {modified_private_node_inventory:?}");
706        }
707
708        if modified_private_node_inventory
709            .full_cone_nat_gateway_vms
710            .is_empty()
711        {
712            error!("There are no full cone NAT Gateway VMs available to upscale");
713            return Ok(());
714        }
715
716        let private_node_ip_map = modified_private_node_inventory
717            .full_cone_private_node_and_gateway_map()?
718            .into_iter()
719            .map(|(k, v)| {
720                let gateway_name = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
721                    debug!("Upscaling, using public IP address for gateway name");
722                    v.public_ip_addr.to_string()
723                } else {
724                    v.name.clone()
725                };
726                (gateway_name, k.private_ip_addr)
727            })
728            .collect::<HashMap<String, IpAddr>>();
729
730        if private_node_ip_map.is_empty() {
731            println!("There are no full cone private node VM available to be routed through the full cone NAT Gateway");
732            return Err(Error::EmptyInventory(
733                AnsibleInventoryType::FullConePrivateNodes,
734            ));
735        }
736
737        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
738            &options.name,
739            private_node_ip_map.clone(),
740            "step1",
741        );
742        debug!("Provisioning Full Cone NAT Gateway - Step 1 with vars: {vars}");
743        let gateway_inventory = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
744            debug!("Upscaling, using static inventory for full cone nat gateway.");
745            generate_full_cone_nat_gateway_static_environment_inventory(
746                &modified_private_node_inventory.full_cone_nat_gateway_vms,
747                &options.name,
748                &options.output_inventory_dir_path,
749            )?;
750
751            AnsibleInventoryType::FullConeNatGatewayStatic
752        } else {
753            AnsibleInventoryType::FullConeNatGateway
754        };
755        self.ansible_runner.run_playbook(
756            AnsiblePlaybook::StaticFullConeNatGateway,
757            gateway_inventory,
758            Some(vars),
759        )?;
760
761        // setup private node config
762        self.print_ansible_run_banner("Provisioning Full Cone Private Node Config");
763
764        generate_full_cone_private_node_static_environment_inventory(
765            &options.name,
766            &options.output_inventory_dir_path,
767            &private_node_inventory.full_cone_private_node_vms,
768            &private_node_inventory.full_cone_nat_gateway_vms,
769            &self.ssh_client.private_key_path,
770        )
771        .inspect_err(|err| {
772            error!("Failed to generate full cone private node static inv with err: {err:?}")
773        })?;
774
775        // For a new deployment, it's quite probable that SSH is available, because this part occurs
776        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
777        // check that SSH is available before proceeding.
778        println!("Obtaining IP addresses for nodes...");
779        let inventory = self
780            .ansible_runner
781            .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)?;
782
783        println!("Waiting for SSH availability on Symmetric Private nodes...");
784        for vm in inventory.iter() {
785            println!(
786                "Checking SSH availability for {}: {}",
787                vm.name, vm.public_ip_addr
788            );
789            self.ssh_client
790                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
791                .map_err(|e| {
792                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
793                    e
794                })?;
795        }
796
797        println!("SSH is available on all nodes. Proceeding with provisioning...");
798
799        self.ansible_runner.run_playbook(
800            AnsiblePlaybook::PrivateNodeConfig,
801            AnsibleInventoryType::FullConePrivateNodes,
802            Some(
803                extra_vars::build_full_cone_private_node_config_extra_vars_docs(
804                    &private_node_inventory,
805                )?,
806            ),
807        )?;
808
809        // Step 2 of Full Cone NAT Gateway
810
811        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
812            &options.name,
813            private_node_ip_map,
814            "step2",
815        );
816
817        self.print_ansible_run_banner("Provisioning Full Cone NAT Gateway - Step 2");
818        debug!("Provisioning Full Cone NAT Gateway - Step 2 with vars: {vars}");
819        self.ansible_runner.run_playbook(
820            AnsiblePlaybook::StaticFullConeNatGateway,
821            gateway_inventory,
822            Some(vars),
823        )?;
824
825        // provision the nodes
826
827        let home_dir = std::env::var("HOME").inspect_err(|err| {
828            println!("Failed to get home directory with error: {err:?}",);
829        })?;
830        let known_hosts_path = format!("{home_dir}/.ssh/known_hosts");
831        debug!("Cleaning up known hosts file at {known_hosts_path} ");
832        run_external_command(
833            PathBuf::from("rm"),
834            std::env::current_dir()?,
835            vec![known_hosts_path],
836            false,
837            false,
838        )?;
839
840        self.print_ansible_run_banner("Provision Full Cone Private Nodes");
841
842        self.ssh_client.set_full_cone_nat_routed_vms(
843            &private_node_inventory.full_cone_private_node_vms,
844            &private_node_inventory.full_cone_nat_gateway_vms,
845        )?;
846
847        self.provision_nodes(
848            options,
849            initial_contact_peer,
850            initial_network_contacts_url,
851            NodeType::FullConePrivateNode,
852        )?;
853
854        print_duration(start.elapsed());
855        Ok(())
856    }
857    pub fn provision_symmetric_nat_gateway(
858        &self,
859        options: &ProvisionOptions,
860        private_node_inventory: &PrivateNodeProvisionInventory,
861    ) -> Result<()> {
862        let start = Instant::now();
863        for vm in &private_node_inventory.symmetric_nat_gateway_vms {
864            println!(
865                "Checking SSH availability for Symmetric NAT Gateway: {}",
866                vm.public_ip_addr
867            );
868            self.ssh_client
869                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
870                .map_err(|e| {
871                    println!("Failed to establish SSH connection to Symmetric NAT Gateway: {e}");
872                    e
873                })?;
874        }
875
876        let private_node_ip_map = private_node_inventory
877            .symmetric_private_node_and_gateway_map()?
878            .into_iter()
879            .map(|(k, v)| (v.name.clone(), k.private_ip_addr))
880            .collect::<HashMap<String, IpAddr>>();
881
882        if private_node_ip_map.is_empty() {
883            println!("There are no Symmetric private node VM available to be routed through the Symmetric NAT Gateway");
884            return Err(Error::EmptyInventory(
885                AnsibleInventoryType::SymmetricPrivateNodes,
886            ));
887        }
888
889        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
890            &options.name,
891            private_node_ip_map,
892            "symmetric",
893        );
894        debug!("Provisioning Symmetric NAT Gateway with vars: {vars}");
895        self.ansible_runner.run_playbook(
896            AnsiblePlaybook::SymmetricNatGateway,
897            AnsibleInventoryType::SymmetricNatGateway,
898            Some(vars),
899        )?;
900
901        print_duration(start.elapsed());
902        Ok(())
903    }
904
905    pub fn provision_nodes(
906        &self,
907        options: &ProvisionOptions,
908        initial_contact_peer: Option<String>,
909        initial_network_contacts_url: Option<String>,
910        node_type: NodeType,
911    ) -> Result<()> {
912        let start = Instant::now();
913        let mut write_older_cache_files = false;
914        let (inventory_type, node_count) = match &node_type {
915            NodeType::FullConePrivateNode => (
916                node_type.to_ansible_inventory_type(),
917                options.full_cone_private_node_count,
918            ),
919            NodeType::Generic => (node_type.to_ansible_inventory_type(), options.node_count),
920            NodeType::Genesis => return Err(Error::InvalidNodeType(node_type)),
921            NodeType::PeerCache => {
922                write_older_cache_files = true;
923                (
924                    node_type.to_ansible_inventory_type(),
925                    options.peer_cache_node_count,
926                )
927            }
928            NodeType::SymmetricPrivateNode => (
929                node_type.to_ansible_inventory_type(),
930                options.symmetric_private_node_count,
931            ),
932            NodeType::Upnp => (
933                node_type.to_ansible_inventory_type(),
934                options.upnp_private_node_count,
935            ),
936        };
937
938        // For a new deployment, it's quite probable that SSH is available, because this part occurs
939        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
940        // check that SSH is available before proceeding.
941        println!("Obtaining IP addresses for nodes...");
942        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
943
944        println!("Waiting for SSH availability on {node_type:?} nodes...");
945        for vm in inventory.iter() {
946            println!(
947                "Checking SSH availability for {}: {}",
948                vm.name, vm.public_ip_addr
949            );
950            self.ssh_client
951                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
952                .map_err(|e| {
953                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
954                    e
955                })?;
956        }
957
958        println!("SSH is available on all nodes. Proceeding with provisioning...");
959
960        let playbook = match node_type {
961            NodeType::Generic => AnsiblePlaybook::Nodes,
962            NodeType::PeerCache => AnsiblePlaybook::PeerCacheNodes,
963            NodeType::FullConePrivateNode => AnsiblePlaybook::Nodes,
964            NodeType::SymmetricPrivateNode => AnsiblePlaybook::Nodes,
965            NodeType::Upnp => AnsiblePlaybook::Upnp,
966            _ => return Err(Error::InvalidNodeType(node_type.clone())),
967        };
968        self.ansible_runner.run_playbook(
969            playbook,
970            inventory_type,
971            Some(extra_vars::build_node_extra_vars_doc(
972                &self.cloud_provider.to_string(),
973                options,
974                node_type.clone(),
975                initial_contact_peer,
976                initial_network_contacts_url,
977                node_count,
978                options.evm_network.clone(),
979                write_older_cache_files,
980            )?),
981        )?;
982
983        print_duration(start.elapsed());
984        Ok(())
985    }
986
987    pub fn provision_symmetric_private_nodes(
988        &self,
989        options: &mut ProvisionOptions,
990        initial_contact_peer: Option<String>,
991        initial_network_contacts_url: Option<String>,
992        private_node_inventory: &PrivateNodeProvisionInventory,
993    ) -> Result<()> {
994        let start = Instant::now();
995        self.print_ansible_run_banner("Provision Symmetric Private Node Config");
996
997        generate_symmetric_private_node_static_environment_inventory(
998            &options.name,
999            &options.output_inventory_dir_path,
1000            &private_node_inventory.symmetric_private_node_vms,
1001            &private_node_inventory.symmetric_nat_gateway_vms,
1002            &self.ssh_client.private_key_path,
1003        )
1004        .inspect_err(|err| {
1005            error!("Failed to generate symmetric private node static inv with err: {err:?}")
1006        })?;
1007
1008        self.ssh_client.set_symmetric_nat_routed_vms(
1009            &private_node_inventory.symmetric_private_node_vms,
1010            &private_node_inventory.symmetric_nat_gateway_vms,
1011        )?;
1012
1013        let inventory_type = AnsibleInventoryType::SymmetricPrivateNodes;
1014
1015        // For a new deployment, it's quite probable that SSH is available, because this part occurs
1016        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
1017        // check that SSH is available before proceeding.
1018        println!("Obtaining IP addresses for nodes...");
1019        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
1020
1021        println!("Waiting for SSH availability on Symmetric Private nodes...");
1022        for vm in inventory.iter() {
1023            println!(
1024                "Checking SSH availability for {}: {}",
1025                vm.name, vm.public_ip_addr
1026            );
1027            self.ssh_client
1028                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
1029                .map_err(|e| {
1030                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
1031                    e
1032                })?;
1033        }
1034
1035        println!("SSH is available on all nodes. Proceeding with provisioning...");
1036
1037        self.ansible_runner.run_playbook(
1038            AnsiblePlaybook::PrivateNodeConfig,
1039            inventory_type,
1040            Some(
1041                extra_vars::build_symmetric_private_node_config_extra_vars_doc(
1042                    private_node_inventory,
1043                )?,
1044            ),
1045        )?;
1046
1047        println!("Provisioned Symmetric Private Node Config");
1048        print_duration(start.elapsed());
1049
1050        self.provision_nodes(
1051            options,
1052            initial_contact_peer,
1053            initial_network_contacts_url,
1054            NodeType::SymmetricPrivateNode,
1055        )?;
1056
1057        Ok(())
1058    }
1059
1060    pub async fn provision_downloaders(
1061        &self,
1062        options: &ProvisionOptions,
1063        genesis_multiaddr: Option<String>,
1064        genesis_network_contacts_url: Option<String>,
1065    ) -> Result<()> {
1066        let start = Instant::now();
1067
1068        println!("Running ansible against Client machine to start the downloader script.");
1069        debug!("Running ansible against Client machine to start the downloader script.");
1070
1071        self.ansible_runner.run_playbook(
1072            AnsiblePlaybook::Downloaders,
1073            AnsibleInventoryType::Clients,
1074            Some(extra_vars::build_downloaders_extra_vars_doc(
1075                &self.cloud_provider.to_string(),
1076                options,
1077                genesis_multiaddr,
1078                genesis_network_contacts_url,
1079            )?),
1080        )?;
1081        print_duration(start.elapsed());
1082        Ok(())
1083    }
1084
1085    pub async fn provision_static_downloaders(
1086        &self,
1087        options: &ProvisionOptions,
1088        genesis_multiaddr: Option<String>,
1089        genesis_network_contacts_url: Option<String>,
1090    ) -> Result<()> {
1091        let start = Instant::now();
1092
1093        println!("Running ansible against client machine to start the static downloaders.");
1094        debug!("Running ansible against client machine to start the static downloaders.");
1095
1096        self.ansible_runner.run_playbook(
1097            AnsiblePlaybook::StaticDownloaders,
1098            AnsibleInventoryType::Clients,
1099            Some(extra_vars::build_downloaders_extra_vars_doc(
1100                &self.cloud_provider.to_string(),
1101                options,
1102                genesis_multiaddr,
1103                genesis_network_contacts_url,
1104            )?),
1105        )?;
1106        print_duration(start.elapsed());
1107        Ok(())
1108    }
1109
1110    pub async fn provision_static_uploader(
1111        &self,
1112        options: &ProvisionOptions,
1113        genesis_multiaddr: Option<String>,
1114        genesis_network_contacts_url: Option<String>,
1115    ) -> Result<()> {
1116        let start = Instant::now();
1117
1118        println!("Running ansible against client machine to provision the static uploader");
1119        debug!("Running ansible against client machine to provision the static uploader");
1120
1121        let sk_map = if let Some(wallet_keys) = &options.wallet_secret_keys {
1122            self.prepare_pre_funded_wallets(wallet_keys).await?
1123        } else {
1124            self.deposit_funds_to_clients(&FundingOptions {
1125                evm_data_payments_address: options.evm_data_payments_address.clone(),
1126                evm_network: options.evm_network.clone(),
1127                evm_payment_token_address: options.evm_payment_token_address.clone(),
1128                evm_rpc_url: options.evm_rpc_url.clone(),
1129                funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
1130                gas_amount: options.gas_amount,
1131                token_amount: options.token_amount,
1132                uploaders_count: Some(1),
1133            })
1134            .await?
1135        };
1136
1137        self.ansible_runner.run_playbook(
1138            AnsiblePlaybook::StaticUploader,
1139            AnsibleInventoryType::Clients,
1140            Some(extra_vars::build_clients_extra_vars_doc(
1141                &self.cloud_provider.to_string(),
1142                options,
1143                genesis_multiaddr,
1144                genesis_network_contacts_url,
1145                &sk_map,
1146            )?),
1147        )?;
1148        print_duration(start.elapsed());
1149        Ok(())
1150    }
1151
1152    pub async fn provision_uploaders(
1153        &self,
1154        options: &ProvisionOptions,
1155        genesis_multiaddr: Option<String>,
1156        genesis_network_contacts_url: Option<String>,
1157    ) -> Result<()> {
1158        let start = Instant::now();
1159
1160        let sk_map = if let Some(wallet_keys) = &options.wallet_secret_keys {
1161            self.prepare_pre_funded_wallets(wallet_keys).await?
1162        } else {
1163            self.deposit_funds_to_clients(&FundingOptions {
1164                evm_data_payments_address: options.evm_data_payments_address.clone(),
1165                evm_network: options.evm_network.clone(),
1166                evm_payment_token_address: options.evm_payment_token_address.clone(),
1167                evm_rpc_url: options.evm_rpc_url.clone(),
1168                funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
1169                gas_amount: options.gas_amount,
1170                token_amount: options.token_amount,
1171                uploaders_count: options.uploaders_count,
1172            })
1173            .await?
1174        };
1175
1176        self.ansible_runner.run_playbook(
1177            AnsiblePlaybook::Uploaders,
1178            AnsibleInventoryType::Clients,
1179            Some(extra_vars::build_clients_extra_vars_doc(
1180                &self.cloud_provider.to_string(),
1181                options,
1182                genesis_multiaddr,
1183                genesis_network_contacts_url,
1184                &sk_map,
1185            )?),
1186        )?;
1187        print_duration(start.elapsed());
1188        Ok(())
1189    }
1190
1191    pub fn start_nodes(
1192        &self,
1193        environment_name: &str,
1194        interval: Duration,
1195        node_type: Option<NodeType>,
1196        custom_inventory: Option<Vec<VirtualMachine>>,
1197    ) -> Result<()> {
1198        let mut extra_vars = ExtraVarsDocBuilder::default();
1199        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1200
1201        if let Some(node_type) = node_type {
1202            println!("Running the start nodes playbook for {node_type:?} nodes");
1203            self.ansible_runner.run_playbook(
1204                AnsiblePlaybook::StartNodes,
1205                node_type.to_ansible_inventory_type(),
1206                Some(extra_vars.build()),
1207            )?;
1208            return Ok(());
1209        }
1210
1211        if let Some(custom_inventory) = custom_inventory {
1212            println!("Running the start nodes playbook with a custom inventory");
1213            generate_custom_environment_inventory(
1214                &custom_inventory,
1215                environment_name,
1216                &self.ansible_runner.working_directory_path.join("inventory"),
1217            )?;
1218            self.ansible_runner.run_playbook(
1219                AnsiblePlaybook::StartNodes,
1220                AnsibleInventoryType::Custom,
1221                Some(extra_vars.build()),
1222            )?;
1223            return Ok(());
1224        }
1225
1226        println!("Running the start nodes playbook for all node types");
1227        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1228            self.ansible_runner.run_playbook(
1229                AnsiblePlaybook::StartNodes,
1230                node_inv_type,
1231                Some(extra_vars.build()),
1232            )?;
1233        }
1234        Ok(())
1235    }
1236
1237    pub fn status(&self) -> Result<()> {
1238        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1239            self.ansible_runner
1240                .run_playbook(AnsiblePlaybook::Status, node_inv_type, None)?;
1241        }
1242        Ok(())
1243    }
1244
1245    pub fn start_telegraf(
1246        &self,
1247        environment_name: &str,
1248        node_type: Option<NodeType>,
1249        custom_inventory: Option<Vec<VirtualMachine>>,
1250    ) -> Result<()> {
1251        if let Some(node_type) = node_type {
1252            println!("Running the start telegraf playbook for {node_type:?} nodes");
1253            self.ansible_runner.run_playbook(
1254                AnsiblePlaybook::StartTelegraf,
1255                node_type.to_ansible_inventory_type(),
1256                None,
1257            )?;
1258            return Ok(());
1259        }
1260
1261        if let Some(custom_inventory) = custom_inventory {
1262            println!("Running the start telegraf playbook with a custom inventory");
1263            generate_custom_environment_inventory(
1264                &custom_inventory,
1265                environment_name,
1266                &self.ansible_runner.working_directory_path.join("inventory"),
1267            )?;
1268            self.ansible_runner.run_playbook(
1269                AnsiblePlaybook::StartTelegraf,
1270                AnsibleInventoryType::Custom,
1271                None,
1272            )?;
1273            return Ok(());
1274        }
1275
1276        println!("Running the start telegraf playbook for all node types");
1277        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1278            self.ansible_runner.run_playbook(
1279                AnsiblePlaybook::StartTelegraf,
1280                node_inv_type,
1281                None,
1282            )?;
1283        }
1284
1285        Ok(())
1286    }
1287
1288    pub fn stop_nodes(
1289        &self,
1290        environment_name: &str,
1291        interval: Duration,
1292        node_type: Option<NodeType>,
1293        custom_inventory: Option<Vec<VirtualMachine>>,
1294        delay: Option<u64>,
1295        service_names: Option<Vec<String>>,
1296    ) -> Result<()> {
1297        let mut extra_vars = ExtraVarsDocBuilder::default();
1298        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1299        if let Some(delay) = delay {
1300            extra_vars.add_variable("delay", &delay.to_string());
1301        }
1302        if let Some(service_names) = service_names {
1303            extra_vars.add_list_variable("service_names", service_names);
1304        }
1305        let extra_vars = extra_vars.build();
1306
1307        if let Some(node_type) = node_type {
1308            println!("Running the stop nodes playbook for {node_type:?} nodes");
1309            self.ansible_runner.run_playbook(
1310                AnsiblePlaybook::StopNodes,
1311                node_type.to_ansible_inventory_type(),
1312                Some(extra_vars),
1313            )?;
1314            return Ok(());
1315        }
1316
1317        if let Some(custom_inventory) = custom_inventory {
1318            println!("Running the stop nodes playbook with a custom inventory");
1319            generate_custom_environment_inventory(
1320                &custom_inventory,
1321                environment_name,
1322                &self.ansible_runner.working_directory_path.join("inventory"),
1323            )?;
1324            self.ansible_runner.run_playbook(
1325                AnsiblePlaybook::StopNodes,
1326                AnsibleInventoryType::Custom,
1327                Some(extra_vars),
1328            )?;
1329            return Ok(());
1330        }
1331
1332        println!("Running the stop nodes playbook for all node types");
1333        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1334            self.ansible_runner.run_playbook(
1335                AnsiblePlaybook::StopNodes,
1336                node_inv_type,
1337                Some(extra_vars.clone()),
1338            )?;
1339        }
1340
1341        Ok(())
1342    }
1343
1344    pub fn stop_telegraf(
1345        &self,
1346        environment_name: &str,
1347        node_type: Option<NodeType>,
1348        custom_inventory: Option<Vec<VirtualMachine>>,
1349    ) -> Result<()> {
1350        if let Some(node_type) = node_type {
1351            println!("Running the stop telegraf playbook for {node_type:?} nodes");
1352            self.ansible_runner.run_playbook(
1353                AnsiblePlaybook::StopTelegraf,
1354                node_type.to_ansible_inventory_type(),
1355                None,
1356            )?;
1357            return Ok(());
1358        }
1359
1360        if let Some(custom_inventory) = custom_inventory {
1361            println!("Running the stop telegraf playbook with a custom inventory");
1362            generate_custom_environment_inventory(
1363                &custom_inventory,
1364                environment_name,
1365                &self.ansible_runner.working_directory_path.join("inventory"),
1366            )?;
1367            self.ansible_runner.run_playbook(
1368                AnsiblePlaybook::StopTelegraf,
1369                AnsibleInventoryType::Custom,
1370                None,
1371            )?;
1372            return Ok(());
1373        }
1374
1375        println!("Running the stop telegraf playbook for all node types");
1376        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1377            self.ansible_runner
1378                .run_playbook(AnsiblePlaybook::StopTelegraf, node_inv_type, None)?;
1379        }
1380
1381        Ok(())
1382    }
1383
1384    pub fn upgrade_node_telegraf(&self, name: &str) -> Result<()> {
1385        self.ansible_runner.run_playbook(
1386            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1387            AnsibleInventoryType::PeerCacheNodes,
1388            Some(extra_vars::build_node_telegraf_upgrade(
1389                name,
1390                &NodeType::PeerCache,
1391            )?),
1392        )?;
1393        self.ansible_runner.run_playbook(
1394            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1395            AnsibleInventoryType::Nodes,
1396            Some(extra_vars::build_node_telegraf_upgrade(
1397                name,
1398                &NodeType::Generic,
1399            )?),
1400        )?;
1401
1402        self.ansible_runner.run_playbook(
1403            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1404            AnsibleInventoryType::SymmetricPrivateNodes,
1405            Some(extra_vars::build_node_telegraf_upgrade(
1406                name,
1407                &NodeType::SymmetricPrivateNode,
1408            )?),
1409        )?;
1410
1411        self.ansible_runner.run_playbook(
1412            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1413            AnsibleInventoryType::FullConePrivateNodes,
1414            Some(extra_vars::build_node_telegraf_upgrade(
1415                name,
1416                &NodeType::FullConePrivateNode,
1417            )?),
1418        )?;
1419        Ok(())
1420    }
1421
1422    pub fn upgrade_client_telegraf(&self, name: &str) -> Result<()> {
1423        self.ansible_runner.run_playbook(
1424            AnsiblePlaybook::UpgradeClientTelegrafConfig,
1425            AnsibleInventoryType::Clients,
1426            Some(extra_vars::build_client_telegraf_upgrade(name)?),
1427        )?;
1428        Ok(())
1429    }
1430
1431    pub fn upgrade_nodes(&self, options: &UpgradeOptions) -> Result<()> {
1432        if let Some(custom_inventory) = &options.custom_inventory {
1433            println!("Running the UpgradeNodes with a custom inventory");
1434            generate_custom_environment_inventory(
1435                custom_inventory,
1436                &options.name,
1437                &self.ansible_runner.working_directory_path.join("inventory"),
1438            )?;
1439            match self.ansible_runner.run_playbook(
1440                AnsiblePlaybook::UpgradeNodes,
1441                AnsibleInventoryType::Custom,
1442                Some(options.get_ansible_vars()),
1443            ) {
1444                Ok(()) => println!("All nodes were successfully upgraded"),
1445                Err(_) => {
1446                    println!("WARNING: some nodes may not have been upgraded or restarted");
1447                }
1448            }
1449            return Ok(());
1450        }
1451
1452        if let Some(node_type) = &options.node_type {
1453            println!("Running the UpgradeNodes playbook for {node_type:?} nodes");
1454            match self.ansible_runner.run_playbook(
1455                AnsiblePlaybook::UpgradeNodes,
1456                node_type.to_ansible_inventory_type(),
1457                Some(options.get_ansible_vars()),
1458            ) {
1459                Ok(()) => println!("All {node_type:?} nodes were successfully upgraded"),
1460                Err(_) => {
1461                    println!(
1462                        "WARNING: some {node_type:?} nodes may not have been upgraded or restarted"
1463                    );
1464                }
1465            }
1466            return Ok(());
1467        }
1468
1469        println!("Running the UpgradeNodes playbook for all node types");
1470
1471        match self.ansible_runner.run_playbook(
1472            AnsiblePlaybook::UpgradeNodes,
1473            AnsibleInventoryType::PeerCacheNodes,
1474            Some(options.get_ansible_vars()),
1475        ) {
1476            Ok(()) => println!("All Peer Cache nodes were successfully upgraded"),
1477            Err(_) => {
1478                println!("WARNING: some Peer Cacche nodes may not have been upgraded or restarted");
1479            }
1480        }
1481        match self.ansible_runner.run_playbook(
1482            AnsiblePlaybook::UpgradeNodes,
1483            AnsibleInventoryType::Nodes,
1484            Some(options.get_ansible_vars()),
1485        ) {
1486            Ok(()) => println!("All generic nodes were successfully upgraded"),
1487            Err(_) => {
1488                println!("WARNING: some nodes may not have been upgraded or restarted");
1489            }
1490        }
1491        match self.ansible_runner.run_playbook(
1492            AnsiblePlaybook::UpgradeNodes,
1493            AnsibleInventoryType::SymmetricPrivateNodes,
1494            Some(options.get_ansible_vars()),
1495        ) {
1496            Ok(()) => println!("All private nodes were successfully upgraded"),
1497            Err(_) => {
1498                println!("WARNING: some nodes may not have been upgraded or restarted");
1499            }
1500        }
1501        // Don't use AnsibleInventoryType::iter_node_type() here, because the genesis node should be upgraded last
1502        match self.ansible_runner.run_playbook(
1503            AnsiblePlaybook::UpgradeNodes,
1504            AnsibleInventoryType::Genesis,
1505            Some(options.get_ansible_vars()),
1506        ) {
1507            Ok(()) => println!("The genesis nodes was successfully upgraded"),
1508            Err(_) => {
1509                println!("WARNING: the genesis node may not have been upgraded or restarted");
1510            }
1511        }
1512        Ok(())
1513    }
1514
1515    pub fn upgrade_antctl(
1516        &self,
1517        environment_name: &str,
1518        version: &Version,
1519        node_type: Option<NodeType>,
1520        custom_inventory: Option<Vec<VirtualMachine>>,
1521    ) -> Result<()> {
1522        let mut extra_vars = ExtraVarsDocBuilder::default();
1523        extra_vars.add_variable("version", &version.to_string());
1524
1525        if let Some(node_type) = node_type {
1526            println!("Running the upgrade safenode-manager playbook for {node_type:?} nodes");
1527            self.ansible_runner.run_playbook(
1528                AnsiblePlaybook::UpgradeAntctl,
1529                node_type.to_ansible_inventory_type(),
1530                Some(extra_vars.build()),
1531            )?;
1532            return Ok(());
1533        }
1534
1535        if let Some(custom_inventory) = custom_inventory {
1536            println!("Running the upgrade safenode-manager playbook with a custom inventory");
1537            generate_custom_environment_inventory(
1538                &custom_inventory,
1539                environment_name,
1540                &self.ansible_runner.working_directory_path.join("inventory"),
1541            )?;
1542            self.ansible_runner.run_playbook(
1543                AnsiblePlaybook::UpgradeAntctl,
1544                AnsibleInventoryType::Custom,
1545                Some(extra_vars.build()),
1546            )?;
1547            return Ok(());
1548        }
1549
1550        println!("Running the upgrade safenode-manager playbook for all node types");
1551        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1552            self.ansible_runner.run_playbook(
1553                AnsiblePlaybook::UpgradeAntctl,
1554                node_inv_type,
1555                Some(extra_vars.build()),
1556            )?;
1557        }
1558
1559        Ok(())
1560    }
1561
1562    pub fn upgrade_nginx_config(
1563        &self,
1564        environment_name: &str,
1565        custom_inventory: Option<Vec<VirtualMachine>>,
1566    ) -> Result<()> {
1567        if let Some(custom_inventory) = custom_inventory {
1568            println!("Running the upgrade nginx config playbook with a custom inventory");
1569            generate_custom_environment_inventory(
1570                &custom_inventory,
1571                environment_name,
1572                &self.ansible_runner.working_directory_path.join("inventory"),
1573            )?;
1574            self.ansible_runner.run_playbook(
1575                AnsiblePlaybook::UpgradeNginx,
1576                AnsibleInventoryType::Custom,
1577                None,
1578            )?;
1579            return Ok(());
1580        }
1581
1582        println!("Running the upgrade nginx config playbook for peer cache nodes");
1583        self.ansible_runner.run_playbook(
1584            AnsiblePlaybook::UpgradeNginx,
1585            AnsibleInventoryType::PeerCacheNodes,
1586            None,
1587        )?;
1588        Ok(())
1589    }
1590
1591    pub fn upgrade_geoip_telegraf(&self, name: &str) -> Result<()> {
1592        self.ansible_runner.run_playbook(
1593            AnsiblePlaybook::UpgradeGeoIpTelegrafConfig,
1594            AnsibleInventoryType::PeerCacheNodes,
1595            Some(extra_vars::build_node_telegraf_upgrade(
1596                name,
1597                &NodeType::PeerCache,
1598            )?),
1599        )?;
1600        Ok(())
1601    }
1602
1603    pub fn print_ansible_run_banner(&self, s: &str) {
1604        let ansible_run_msg = "Ansible Run: ";
1605        let line = "=".repeat(s.len() + ansible_run_msg.len());
1606        println!("{line}\n{ansible_run_msg}{s}\n{line}");
1607    }
1608}
sn_testnet_deploy/ansible/provisioning.rs

sn_testnet_deploy/ansible/
provisioning.rs
