sn_testnet_deploy/ansible/
provisioning.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use super::{
8    extra_vars::ExtraVarsDocBuilder,
9    inventory::{
10        generate_full_cone_private_node_static_environment_inventory,
11        generate_symmetric_private_node_static_environment_inventory,
12    },
13    AnsibleInventoryType, AnsiblePlaybook, AnsibleRunner,
14};
15use crate::{
16    ansible::inventory::{
17        generate_custom_environment_inventory,
18        generate_full_cone_nat_gateway_static_environment_inventory,
19    },
20    bootstrap::BootstrapOptions,
21    deploy::DeployOptions,
22    error::{Error, Result},
23    funding::FundingOptions,
24    inventory::{DeploymentNodeRegistries, VirtualMachine},
25    print_duration, run_external_command, BinaryOption, CloudProvider, EvmNetwork, LogFormat,
26    NodeType, SshClient, UpgradeOptions,
27};
28use ant_service_management::NodeRegistry;
29use evmlib::common::U256;
30use log::{debug, error, trace};
31use semver::Version;
32use serde::{Deserialize, Serialize};
33use std::{
34    collections::HashMap,
35    net::{IpAddr, SocketAddr},
36    path::PathBuf,
37    time::{Duration, Instant},
38};
39use walkdir::WalkDir;
40
41use crate::ansible::extra_vars;
42
43pub const DEFAULT_BETA_ENCRYPTION_KEY: &str =
44    "49113d2083f57a976076adbe85decb75115820de1e6e74b47e0429338cef124a";
45
46#[derive(Clone, Serialize, Deserialize)]
47pub struct ProvisionOptions {
48    /// The safe version is also in the binary option, but only for an initial deployment.
49    /// For the upscale, it needs to be provided explicitly, because currently it is not
50    /// recorded in the inventory.
51    pub ant_version: Option<String>,
52    pub binary_option: BinaryOption,
53    pub chunk_size: Option<u64>,
54    pub downloaders_count: u16,
55    pub enable_telegraf: bool,
56    pub env_variables: Option<Vec<(String, String)>>,
57    pub evm_data_payments_address: Option<String>,
58    pub evm_network: EvmNetwork,
59    pub evm_payment_token_address: Option<String>,
60    pub evm_rpc_url: Option<String>,
61    pub full_cone_private_node_count: u16,
62    pub funding_wallet_secret_key: Option<String>,
63    pub gas_amount: Option<U256>,
64    pub interval: Duration,
65    pub log_format: Option<LogFormat>,
66    pub logstash_details: Option<(String, Vec<SocketAddr>)>,
67    pub max_archived_log_files: u16,
68    pub max_log_files: u16,
69    pub name: String,
70    pub network_id: Option<u8>,
71    pub node_count: u16,
72    pub output_inventory_dir_path: PathBuf,
73    pub peer_cache_node_count: u16,
74    pub symmetric_private_node_count: u16,
75    pub public_rpc: bool,
76    pub rewards_address: String,
77    pub token_amount: Option<U256>,
78    pub uploaders_count: Option<u16>,
79}
80
81/// These are obtained by running the inventory playbook
82#[derive(Clone, Debug)]
83pub struct PrivateNodeProvisionInventory {
84    pub full_cone_nat_gateway_vms: Vec<VirtualMachine>,
85    pub full_cone_private_node_vms: Vec<VirtualMachine>,
86    pub symmetric_nat_gateway_vms: Vec<VirtualMachine>,
87    pub symmetric_private_node_vms: Vec<VirtualMachine>,
88}
89
90impl PrivateNodeProvisionInventory {
91    pub fn new(
92        provisioner: &AnsibleProvisioner,
93        full_cone_private_node_vm_count: Option<u16>,
94        symmetric_private_node_vm_count: Option<u16>,
95    ) -> Result<Self> {
96        // All the environment types set private_node_vm count to >0 if not specified.
97        let should_provision_full_cone_private_nodes = full_cone_private_node_vm_count
98            .map(|count| count > 0)
99            .unwrap_or(true);
100        let should_provision_symmetric_private_nodes = symmetric_private_node_vm_count
101            .map(|count| count > 0)
102            .unwrap_or(true);
103
104        let mut inventory = Self {
105            full_cone_nat_gateway_vms: Default::default(),
106            full_cone_private_node_vms: Default::default(),
107            symmetric_nat_gateway_vms: Default::default(),
108            symmetric_private_node_vms: Default::default(),
109        };
110
111        if should_provision_full_cone_private_nodes {
112            let full_cone_private_node_vms = provisioner
113                .ansible_runner
114                .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)
115                .inspect_err(|err| {
116                    println!("Failed to obtain the inventory of Full Cone private node: {err:?}");
117                })?;
118
119            let full_cone_nat_gateway_inventory = provisioner
120                .ansible_runner
121                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)
122                .inspect_err(|err| {
123                    println!("Failed to get Full Cone NAT Gateway inventory {err:?}");
124                })?;
125
126            if full_cone_nat_gateway_inventory.len() != full_cone_private_node_vms.len() {
127                println!("The number of Full Cone private nodes does not match the number of Full Cone NAT Gateway VMs");
128                return Err(Error::VmCountMismatch(
129                    Some(AnsibleInventoryType::FullConePrivateNodes),
130                    Some(AnsibleInventoryType::FullConeNatGateway),
131                ));
132            }
133
134            inventory.full_cone_private_node_vms = full_cone_private_node_vms;
135            inventory.full_cone_nat_gateway_vms = full_cone_nat_gateway_inventory;
136        }
137
138        if should_provision_symmetric_private_nodes {
139            let symmetric_private_node_vms = provisioner
140                .ansible_runner
141                .get_inventory(AnsibleInventoryType::SymmetricPrivateNodes, true)
142                .inspect_err(|err| {
143                    println!("Failed to obtain the inventory of Symmetric private node: {err:?}");
144                })?;
145
146            let symmetric_nat_gateway_inventory = provisioner
147                .ansible_runner
148                .get_inventory(AnsibleInventoryType::SymmetricNatGateway, true)
149                .inspect_err(|err| {
150                    println!("Failed to get Symmetric NAT Gateway inventory {err:?}");
151                })?;
152
153            if symmetric_nat_gateway_inventory.len() != symmetric_private_node_vms.len() {
154                println!("The number of Symmetric private nodes does not match the number of Symmetric NAT Gateway VMs");
155                return Err(Error::VmCountMismatch(
156                    Some(AnsibleInventoryType::SymmetricPrivateNodes),
157                    Some(AnsibleInventoryType::SymmetricNatGateway),
158                ));
159            }
160
161            inventory.symmetric_private_node_vms = symmetric_private_node_vms;
162            inventory.symmetric_nat_gateway_vms = symmetric_nat_gateway_inventory;
163        }
164
165        Ok(inventory)
166    }
167
168    pub fn should_provision_full_cone_private_nodes(&self) -> bool {
169        !self.full_cone_private_node_vms.is_empty()
170    }
171
172    pub fn should_provision_symmetric_private_nodes(&self) -> bool {
173        !self.symmetric_private_node_vms.is_empty()
174    }
175
176    pub fn symmetric_private_node_and_gateway_map(
177        &self,
178    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
179        Self::match_private_node_vm_and_gateway_vm(
180            &self.symmetric_private_node_vms,
181            &self.symmetric_nat_gateway_vms,
182        )
183    }
184
185    pub fn full_cone_private_node_and_gateway_map(
186        &self,
187    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
188        Self::match_private_node_vm_and_gateway_vm(
189            &self.full_cone_private_node_vms,
190            &self.full_cone_nat_gateway_vms,
191        )
192    }
193
194    pub fn match_private_node_vm_and_gateway_vm(
195        private_node_vms: &[VirtualMachine],
196        nat_gateway_vms: &[VirtualMachine],
197    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
198        if private_node_vms.len() != nat_gateway_vms.len() {
199            println!(
200            "The number of private node VMs ({}) does not match the number of NAT Gateway VMs ({})",
201            private_node_vms.len(),
202            nat_gateway_vms.len()
203        );
204            error!("The number of private node VMs does not match the number of NAT Gateway VMs: Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}");
205            return Err(Error::VmCountMismatch(None, None));
206        }
207
208        let mut map = HashMap::new();
209        for private_vm in private_node_vms {
210            let nat_gateway = nat_gateway_vms
211                .iter()
212                .find(|vm| {
213                    let private_node_name = private_vm.name.split('-').last().unwrap();
214                    let nat_gateway_name = vm.name.split('-').last().unwrap();
215                    private_node_name == nat_gateway_name
216                })
217                .ok_or_else(|| {
218                    println!(
219                        "Failed to find a matching NAT Gateway for private node: {}",
220                        private_vm.name
221                    );
222                    error!("Failed to find a matching NAT Gateway for private node: {}. Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}", private_vm.name);
223                    Error::VmCountMismatch(None, None)
224                })?;
225
226            let _ = map.insert(private_vm.clone(), nat_gateway.clone());
227        }
228
229        Ok(map)
230    }
231}
232
233impl From<BootstrapOptions> for ProvisionOptions {
234    fn from(bootstrap_options: BootstrapOptions) -> Self {
235        ProvisionOptions {
236            ant_version: None,
237            binary_option: bootstrap_options.binary_option,
238            chunk_size: bootstrap_options.chunk_size,
239            downloaders_count: 0,
240            enable_telegraf: true,
241            env_variables: bootstrap_options.env_variables,
242            evm_data_payments_address: bootstrap_options.evm_data_payments_address,
243            evm_network: bootstrap_options.evm_network,
244            evm_payment_token_address: bootstrap_options.evm_payment_token_address,
245            evm_rpc_url: bootstrap_options.evm_rpc_url,
246            full_cone_private_node_count: bootstrap_options.full_cone_private_node_count,
247            funding_wallet_secret_key: None,
248            gas_amount: None,
249            interval: bootstrap_options.interval,
250            log_format: bootstrap_options.log_format,
251            logstash_details: None,
252            max_archived_log_files: bootstrap_options.max_archived_log_files,
253            max_log_files: bootstrap_options.max_log_files,
254            name: bootstrap_options.name,
255            network_id: bootstrap_options.network_id,
256            node_count: bootstrap_options.node_count,
257            output_inventory_dir_path: bootstrap_options.output_inventory_dir_path,
258            peer_cache_node_count: 0,
259            symmetric_private_node_count: bootstrap_options.symmetric_private_node_count,
260            public_rpc: false,
261            rewards_address: bootstrap_options.rewards_address,
262            token_amount: None,
263            uploaders_count: None,
264        }
265    }
266}
267
268impl From<DeployOptions> for ProvisionOptions {
269    fn from(deploy_options: DeployOptions) -> Self {
270        ProvisionOptions {
271            ant_version: None,
272            binary_option: deploy_options.binary_option,
273            chunk_size: deploy_options.chunk_size,
274            downloaders_count: deploy_options.downloaders_count,
275            enable_telegraf: deploy_options.enable_telegraf,
276            env_variables: deploy_options.env_variables,
277            evm_data_payments_address: deploy_options.evm_data_payments_address,
278            evm_network: deploy_options.evm_network,
279            evm_payment_token_address: deploy_options.evm_payment_token_address,
280            evm_rpc_url: deploy_options.evm_rpc_url,
281            full_cone_private_node_count: deploy_options.full_cone_private_node_count,
282            funding_wallet_secret_key: deploy_options.funding_wallet_secret_key,
283            gas_amount: deploy_options.initial_gas,
284            interval: deploy_options.interval,
285            log_format: deploy_options.log_format,
286            logstash_details: deploy_options.logstash_details,
287            max_archived_log_files: deploy_options.max_archived_log_files,
288            max_log_files: deploy_options.max_log_files,
289            name: deploy_options.name,
290            network_id: deploy_options.network_id,
291            node_count: deploy_options.node_count,
292            output_inventory_dir_path: deploy_options.output_inventory_dir_path,
293            peer_cache_node_count: deploy_options.peer_cache_node_count,
294            symmetric_private_node_count: deploy_options.symmetric_private_node_count,
295            public_rpc: deploy_options.public_rpc,
296            rewards_address: deploy_options.rewards_address,
297            token_amount: deploy_options.initial_tokens,
298            uploaders_count: Some(deploy_options.uploaders_count),
299        }
300    }
301}
302
303#[derive(Clone)]
304pub struct AnsibleProvisioner {
305    pub ansible_runner: AnsibleRunner,
306    pub cloud_provider: CloudProvider,
307    pub ssh_client: SshClient,
308}
309
310impl AnsibleProvisioner {
311    pub fn new(
312        ansible_runner: AnsibleRunner,
313        cloud_provider: CloudProvider,
314        ssh_client: SshClient,
315    ) -> Self {
316        Self {
317            ansible_runner,
318            cloud_provider,
319            ssh_client,
320        }
321    }
322
323    pub fn build_safe_network_binaries(&self, options: &ProvisionOptions) -> Result<()> {
324        let start = Instant::now();
325        println!("Obtaining IP address for build VM...");
326        let build_inventory = self
327            .ansible_runner
328            .get_inventory(AnsibleInventoryType::Build, true)?;
329        let build_ip = build_inventory[0].public_ip_addr;
330        self.ssh_client
331            .wait_for_ssh_availability(&build_ip, &self.cloud_provider.get_ssh_user())?;
332
333        println!("Running ansible against build VM...");
334        let extra_vars = extra_vars::build_binaries_extra_vars_doc(options)?;
335        self.ansible_runner.run_playbook(
336            AnsiblePlaybook::Build,
337            AnsibleInventoryType::Build,
338            Some(extra_vars),
339        )?;
340        print_duration(start.elapsed());
341        Ok(())
342    }
343
344    pub fn cleanup_node_logs(&self, setup_cron: bool) -> Result<()> {
345        for node_inv_type in AnsibleInventoryType::iter_node_type() {
346            self.ansible_runner.run_playbook(
347                AnsiblePlaybook::CleanupLogs,
348                node_inv_type,
349                Some(format!("{{ \"setup_cron\": \"{setup_cron}\" }}")),
350            )?;
351        }
352
353        Ok(())
354    }
355
356    pub fn copy_logs(&self, name: &str, resources_only: bool) -> Result<()> {
357        for node_inv_type in AnsibleInventoryType::iter_node_type() {
358            self.ansible_runner.run_playbook(
359                AnsiblePlaybook::CopyLogs,
360                node_inv_type,
361                Some(format!(
362                    "{{ \"env_name\": \"{name}\", \"resources_only\" : \"{resources_only}\" }}"
363                )),
364            )?;
365        }
366        Ok(())
367    }
368
369    pub fn get_all_node_inventory(&self) -> Result<Vec<VirtualMachine>> {
370        let mut all_node_inventory = Vec::new();
371        for node_inv_type in AnsibleInventoryType::iter_node_type() {
372            all_node_inventory.extend(self.ansible_runner.get_inventory(node_inv_type, false)?);
373        }
374
375        Ok(all_node_inventory)
376    }
377
378    pub fn get_symmetric_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
379        self.ansible_runner
380            .get_inventory(AnsibleInventoryType::SymmetricNatGateway, false)
381    }
382
383    pub fn get_full_cone_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
384        self.ansible_runner
385            .get_inventory(AnsibleInventoryType::FullConeNatGateway, false)
386    }
387
388    pub fn get_node_registries(
389        &self,
390        inventory_type: &AnsibleInventoryType,
391    ) -> Result<DeploymentNodeRegistries> {
392        debug!("Fetching node manager inventory for {inventory_type:?}");
393        let temp_dir_path = tempfile::tempdir()?.into_path();
394        let temp_dir_json = serde_json::to_string(&temp_dir_path)?;
395
396        self.ansible_runner.run_playbook(
397            AnsiblePlaybook::AntCtlInventory,
398            *inventory_type,
399            Some(format!("{{ \"dest\": {temp_dir_json} }}")),
400        )?;
401
402        let node_registry_paths = WalkDir::new(temp_dir_path)
403            .into_iter()
404            .flatten()
405            .filter_map(|entry| {
406                if entry.file_type().is_file()
407                    && entry.path().extension().is_some_and(|ext| ext == "json")
408                {
409                    // tempdir/<testnet_name>-node/var/safenode-manager/node_registry.json
410                    let mut vm_name = entry.path().to_path_buf();
411                    trace!("Found file with json extension: {vm_name:?}");
412                    vm_name.pop();
413                    vm_name.pop();
414                    vm_name.pop();
415                    // Extract the <testnet_name>-node string
416                    trace!("Extracting the vm name from the path");
417                    let vm_name = vm_name.file_name()?.to_str()?;
418                    trace!("Extracted vm name from path: {vm_name}");
419                    Some((vm_name.to_string(), entry.path().to_path_buf()))
420                } else {
421                    None
422                }
423            })
424            .collect::<Vec<(String, PathBuf)>>();
425
426        let mut node_registries = Vec::new();
427        let mut failed_vms = Vec::new();
428        for (vm_name, file_path) in node_registry_paths {
429            match NodeRegistry::load(&file_path) {
430                Ok(node_registry) => node_registries.push((vm_name.clone(), node_registry)),
431                Err(_) => failed_vms.push(vm_name.clone()),
432            }
433        }
434
435        let deployment_registries = DeploymentNodeRegistries {
436            inventory_type: *inventory_type,
437            retrieved_registries: node_registries,
438            failed_vms,
439        };
440        Ok(deployment_registries)
441    }
442
443    pub fn provision_evm_nodes(&self, options: &ProvisionOptions) -> Result<()> {
444        let start = Instant::now();
445        println!("Obtaining IP address for EVM nodes...");
446        let evm_node_inventory = self
447            .ansible_runner
448            .get_inventory(AnsibleInventoryType::EvmNodes, true)?;
449        let evm_node_ip = evm_node_inventory[0].public_ip_addr;
450        self.ssh_client
451            .wait_for_ssh_availability(&evm_node_ip, &self.cloud_provider.get_ssh_user())?;
452
453        println!("Running ansible against EVM nodes...");
454        self.ansible_runner.run_playbook(
455            AnsiblePlaybook::EvmNodes,
456            AnsibleInventoryType::EvmNodes,
457            Some(extra_vars::build_evm_nodes_extra_vars_doc(
458                &options.name,
459                &self.cloud_provider,
460                &options.binary_option,
461            )),
462        )?;
463        print_duration(start.elapsed());
464        Ok(())
465    }
466
467    pub fn provision_genesis_node(&self, options: &ProvisionOptions) -> Result<()> {
468        let start = Instant::now();
469        let genesis_inventory = self
470            .ansible_runner
471            .get_inventory(AnsibleInventoryType::Genesis, true)?;
472        let genesis_ip = genesis_inventory[0].public_ip_addr;
473        self.ssh_client
474            .wait_for_ssh_availability(&genesis_ip, &self.cloud_provider.get_ssh_user())?;
475        self.ansible_runner.run_playbook(
476            AnsiblePlaybook::Genesis,
477            AnsibleInventoryType::Genesis,
478            Some(extra_vars::build_node_extra_vars_doc(
479                &self.cloud_provider.to_string(),
480                options,
481                NodeType::Genesis,
482                None,
483                None,
484                1,
485                options.evm_network.clone(),
486                false,
487            )?),
488        )?;
489
490        print_duration(start.elapsed());
491
492        Ok(())
493    }
494
495    pub fn provision_full_cone(
496        &self,
497        options: &ProvisionOptions,
498        initial_contact_peer: Option<String>,
499        initial_network_contacts_url: Option<String>,
500        private_node_inventory: PrivateNodeProvisionInventory,
501        new_full_cone_nat_gateway_new_vms_for_upscale: Option<Vec<VirtualMachine>>,
502    ) -> Result<()> {
503        // Step 1 of Full Cone NAT Gateway
504        let start = Instant::now();
505        self.print_ansible_run_banner("Provision Full Cone NAT Gateway - Step 1");
506
507        for vm in new_full_cone_nat_gateway_new_vms_for_upscale
508            .as_ref()
509            .unwrap_or(&private_node_inventory.full_cone_nat_gateway_vms)
510            .iter()
511        {
512            println!(
513                "Checking SSH availability for Full Cone NAT Gateway: {}",
514                vm.public_ip_addr
515            );
516            self.ssh_client
517                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
518                .map_err(|e| {
519                    println!(
520                        "Failed to establish SSH connection to Full Cone NAT Gateway: {}",
521                        e
522                    );
523                    e
524                })?;
525        }
526
527        let mut modified_private_node_inventory = private_node_inventory.clone();
528
529        // If we are upscaling, then we cannot access the gateway VMs which are already deployed.
530        if let Some(new_full_cone_nat_gateway_new_vms_for_upscale) =
531            &new_full_cone_nat_gateway_new_vms_for_upscale
532        {
533            debug!("Removing existing full cone NAT Gateway and private node VMs from the inventory. Old inventory: {modified_private_node_inventory:?}");
534            let mut names_to_keep = Vec::new();
535
536            for vm in new_full_cone_nat_gateway_new_vms_for_upscale.iter() {
537                let nat_gateway_name = vm.name.split('-').last().unwrap();
538                names_to_keep.push(nat_gateway_name);
539            }
540
541            modified_private_node_inventory
542                .full_cone_nat_gateway_vms
543                .retain(|vm| {
544                    let nat_gateway_name = vm.name.split('-').last().unwrap();
545                    names_to_keep.contains(&nat_gateway_name)
546                });
547            modified_private_node_inventory
548                .full_cone_private_node_vms
549                .retain(|vm| {
550                    let nat_gateway_name = vm.name.split('-').last().unwrap();
551                    names_to_keep.contains(&nat_gateway_name)
552                });
553            debug!("New inventory after removing existing full cone NAT Gateway and private node VMs: {modified_private_node_inventory:?}");
554        }
555
556        if modified_private_node_inventory
557            .full_cone_nat_gateway_vms
558            .is_empty()
559        {
560            error!("There are no full cone NAT Gateway VMs available to upscale");
561            return Ok(());
562        }
563
564        let private_node_ip_map = modified_private_node_inventory
565            .full_cone_private_node_and_gateway_map()?
566            .into_iter()
567            .map(|(k, v)| {
568                let gateway_name = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
569                    debug!("Upscaling, using public IP address for gateway name");
570                    v.public_ip_addr.to_string()
571                } else {
572                    v.name.clone()
573                };
574                (gateway_name, k.private_ip_addr)
575            })
576            .collect::<HashMap<String, IpAddr>>();
577
578        if private_node_ip_map.is_empty() {
579            println!("There are no full cone private node VM available to be routed through the full cone NAT Gateway");
580            return Err(Error::EmptyInventory(
581                AnsibleInventoryType::FullConePrivateNodes,
582            ));
583        }
584
585        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
586            &options.name,
587            private_node_ip_map.clone(),
588            "step1",
589        );
590        debug!("Provisioning Full Cone NAT Gateway - Step 1 with vars: {vars}");
591        let gateway_inventory = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
592            debug!("Upscaling, using static inventory for full cone nat gateway.");
593            generate_full_cone_nat_gateway_static_environment_inventory(
594                &modified_private_node_inventory.full_cone_nat_gateway_vms,
595                &options.name,
596                &options.output_inventory_dir_path,
597            )?;
598
599            AnsibleInventoryType::FullConeNatGatewayStatic
600        } else {
601            AnsibleInventoryType::FullConeNatGateway
602        };
603        self.ansible_runner.run_playbook(
604            AnsiblePlaybook::FullConeNatGateway,
605            gateway_inventory,
606            Some(vars),
607        )?;
608
609        // setup private node config
610        self.print_ansible_run_banner("Provisioning Full Cone Private Node Config");
611
612        generate_full_cone_private_node_static_environment_inventory(
613            &options.name,
614            &options.output_inventory_dir_path,
615            &private_node_inventory.full_cone_private_node_vms,
616            &private_node_inventory.full_cone_nat_gateway_vms,
617            &self.ssh_client.private_key_path,
618        )
619        .inspect_err(|err| {
620            error!("Failed to generate full cone private node static inv with err: {err:?}")
621        })?;
622
623        // For a new deployment, it's quite probable that SSH is available, because this part occurs
624        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
625        // check that SSH is available before proceeding.
626        println!("Obtaining IP addresses for nodes...");
627        let inventory = self
628            .ansible_runner
629            .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)?;
630
631        println!("Waiting for SSH availability on Symmetric Private nodes...");
632        for vm in inventory.iter() {
633            println!(
634                "Checking SSH availability for {}: {}",
635                vm.name, vm.public_ip_addr
636            );
637            self.ssh_client
638                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
639                .map_err(|e| {
640                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
641                    e
642                })?;
643        }
644
645        println!("SSH is available on all nodes. Proceeding with provisioning...");
646
647        self.ansible_runner.run_playbook(
648            AnsiblePlaybook::PrivateNodeConfig,
649            AnsibleInventoryType::FullConePrivateNodes,
650            Some(
651                extra_vars::build_full_cone_private_node_config_extra_vars_docs(
652                    &private_node_inventory,
653                )?,
654            ),
655        )?;
656
657        // Step 2 of Full Cone NAT Gateway
658
659        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
660            &options.name,
661            private_node_ip_map,
662            "step2",
663        );
664
665        self.print_ansible_run_banner("Provisioning Full Cone NAT Gateway - Step 2");
666        debug!("Provisioning Full Cone NAT Gateway - Step 2 with vars: {vars}");
667        self.ansible_runner.run_playbook(
668            AnsiblePlaybook::FullConeNatGateway,
669            gateway_inventory,
670            Some(vars),
671        )?;
672
673        // provision the nodes
674
675        let home_dir = std::env::var("HOME").inspect_err(|err| {
676            println!("Failed to get home directory with error: {err:?}",);
677        })?;
678        let known_hosts_path = format!("{}/.ssh/known_hosts", home_dir);
679        debug!("Cleaning up known hosts file at {known_hosts_path} ");
680        run_external_command(
681            PathBuf::from("rm"),
682            std::env::current_dir()?,
683            vec![known_hosts_path],
684            false,
685            false,
686        )?;
687
688        self.print_ansible_run_banner("Provision Full Cone Private Nodes");
689
690        self.ssh_client.set_full_cone_nat_routed_vms(
691            &private_node_inventory.full_cone_private_node_vms,
692            &private_node_inventory.full_cone_nat_gateway_vms,
693        )?;
694
695        self.provision_nodes(
696            options,
697            initial_contact_peer,
698            initial_network_contacts_url,
699            NodeType::FullConePrivateNode,
700        )?;
701
702        print_duration(start.elapsed());
703        Ok(())
704    }
705    pub fn provision_symmetric_nat_gateway(
706        &self,
707        options: &ProvisionOptions,
708        private_node_inventory: &PrivateNodeProvisionInventory,
709    ) -> Result<()> {
710        let start = Instant::now();
711        for vm in &private_node_inventory.symmetric_nat_gateway_vms {
712            println!(
713                "Checking SSH availability for Symmetric NAT Gateway: {}",
714                vm.public_ip_addr
715            );
716            self.ssh_client
717                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
718                .map_err(|e| {
719                    println!(
720                        "Failed to establish SSH connection to Symmetric NAT Gateway: {}",
721                        e
722                    );
723                    e
724                })?;
725        }
726
727        let private_node_ip_map = private_node_inventory
728            .symmetric_private_node_and_gateway_map()?
729            .into_iter()
730            .map(|(k, v)| (v.name.clone(), k.private_ip_addr))
731            .collect::<HashMap<String, IpAddr>>();
732
733        if private_node_ip_map.is_empty() {
734            println!("There are no Symmetric private node VM available to be routed through the Symmetric NAT Gateway");
735            return Err(Error::EmptyInventory(
736                AnsibleInventoryType::SymmetricPrivateNodes,
737            ));
738        }
739
740        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
741            &options.name,
742            private_node_ip_map,
743            "symmetric",
744        );
745        debug!("Provisioning Symmetric NAT Gateway with vars: {vars}");
746        self.ansible_runner.run_playbook(
747            AnsiblePlaybook::SymmetricNatGateway,
748            AnsibleInventoryType::SymmetricNatGateway,
749            Some(vars),
750        )?;
751
752        print_duration(start.elapsed());
753        Ok(())
754    }
755
756    pub fn provision_nodes(
757        &self,
758        options: &ProvisionOptions,
759        initial_contact_peer: Option<String>,
760        initial_network_contacts_url: Option<String>,
761        node_type: NodeType,
762    ) -> Result<()> {
763        let start = Instant::now();
764        let mut relay = false;
765        let (inventory_type, node_count) = match &node_type {
766            NodeType::FullConePrivateNode => {
767                relay = true;
768                (
769                    node_type.to_ansible_inventory_type(),
770                    options.full_cone_private_node_count,
771                )
772            }
773            // use provision_genesis_node fn
774            NodeType::Generic => (node_type.to_ansible_inventory_type(), options.node_count),
775            NodeType::Genesis => return Err(Error::InvalidNodeType(node_type)),
776            NodeType::PeerCache => (
777                node_type.to_ansible_inventory_type(),
778                options.peer_cache_node_count,
779            ),
780            NodeType::SymmetricPrivateNode => {
781                relay = true;
782                (
783                    node_type.to_ansible_inventory_type(),
784                    options.symmetric_private_node_count,
785                )
786            }
787        };
788
789        // For a new deployment, it's quite probable that SSH is available, because this part occurs
790        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
791        // check that SSH is available before proceeding.
792        println!("Obtaining IP addresses for nodes...");
793        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
794
795        println!("Waiting for SSH availability on {node_type:?} nodes...");
796        for vm in inventory.iter() {
797            println!(
798                "Checking SSH availability for {}: {}",
799                vm.name, vm.public_ip_addr
800            );
801            self.ssh_client
802                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
803                .map_err(|e| {
804                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
805                    e
806                })?;
807        }
808
809        println!("SSH is available on all nodes. Proceeding with provisioning...");
810
811        let playbook = match node_type {
812            NodeType::Generic => AnsiblePlaybook::Nodes,
813            NodeType::PeerCache => AnsiblePlaybook::PeerCacheNodes,
814            NodeType::FullConePrivateNode => AnsiblePlaybook::Nodes,
815            NodeType::SymmetricPrivateNode => AnsiblePlaybook::Nodes,
816            _ => return Err(Error::InvalidNodeType(node_type.clone())),
817        };
818        self.ansible_runner.run_playbook(
819            playbook,
820            inventory_type,
821            Some(extra_vars::build_node_extra_vars_doc(
822                &self.cloud_provider.to_string(),
823                options,
824                node_type.clone(),
825                initial_contact_peer,
826                initial_network_contacts_url,
827                node_count,
828                options.evm_network.clone(),
829                relay,
830            )?),
831        )?;
832
833        print_duration(start.elapsed());
834        Ok(())
835    }
836
837    pub fn provision_symmetric_private_nodes(
838        &self,
839        options: &mut ProvisionOptions,
840        initial_contact_peer: Option<String>,
841        initial_network_contacts_url: Option<String>,
842        private_node_inventory: &PrivateNodeProvisionInventory,
843    ) -> Result<()> {
844        let start = Instant::now();
845        self.print_ansible_run_banner("Provision Symmetric Private Node Config");
846
847        generate_symmetric_private_node_static_environment_inventory(
848            &options.name,
849            &options.output_inventory_dir_path,
850            &private_node_inventory.symmetric_private_node_vms,
851            &private_node_inventory.symmetric_nat_gateway_vms,
852            &self.ssh_client.private_key_path,
853        )
854        .inspect_err(|err| {
855            error!("Failed to generate symmetric private node static inv with err: {err:?}")
856        })?;
857
858        self.ssh_client.set_symmetric_nat_routed_vms(
859            &private_node_inventory.symmetric_private_node_vms,
860            &private_node_inventory.symmetric_nat_gateway_vms,
861        )?;
862
863        let inventory_type = AnsibleInventoryType::SymmetricPrivateNodes;
864
865        // For a new deployment, it's quite probable that SSH is available, because this part occurs
866        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
867        // check that SSH is available before proceeding.
868        println!("Obtaining IP addresses for nodes...");
869        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
870
871        println!("Waiting for SSH availability on Symmetric Private nodes...");
872        for vm in inventory.iter() {
873            println!(
874                "Checking SSH availability for {}: {}",
875                vm.name, vm.public_ip_addr
876            );
877            self.ssh_client
878                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
879                .map_err(|e| {
880                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
881                    e
882                })?;
883        }
884
885        println!("SSH is available on all nodes. Proceeding with provisioning...");
886
887        self.ansible_runner.run_playbook(
888            AnsiblePlaybook::PrivateNodeConfig,
889            inventory_type,
890            Some(
891                extra_vars::build_symmetric_private_node_config_extra_vars_doc(
892                    private_node_inventory,
893                )?,
894            ),
895        )?;
896
897        println!("Provisioned Symmetric Private Node Config");
898        print_duration(start.elapsed());
899
900        self.provision_nodes(
901            options,
902            initial_contact_peer,
903            initial_network_contacts_url,
904            NodeType::SymmetricPrivateNode,
905        )?;
906
907        Ok(())
908    }
909
910    pub async fn provision_uploaders(
911        &self,
912        options: &ProvisionOptions,
913        genesis_multiaddr: Option<String>,
914        genesis_network_contacts_url: Option<String>,
915    ) -> Result<()> {
916        let start = Instant::now();
917
918        let sk_map = self
919            .deposit_funds_to_uploaders(&FundingOptions {
920                evm_data_payments_address: options.evm_data_payments_address.clone(),
921                evm_network: options.evm_network.clone(),
922                evm_payment_token_address: options.evm_payment_token_address.clone(),
923                evm_rpc_url: options.evm_rpc_url.clone(),
924                funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
925                gas_amount: options.gas_amount,
926                token_amount: options.token_amount,
927                uploaders_count: options.uploaders_count,
928            })
929            .await?;
930
931        println!("Running ansible against uploader machine to start the uploader script.");
932        debug!("Running ansible against uploader machine to start the uploader script.");
933
934        self.ansible_runner.run_playbook(
935            AnsiblePlaybook::Uploaders,
936            AnsibleInventoryType::Uploaders,
937            Some(extra_vars::build_uploaders_extra_vars_doc(
938                &self.cloud_provider.to_string(),
939                options,
940                genesis_multiaddr,
941                genesis_network_contacts_url,
942                &sk_map,
943            )?),
944        )?;
945        print_duration(start.elapsed());
946        Ok(())
947    }
948
949    pub fn start_nodes(
950        &self,
951        environment_name: &str,
952        interval: Duration,
953        node_type: Option<NodeType>,
954        custom_inventory: Option<Vec<VirtualMachine>>,
955    ) -> Result<()> {
956        let mut extra_vars = ExtraVarsDocBuilder::default();
957        extra_vars.add_variable("interval", &interval.as_millis().to_string());
958
959        if let Some(node_type) = node_type {
960            println!("Running the start nodes playbook for {node_type:?} nodes");
961            self.ansible_runner.run_playbook(
962                AnsiblePlaybook::StartNodes,
963                node_type.to_ansible_inventory_type(),
964                Some(extra_vars.build()),
965            )?;
966            return Ok(());
967        }
968
969        if let Some(custom_inventory) = custom_inventory {
970            println!("Running the start nodes playbook with a custom inventory");
971            generate_custom_environment_inventory(
972                &custom_inventory,
973                environment_name,
974                &self.ansible_runner.working_directory_path.join("inventory"),
975            )?;
976            self.ansible_runner.run_playbook(
977                AnsiblePlaybook::StartNodes,
978                AnsibleInventoryType::Custom,
979                Some(extra_vars.build()),
980            )?;
981            return Ok(());
982        }
983
984        println!("Running the start nodes playbook for all node types");
985        for node_inv_type in AnsibleInventoryType::iter_node_type() {
986            self.ansible_runner.run_playbook(
987                AnsiblePlaybook::StartNodes,
988                node_inv_type,
989                Some(extra_vars.build()),
990            )?;
991        }
992        Ok(())
993    }
994
995    pub fn status(&self) -> Result<()> {
996        for node_inv_type in AnsibleInventoryType::iter_node_type() {
997            self.ansible_runner
998                .run_playbook(AnsiblePlaybook::Status, node_inv_type, None)?;
999        }
1000        Ok(())
1001    }
1002
1003    pub fn start_telegraf(
1004        &self,
1005        environment_name: &str,
1006        node_type: Option<NodeType>,
1007        custom_inventory: Option<Vec<VirtualMachine>>,
1008    ) -> Result<()> {
1009        if let Some(node_type) = node_type {
1010            println!("Running the start telegraf playbook for {node_type:?} nodes");
1011            self.ansible_runner.run_playbook(
1012                AnsiblePlaybook::StartTelegraf,
1013                node_type.to_ansible_inventory_type(),
1014                None,
1015            )?;
1016            return Ok(());
1017        }
1018
1019        if let Some(custom_inventory) = custom_inventory {
1020            println!("Running the start telegraf playbook with a custom inventory");
1021            generate_custom_environment_inventory(
1022                &custom_inventory,
1023                environment_name,
1024                &self.ansible_runner.working_directory_path.join("inventory"),
1025            )?;
1026            self.ansible_runner.run_playbook(
1027                AnsiblePlaybook::StartTelegraf,
1028                AnsibleInventoryType::Custom,
1029                None,
1030            )?;
1031            return Ok(());
1032        }
1033
1034        println!("Running the start telegraf playbook for all node types");
1035        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1036            self.ansible_runner.run_playbook(
1037                AnsiblePlaybook::StartTelegraf,
1038                node_inv_type,
1039                None,
1040            )?;
1041        }
1042
1043        Ok(())
1044    }
1045
1046    pub fn stop_nodes(
1047        &self,
1048        environment_name: &str,
1049        interval: Duration,
1050        node_type: Option<NodeType>,
1051        custom_inventory: Option<Vec<VirtualMachine>>,
1052        delay: Option<u64>,
1053        service_names: Option<Vec<String>>,
1054    ) -> Result<()> {
1055        let mut extra_vars = ExtraVarsDocBuilder::default();
1056        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1057        if let Some(delay) = delay {
1058            extra_vars.add_variable("delay", &delay.to_string());
1059        }
1060        if let Some(service_names) = service_names {
1061            extra_vars.add_list_variable("service_names", service_names);
1062        }
1063        let extra_vars = extra_vars.build();
1064
1065        if let Some(node_type) = node_type {
1066            println!("Running the stop nodes playbook for {node_type:?} nodes");
1067            self.ansible_runner.run_playbook(
1068                AnsiblePlaybook::StopNodes,
1069                node_type.to_ansible_inventory_type(),
1070                Some(extra_vars),
1071            )?;
1072            return Ok(());
1073        }
1074
1075        if let Some(custom_inventory) = custom_inventory {
1076            println!("Running the stop nodes playbook with a custom inventory");
1077            generate_custom_environment_inventory(
1078                &custom_inventory,
1079                environment_name,
1080                &self.ansible_runner.working_directory_path.join("inventory"),
1081            )?;
1082            self.ansible_runner.run_playbook(
1083                AnsiblePlaybook::StopNodes,
1084                AnsibleInventoryType::Custom,
1085                Some(extra_vars),
1086            )?;
1087            return Ok(());
1088        }
1089
1090        println!("Running the stop nodes playbook for all node types");
1091        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1092            self.ansible_runner.run_playbook(
1093                AnsiblePlaybook::StopNodes,
1094                node_inv_type,
1095                Some(extra_vars.clone()),
1096            )?;
1097        }
1098
1099        Ok(())
1100    }
1101
1102    pub fn stop_telegraf(
1103        &self,
1104        environment_name: &str,
1105        node_type: Option<NodeType>,
1106        custom_inventory: Option<Vec<VirtualMachine>>,
1107    ) -> Result<()> {
1108        if let Some(node_type) = node_type {
1109            println!("Running the stop telegraf playbook for {node_type:?} nodes");
1110            self.ansible_runner.run_playbook(
1111                AnsiblePlaybook::StopTelegraf,
1112                node_type.to_ansible_inventory_type(),
1113                None,
1114            )?;
1115            return Ok(());
1116        }
1117
1118        if let Some(custom_inventory) = custom_inventory {
1119            println!("Running the stop telegraf playbook with a custom inventory");
1120            generate_custom_environment_inventory(
1121                &custom_inventory,
1122                environment_name,
1123                &self.ansible_runner.working_directory_path.join("inventory"),
1124            )?;
1125            self.ansible_runner.run_playbook(
1126                AnsiblePlaybook::StopTelegraf,
1127                AnsibleInventoryType::Custom,
1128                None,
1129            )?;
1130            return Ok(());
1131        }
1132
1133        println!("Running the stop telegraf playbook for all node types");
1134        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1135            self.ansible_runner
1136                .run_playbook(AnsiblePlaybook::StopTelegraf, node_inv_type, None)?;
1137        }
1138
1139        Ok(())
1140    }
1141
1142    pub fn upgrade_node_telegraf(&self, name: &str) -> Result<()> {
1143        self.ansible_runner.run_playbook(
1144            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1145            AnsibleInventoryType::PeerCacheNodes,
1146            Some(extra_vars::build_node_telegraf_upgrade(
1147                name,
1148                &NodeType::PeerCache,
1149            )?),
1150        )?;
1151        self.ansible_runner.run_playbook(
1152            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1153            AnsibleInventoryType::Nodes,
1154            Some(extra_vars::build_node_telegraf_upgrade(
1155                name,
1156                &NodeType::Generic,
1157            )?),
1158        )?;
1159
1160        self.ansible_runner.run_playbook(
1161            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1162            AnsibleInventoryType::SymmetricPrivateNodes,
1163            Some(extra_vars::build_node_telegraf_upgrade(
1164                name,
1165                &NodeType::SymmetricPrivateNode,
1166            )?),
1167        )?;
1168
1169        self.ansible_runner.run_playbook(
1170            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1171            AnsibleInventoryType::FullConePrivateNodes,
1172            Some(extra_vars::build_node_telegraf_upgrade(
1173                name,
1174                &NodeType::FullConePrivateNode,
1175            )?),
1176        )?;
1177        Ok(())
1178    }
1179
1180    pub fn upgrade_uploader_telegraf(&self, name: &str) -> Result<()> {
1181        self.ansible_runner.run_playbook(
1182            AnsiblePlaybook::UpgradeUploaderTelegrafConfig,
1183            AnsibleInventoryType::Uploaders,
1184            Some(extra_vars::build_uploader_telegraf_upgrade(name)?),
1185        )?;
1186        Ok(())
1187    }
1188
1189    pub fn upgrade_nodes(&self, options: &UpgradeOptions) -> Result<()> {
1190        if let Some(custom_inventory) = &options.custom_inventory {
1191            println!("Running the UpgradeNodes with a custom inventory");
1192            generate_custom_environment_inventory(
1193                custom_inventory,
1194                &options.name,
1195                &self.ansible_runner.working_directory_path.join("inventory"),
1196            )?;
1197            match self.ansible_runner.run_playbook(
1198                AnsiblePlaybook::UpgradeNodes,
1199                AnsibleInventoryType::Custom,
1200                Some(options.get_ansible_vars()),
1201            ) {
1202                Ok(()) => println!("All nodes were successfully upgraded"),
1203                Err(_) => {
1204                    println!("WARNING: some nodes may not have been upgraded or restarted");
1205                }
1206            }
1207            return Ok(());
1208        }
1209
1210        if let Some(node_type) = &options.node_type {
1211            println!("Running the UpgradeNodes playbook for {node_type:?} nodes");
1212            match self.ansible_runner.run_playbook(
1213                AnsiblePlaybook::UpgradeNodes,
1214                node_type.to_ansible_inventory_type(),
1215                Some(options.get_ansible_vars()),
1216            ) {
1217                Ok(()) => println!("All {node_type:?} nodes were successfully upgraded"),
1218                Err(_) => {
1219                    println!(
1220                        "WARNING: some {node_type:?} nodes may not have been upgraded or restarted"
1221                    );
1222                }
1223            }
1224            return Ok(());
1225        }
1226
1227        println!("Running the UpgradeNodes playbook for all node types");
1228
1229        match self.ansible_runner.run_playbook(
1230            AnsiblePlaybook::UpgradeNodes,
1231            AnsibleInventoryType::PeerCacheNodes,
1232            Some(options.get_ansible_vars()),
1233        ) {
1234            Ok(()) => println!("All Peer Cache nodes were successfully upgraded"),
1235            Err(_) => {
1236                println!("WARNING: some Peer Cacche nodes may not have been upgraded or restarted");
1237            }
1238        }
1239        match self.ansible_runner.run_playbook(
1240            AnsiblePlaybook::UpgradeNodes,
1241            AnsibleInventoryType::Nodes,
1242            Some(options.get_ansible_vars()),
1243        ) {
1244            Ok(()) => println!("All generic nodes were successfully upgraded"),
1245            Err(_) => {
1246                println!("WARNING: some nodes may not have been upgraded or restarted");
1247            }
1248        }
1249        match self.ansible_runner.run_playbook(
1250            AnsiblePlaybook::UpgradeNodes,
1251            AnsibleInventoryType::SymmetricPrivateNodes,
1252            Some(options.get_ansible_vars()),
1253        ) {
1254            Ok(()) => println!("All private nodes were successfully upgraded"),
1255            Err(_) => {
1256                println!("WARNING: some nodes may not have been upgraded or restarted");
1257            }
1258        }
1259        // Don't use AnsibleInventoryType::iter_node_type() here, because the genesis node should be upgraded last
1260        match self.ansible_runner.run_playbook(
1261            AnsiblePlaybook::UpgradeNodes,
1262            AnsibleInventoryType::Genesis,
1263            Some(options.get_ansible_vars()),
1264        ) {
1265            Ok(()) => println!("The genesis nodes was successfully upgraded"),
1266            Err(_) => {
1267                println!("WARNING: the genesis node may not have been upgraded or restarted");
1268            }
1269        }
1270        Ok(())
1271    }
1272
1273    pub fn upgrade_antctl(
1274        &self,
1275        environment_name: &str,
1276        version: &Version,
1277        node_type: Option<NodeType>,
1278        custom_inventory: Option<Vec<VirtualMachine>>,
1279    ) -> Result<()> {
1280        let mut extra_vars = ExtraVarsDocBuilder::default();
1281        extra_vars.add_variable("version", &version.to_string());
1282
1283        if let Some(node_type) = node_type {
1284            println!("Running the upgrade safenode-manager playbook for {node_type:?} nodes");
1285            self.ansible_runner.run_playbook(
1286                AnsiblePlaybook::UpgradeAntctl,
1287                node_type.to_ansible_inventory_type(),
1288                Some(extra_vars.build()),
1289            )?;
1290            return Ok(());
1291        }
1292
1293        if let Some(custom_inventory) = custom_inventory {
1294            println!("Running the upgrade safenode-manager playbook with a custom inventory");
1295            generate_custom_environment_inventory(
1296                &custom_inventory,
1297                environment_name,
1298                &self.ansible_runner.working_directory_path.join("inventory"),
1299            )?;
1300            self.ansible_runner.run_playbook(
1301                AnsiblePlaybook::UpgradeAntctl,
1302                AnsibleInventoryType::Custom,
1303                Some(extra_vars.build()),
1304            )?;
1305            return Ok(());
1306        }
1307
1308        println!("Running the upgrade safenode-manager playbook for all node types");
1309        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1310            self.ansible_runner.run_playbook(
1311                AnsiblePlaybook::UpgradeAntctl,
1312                node_inv_type,
1313                Some(extra_vars.build()),
1314            )?;
1315        }
1316
1317        Ok(())
1318    }
1319
1320    pub fn print_ansible_run_banner(&self, s: &str) {
1321        let ansible_run_msg = "Ansible Run: ";
1322        let line = "=".repeat(s.len() + ansible_run_msg.len());
1323        println!("{}\n{}{}\n{}", line, ansible_run_msg, s, line);
1324    }
1325}
sn_testnet_deploy/ansible/provisioning.rs

sn_testnet_deploy/ansible/
provisioning.rs
