sn_testnet_deploy/ansible/
provisioning.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use super::{
8    extra_vars::ExtraVarsDocBuilder,
9    inventory::{
10        generate_full_cone_private_node_static_environment_inventory,
11        generate_symmetric_private_node_static_environment_inventory,
12    },
13    AnsibleInventoryType, AnsiblePlaybook, AnsibleRunner,
14};
15use crate::{
16    ansible::inventory::{
17        generate_custom_environment_inventory,
18        generate_full_cone_nat_gateway_static_environment_inventory,
19    },
20    bootstrap::BootstrapOptions,
21    deploy::DeployOptions,
22    error::{Error, Result},
23    funding::FundingOptions,
24    inventory::{DeploymentNodeRegistries, VirtualMachine},
25    print_duration, run_external_command, BinaryOption, CloudProvider, EvmNetwork, LogFormat,
26    NodeType, SshClient, UpgradeOptions,
27};
28use ant_service_management::NodeRegistry;
29use evmlib::common::U256;
30use log::{debug, error, trace};
31use semver::Version;
32use std::{
33    collections::HashMap,
34    net::{IpAddr, SocketAddr},
35    path::PathBuf,
36    time::{Duration, Instant},
37};
38use walkdir::WalkDir;
39
40use crate::ansible::extra_vars;
41
42pub const DEFAULT_BETA_ENCRYPTION_KEY: &str =
43    "49113d2083f57a976076adbe85decb75115820de1e6e74b47e0429338cef124a";
44
45#[derive(Clone)]
46pub struct ProvisionOptions {
47    /// The safe version is also in the binary option, but only for an initial deployment.
48    /// For the upscale, it needs to be provided explicitly, because currently it is not
49    /// recorded in the inventory.
50    pub ant_version: Option<String>,
51    pub binary_option: BinaryOption,
52    pub chunk_size: Option<u64>,
53    pub downloaders_count: u16,
54    pub env_variables: Option<Vec<(String, String)>>,
55    pub evm_data_payments_address: Option<String>,
56    pub evm_network: EvmNetwork,
57    pub evm_payment_token_address: Option<String>,
58    pub evm_rpc_url: Option<String>,
59    pub full_cone_private_node_count: u16,
60    pub funding_wallet_secret_key: Option<String>,
61    pub gas_amount: Option<U256>,
62    pub interval: Duration,
63    pub log_format: Option<LogFormat>,
64    pub logstash_details: Option<(String, Vec<SocketAddr>)>,
65    pub max_archived_log_files: u16,
66    pub max_log_files: u16,
67    pub name: String,
68    pub network_id: Option<u8>,
69    pub node_count: u16,
70    pub output_inventory_dir_path: PathBuf,
71    pub peer_cache_node_count: u16,
72    pub symmetric_private_node_count: u16,
73    pub public_rpc: bool,
74    pub rewards_address: String,
75    pub token_amount: Option<U256>,
76    pub uploaders_count: Option<u16>,
77}
78
79/// These are obtained by running the inventory playbook
80#[derive(Clone, Debug)]
81pub struct PrivateNodeProvisionInventory {
82    pub full_cone_nat_gateway_vms: Vec<VirtualMachine>,
83    pub full_cone_private_node_vms: Vec<VirtualMachine>,
84    pub symmetric_nat_gateway_vms: Vec<VirtualMachine>,
85    pub symmetric_private_node_vms: Vec<VirtualMachine>,
86}
87
88impl PrivateNodeProvisionInventory {
89    pub fn new(
90        provisioner: &AnsibleProvisioner,
91        full_cone_private_node_vm_count: Option<u16>,
92        symmetric_private_node_vm_count: Option<u16>,
93    ) -> Result<Self> {
94        // All the environment types set private_node_vm count to >0 if not specified.
95        let should_provision_full_cone_private_nodes = full_cone_private_node_vm_count
96            .map(|count| count > 0)
97            .unwrap_or(true);
98        let should_provision_symmetric_private_nodes = symmetric_private_node_vm_count
99            .map(|count| count > 0)
100            .unwrap_or(true);
101
102        let mut inventory = Self {
103            full_cone_nat_gateway_vms: Default::default(),
104            full_cone_private_node_vms: Default::default(),
105            symmetric_nat_gateway_vms: Default::default(),
106            symmetric_private_node_vms: Default::default(),
107        };
108
109        if should_provision_full_cone_private_nodes {
110            let full_cone_private_node_vms = provisioner
111                .ansible_runner
112                .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)
113                .inspect_err(|err| {
114                    println!("Failed to obtain the inventory of Full Cone private node: {err:?}");
115                })?;
116
117            let full_cone_nat_gateway_inventory = provisioner
118                .ansible_runner
119                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)
120                .inspect_err(|err| {
121                    println!("Failed to get Full Cone NAT Gateway inventory {err:?}");
122                })?;
123
124            if full_cone_nat_gateway_inventory.len() != full_cone_private_node_vms.len() {
125                println!("The number of Full Cone private nodes does not match the number of Full Cone NAT Gateway VMs");
126                return Err(Error::VmCountMismatch(
127                    Some(AnsibleInventoryType::FullConePrivateNodes),
128                    Some(AnsibleInventoryType::FullConeNatGateway),
129                ));
130            }
131
132            inventory.full_cone_private_node_vms = full_cone_private_node_vms;
133            inventory.full_cone_nat_gateway_vms = full_cone_nat_gateway_inventory;
134        }
135
136        if should_provision_symmetric_private_nodes {
137            let symmetric_private_node_vms = provisioner
138                .ansible_runner
139                .get_inventory(AnsibleInventoryType::SymmetricPrivateNodes, true)
140                .inspect_err(|err| {
141                    println!("Failed to obtain the inventory of Symmetric private node: {err:?}");
142                })?;
143
144            let symmetric_nat_gateway_inventory = provisioner
145                .ansible_runner
146                .get_inventory(AnsibleInventoryType::SymmetricNatGateway, true)
147                .inspect_err(|err| {
148                    println!("Failed to get Symmetric NAT Gateway inventory {err:?}");
149                })?;
150
151            if symmetric_nat_gateway_inventory.len() != symmetric_private_node_vms.len() {
152                println!("The number of Symmetric private nodes does not match the number of Symmetric NAT Gateway VMs");
153                return Err(Error::VmCountMismatch(
154                    Some(AnsibleInventoryType::SymmetricPrivateNodes),
155                    Some(AnsibleInventoryType::SymmetricNatGateway),
156                ));
157            }
158
159            inventory.symmetric_private_node_vms = symmetric_private_node_vms;
160            inventory.symmetric_nat_gateway_vms = symmetric_nat_gateway_inventory;
161        }
162
163        Ok(inventory)
164    }
165
166    pub fn should_provision_full_cone_private_nodes(&self) -> bool {
167        !self.full_cone_private_node_vms.is_empty()
168    }
169
170    pub fn should_provision_symmetric_private_nodes(&self) -> bool {
171        !self.symmetric_private_node_vms.is_empty()
172    }
173
174    pub fn symmetric_private_node_and_gateway_map(
175        &self,
176    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
177        Self::match_private_node_vm_and_gateway_vm(
178            &self.symmetric_private_node_vms,
179            &self.symmetric_nat_gateway_vms,
180        )
181    }
182
183    pub fn full_cone_private_node_and_gateway_map(
184        &self,
185    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
186        Self::match_private_node_vm_and_gateway_vm(
187            &self.full_cone_private_node_vms,
188            &self.full_cone_nat_gateway_vms,
189        )
190    }
191
192    pub fn match_private_node_vm_and_gateway_vm(
193        private_node_vms: &[VirtualMachine],
194        nat_gateway_vms: &[VirtualMachine],
195    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
196        if private_node_vms.len() != nat_gateway_vms.len() {
197            println!(
198            "The number of private node VMs ({}) does not match the number of NAT Gateway VMs ({})",
199            private_node_vms.len(),
200            nat_gateway_vms.len()
201        );
202            error!("The number of private node VMs does not match the number of NAT Gateway VMs: Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}");
203            return Err(Error::VmCountMismatch(None, None));
204        }
205
206        let mut map = HashMap::new();
207        for private_vm in private_node_vms {
208            let nat_gateway = nat_gateway_vms
209                .iter()
210                .find(|vm| {
211                    let private_node_name = private_vm.name.split('-').last().unwrap();
212                    let nat_gateway_name = vm.name.split('-').last().unwrap();
213                    private_node_name == nat_gateway_name
214                })
215                .ok_or_else(|| {
216                    println!(
217                        "Failed to find a matching NAT Gateway for private node: {}",
218                        private_vm.name
219                    );
220                    error!("Failed to find a matching NAT Gateway for private node: {}. Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}", private_vm.name);
221                    Error::VmCountMismatch(None, None)
222                })?;
223
224            let _ = map.insert(private_vm.clone(), nat_gateway.clone());
225        }
226
227        Ok(map)
228    }
229}
230
231impl From<BootstrapOptions> for ProvisionOptions {
232    fn from(bootstrap_options: BootstrapOptions) -> Self {
233        ProvisionOptions {
234            ant_version: None,
235            binary_option: bootstrap_options.binary_option,
236            chunk_size: bootstrap_options.chunk_size,
237            downloaders_count: 0,
238            env_variables: bootstrap_options.env_variables,
239            evm_data_payments_address: bootstrap_options.evm_data_payments_address,
240            evm_network: bootstrap_options.evm_network,
241            evm_payment_token_address: bootstrap_options.evm_payment_token_address,
242            evm_rpc_url: bootstrap_options.evm_rpc_url,
243            full_cone_private_node_count: bootstrap_options.full_cone_private_node_count,
244            funding_wallet_secret_key: None,
245            gas_amount: None,
246            interval: bootstrap_options.interval,
247            log_format: bootstrap_options.log_format,
248            logstash_details: None,
249            max_archived_log_files: bootstrap_options.max_archived_log_files,
250            max_log_files: bootstrap_options.max_log_files,
251            name: bootstrap_options.name,
252            network_id: bootstrap_options.network_id,
253            node_count: bootstrap_options.node_count,
254            output_inventory_dir_path: bootstrap_options.output_inventory_dir_path,
255            peer_cache_node_count: 0,
256            symmetric_private_node_count: bootstrap_options.symmetric_private_node_count,
257            public_rpc: false,
258            rewards_address: bootstrap_options.rewards_address,
259            token_amount: None,
260            uploaders_count: None,
261        }
262    }
263}
264
265impl From<DeployOptions> for ProvisionOptions {
266    fn from(deploy_options: DeployOptions) -> Self {
267        ProvisionOptions {
268            ant_version: None,
269            binary_option: deploy_options.binary_option,
270            chunk_size: deploy_options.chunk_size,
271            downloaders_count: deploy_options.downloaders_count,
272            env_variables: deploy_options.env_variables,
273            evm_data_payments_address: deploy_options.evm_data_payments_address,
274            evm_network: deploy_options.evm_network,
275            evm_payment_token_address: deploy_options.evm_payment_token_address,
276            evm_rpc_url: deploy_options.evm_rpc_url,
277            full_cone_private_node_count: deploy_options.full_cone_private_node_count,
278            funding_wallet_secret_key: deploy_options.funding_wallet_secret_key,
279            gas_amount: deploy_options.initial_gas,
280            interval: deploy_options.interval,
281            log_format: deploy_options.log_format,
282            logstash_details: deploy_options.logstash_details,
283            max_archived_log_files: deploy_options.max_archived_log_files,
284            max_log_files: deploy_options.max_log_files,
285            name: deploy_options.name,
286            network_id: deploy_options.network_id,
287            node_count: deploy_options.node_count,
288            output_inventory_dir_path: deploy_options.output_inventory_dir_path,
289            peer_cache_node_count: deploy_options.peer_cache_node_count,
290            symmetric_private_node_count: deploy_options.symmetric_private_node_count,
291            public_rpc: deploy_options.public_rpc,
292            rewards_address: deploy_options.rewards_address,
293            token_amount: deploy_options.initial_tokens,
294            uploaders_count: Some(deploy_options.uploaders_count),
295        }
296    }
297}
298
299#[derive(Clone)]
300pub struct AnsibleProvisioner {
301    pub ansible_runner: AnsibleRunner,
302    pub cloud_provider: CloudProvider,
303    pub ssh_client: SshClient,
304}
305
306impl AnsibleProvisioner {
307    pub fn new(
308        ansible_runner: AnsibleRunner,
309        cloud_provider: CloudProvider,
310        ssh_client: SshClient,
311    ) -> Self {
312        Self {
313            ansible_runner,
314            cloud_provider,
315            ssh_client,
316        }
317    }
318
319    pub fn build_safe_network_binaries(&self, options: &ProvisionOptions) -> Result<()> {
320        let start = Instant::now();
321        println!("Obtaining IP address for build VM...");
322        let build_inventory = self
323            .ansible_runner
324            .get_inventory(AnsibleInventoryType::Build, true)?;
325        let build_ip = build_inventory[0].public_ip_addr;
326        self.ssh_client
327            .wait_for_ssh_availability(&build_ip, &self.cloud_provider.get_ssh_user())?;
328
329        println!("Running ansible against build VM...");
330        let extra_vars = extra_vars::build_binaries_extra_vars_doc(options)?;
331        self.ansible_runner.run_playbook(
332            AnsiblePlaybook::Build,
333            AnsibleInventoryType::Build,
334            Some(extra_vars),
335        )?;
336        print_duration(start.elapsed());
337        Ok(())
338    }
339
340    pub fn cleanup_node_logs(&self, setup_cron: bool) -> Result<()> {
341        for node_inv_type in AnsibleInventoryType::iter_node_type() {
342            self.ansible_runner.run_playbook(
343                AnsiblePlaybook::CleanupLogs,
344                node_inv_type,
345                Some(format!("{{ \"setup_cron\": \"{setup_cron}\" }}")),
346            )?;
347        }
348
349        Ok(())
350    }
351
352    pub fn copy_logs(&self, name: &str, resources_only: bool) -> Result<()> {
353        for node_inv_type in AnsibleInventoryType::iter_node_type() {
354            self.ansible_runner.run_playbook(
355                AnsiblePlaybook::CopyLogs,
356                node_inv_type,
357                Some(format!(
358                    "{{ \"env_name\": \"{name}\", \"resources_only\" : \"{resources_only}\" }}"
359                )),
360            )?;
361        }
362        Ok(())
363    }
364
365    pub fn get_all_node_inventory(&self) -> Result<Vec<VirtualMachine>> {
366        let mut all_node_inventory = Vec::new();
367        for node_inv_type in AnsibleInventoryType::iter_node_type() {
368            all_node_inventory.extend(self.ansible_runner.get_inventory(node_inv_type, false)?);
369        }
370
371        Ok(all_node_inventory)
372    }
373
374    pub fn get_symmetric_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
375        self.ansible_runner
376            .get_inventory(AnsibleInventoryType::SymmetricNatGateway, false)
377    }
378
379    pub fn get_full_cone_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
380        self.ansible_runner
381            .get_inventory(AnsibleInventoryType::FullConeNatGateway, false)
382    }
383
384    pub fn get_node_registries(
385        &self,
386        inventory_type: &AnsibleInventoryType,
387    ) -> Result<DeploymentNodeRegistries> {
388        debug!("Fetching node manager inventory for {inventory_type:?}");
389        let temp_dir_path = tempfile::tempdir()?.into_path();
390        let temp_dir_json = serde_json::to_string(&temp_dir_path)?;
391
392        self.ansible_runner.run_playbook(
393            AnsiblePlaybook::AntCtlInventory,
394            *inventory_type,
395            Some(format!("{{ \"dest\": {temp_dir_json} }}")),
396        )?;
397
398        let node_registry_paths = WalkDir::new(temp_dir_path)
399            .into_iter()
400            .flatten()
401            .filter_map(|entry| {
402                if entry.file_type().is_file()
403                    && entry.path().extension().is_some_and(|ext| ext == "json")
404                {
405                    // tempdir/<testnet_name>-node/var/safenode-manager/node_registry.json
406                    let mut vm_name = entry.path().to_path_buf();
407                    trace!("Found file with json extension: {vm_name:?}");
408                    vm_name.pop();
409                    vm_name.pop();
410                    vm_name.pop();
411                    // Extract the <testnet_name>-node string
412                    trace!("Extracting the vm name from the path");
413                    let vm_name = vm_name.file_name()?.to_str()?;
414                    trace!("Extracted vm name from path: {vm_name}");
415                    Some((vm_name.to_string(), entry.path().to_path_buf()))
416                } else {
417                    None
418                }
419            })
420            .collect::<Vec<(String, PathBuf)>>();
421
422        let mut node_registries = Vec::new();
423        let mut failed_vms = Vec::new();
424        for (vm_name, file_path) in node_registry_paths {
425            match NodeRegistry::load(&file_path) {
426                Ok(node_registry) => node_registries.push((vm_name.clone(), node_registry)),
427                Err(_) => failed_vms.push(vm_name.clone()),
428            }
429        }
430
431        let deployment_registries = DeploymentNodeRegistries {
432            inventory_type: *inventory_type,
433            retrieved_registries: node_registries,
434            failed_vms,
435        };
436        Ok(deployment_registries)
437    }
438
439    pub fn provision_evm_nodes(&self, options: &ProvisionOptions) -> Result<()> {
440        let start = Instant::now();
441        println!("Obtaining IP address for EVM nodes...");
442        let evm_node_inventory = self
443            .ansible_runner
444            .get_inventory(AnsibleInventoryType::EvmNodes, true)?;
445        let evm_node_ip = evm_node_inventory[0].public_ip_addr;
446        self.ssh_client
447            .wait_for_ssh_availability(&evm_node_ip, &self.cloud_provider.get_ssh_user())?;
448
449        println!("Running ansible against EVM nodes...");
450        self.ansible_runner.run_playbook(
451            AnsiblePlaybook::EvmNodes,
452            AnsibleInventoryType::EvmNodes,
453            Some(extra_vars::build_evm_nodes_extra_vars_doc(
454                &options.name,
455                &self.cloud_provider,
456            )),
457        )?;
458        print_duration(start.elapsed());
459        Ok(())
460    }
461
462    pub fn provision_genesis_node(&self, options: &ProvisionOptions) -> Result<()> {
463        let start = Instant::now();
464        let genesis_inventory = self
465            .ansible_runner
466            .get_inventory(AnsibleInventoryType::Genesis, true)?;
467        let genesis_ip = genesis_inventory[0].public_ip_addr;
468        self.ssh_client
469            .wait_for_ssh_availability(&genesis_ip, &self.cloud_provider.get_ssh_user())?;
470        self.ansible_runner.run_playbook(
471            AnsiblePlaybook::Genesis,
472            AnsibleInventoryType::Genesis,
473            Some(extra_vars::build_node_extra_vars_doc(
474                &self.cloud_provider.to_string(),
475                options,
476                NodeType::Genesis,
477                None,
478                None,
479                1,
480                options.evm_network.clone(),
481                false,
482            )?),
483        )?;
484
485        print_duration(start.elapsed());
486
487        Ok(())
488    }
489
490    pub fn provision_full_cone(
491        &self,
492        options: &ProvisionOptions,
493        initial_contact_peer: Option<String>,
494        initial_network_contacts_url: Option<String>,
495        private_node_inventory: PrivateNodeProvisionInventory,
496        new_full_cone_nat_gateway_new_vms_for_upscale: Option<Vec<VirtualMachine>>,
497    ) -> Result<()> {
498        // Step 1 of Full Cone NAT Gateway
499        let start = Instant::now();
500        self.print_ansible_run_banner("Provision Full Cone NAT Gateway - Step 1");
501
502        for vm in new_full_cone_nat_gateway_new_vms_for_upscale
503            .as_ref()
504            .unwrap_or(&private_node_inventory.full_cone_nat_gateway_vms)
505            .iter()
506        {
507            println!(
508                "Checking SSH availability for Full Cone NAT Gateway: {}",
509                vm.public_ip_addr
510            );
511            self.ssh_client
512                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
513                .map_err(|e| {
514                    println!(
515                        "Failed to establish SSH connection to Full Cone NAT Gateway: {}",
516                        e
517                    );
518                    e
519                })?;
520        }
521
522        let mut modified_private_node_inventory = private_node_inventory.clone();
523
524        // If we are upscaling, then we cannot access the gateway VMs which are already deployed.
525        if let Some(new_full_cone_nat_gateway_new_vms_for_upscale) =
526            &new_full_cone_nat_gateway_new_vms_for_upscale
527        {
528            debug!("Removing existing full cone NAT Gateway and private node VMs from the inventory. Old inventory: {modified_private_node_inventory:?}");
529            let mut names_to_keep = Vec::new();
530
531            for vm in new_full_cone_nat_gateway_new_vms_for_upscale.iter() {
532                let nat_gateway_name = vm.name.split('-').last().unwrap();
533                names_to_keep.push(nat_gateway_name);
534            }
535
536            modified_private_node_inventory
537                .full_cone_nat_gateway_vms
538                .retain(|vm| {
539                    let nat_gateway_name = vm.name.split('-').last().unwrap();
540                    names_to_keep.contains(&nat_gateway_name)
541                });
542            modified_private_node_inventory
543                .full_cone_private_node_vms
544                .retain(|vm| {
545                    let nat_gateway_name = vm.name.split('-').last().unwrap();
546                    names_to_keep.contains(&nat_gateway_name)
547                });
548            debug!("New inventory after removing existing full cone NAT Gateway and private node VMs: {modified_private_node_inventory:?}");
549        }
550
551        if modified_private_node_inventory
552            .full_cone_nat_gateway_vms
553            .is_empty()
554        {
555            error!("There are no full cone NAT Gateway VMs available to upscale");
556            return Ok(());
557        }
558
559        let private_node_ip_map = modified_private_node_inventory
560            .full_cone_private_node_and_gateway_map()?
561            .into_iter()
562            .map(|(k, v)| {
563                let gateway_name = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
564                    debug!("Upscaling, using public IP address for gateway name");
565                    v.public_ip_addr.to_string()
566                } else {
567                    v.name.clone()
568                };
569                (gateway_name, k.private_ip_addr)
570            })
571            .collect::<HashMap<String, IpAddr>>();
572
573        if private_node_ip_map.is_empty() {
574            println!("There are no full cone private node VM available to be routed through the full cone NAT Gateway");
575            return Err(Error::EmptyInventory(
576                AnsibleInventoryType::FullConePrivateNodes,
577            ));
578        }
579
580        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
581            &options.name,
582            private_node_ip_map.clone(),
583            "step1",
584        );
585        debug!("Provisioning Full Cone NAT Gateway - Step 1 with vars: {vars}");
586        let gateway_inventory = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
587            debug!("Upscaling, using static inventory for full cone nat gateway.");
588            generate_full_cone_nat_gateway_static_environment_inventory(
589                &modified_private_node_inventory.full_cone_nat_gateway_vms,
590                &options.name,
591                &options.output_inventory_dir_path,
592            )?;
593
594            AnsibleInventoryType::FullConeNatGatewayStatic
595        } else {
596            AnsibleInventoryType::FullConeNatGateway
597        };
598        self.ansible_runner.run_playbook(
599            AnsiblePlaybook::FullConeNatGateway,
600            gateway_inventory,
601            Some(vars),
602        )?;
603
604        // setup private node config
605        self.print_ansible_run_banner("Provisioning Full Cone Private Node Config");
606
607        generate_full_cone_private_node_static_environment_inventory(
608            &options.name,
609            &options.output_inventory_dir_path,
610            &private_node_inventory.full_cone_private_node_vms,
611            &private_node_inventory.full_cone_nat_gateway_vms,
612            &self.ssh_client.private_key_path,
613        )
614        .inspect_err(|err| {
615            error!("Failed to generate full cone private node static inv with err: {err:?}")
616        })?;
617
618        // For a new deployment, it's quite probable that SSH is available, because this part occurs
619        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
620        // check that SSH is available before proceeding.
621        println!("Obtaining IP addresses for nodes...");
622        let inventory = self
623            .ansible_runner
624            .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)?;
625
626        println!("Waiting for SSH availability on Symmetric Private nodes...");
627        for vm in inventory.iter() {
628            println!(
629                "Checking SSH availability for {}: {}",
630                vm.name, vm.public_ip_addr
631            );
632            self.ssh_client
633                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
634                .map_err(|e| {
635                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
636                    e
637                })?;
638        }
639
640        println!("SSH is available on all nodes. Proceeding with provisioning...");
641
642        self.ansible_runner.run_playbook(
643            AnsiblePlaybook::PrivateNodeConfig,
644            AnsibleInventoryType::FullConePrivateNodes,
645            Some(
646                extra_vars::build_full_cone_private_node_config_extra_vars_docs(
647                    &private_node_inventory,
648                )?,
649            ),
650        )?;
651
652        // Step 2 of Full Cone NAT Gateway
653
654        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
655            &options.name,
656            private_node_ip_map,
657            "step2",
658        );
659
660        self.print_ansible_run_banner("Provisioning Full Cone NAT Gateway - Step 2");
661        debug!("Provisioning Full Cone NAT Gateway - Step 2 with vars: {vars}");
662        self.ansible_runner.run_playbook(
663            AnsiblePlaybook::FullConeNatGateway,
664            gateway_inventory,
665            Some(vars),
666        )?;
667
668        // provision the nodes
669
670        let home_dir = std::env::var("HOME").inspect_err(|err| {
671            println!("Failed to get home directory with error: {err:?}",);
672        })?;
673        let known_hosts_path = format!("{}/.ssh/known_hosts", home_dir);
674        debug!("Cleaning up known hosts file at {known_hosts_path} ");
675        run_external_command(
676            PathBuf::from("rm"),
677            std::env::current_dir()?,
678            vec![known_hosts_path],
679            false,
680            false,
681        )?;
682
683        self.print_ansible_run_banner("Provision Full Cone Private Nodes");
684
685        self.ssh_client.set_full_cone_nat_routed_vms(
686            &private_node_inventory.full_cone_private_node_vms,
687            &private_node_inventory.full_cone_nat_gateway_vms,
688        )?;
689
690        self.provision_nodes(
691            options,
692            initial_contact_peer,
693            initial_network_contacts_url,
694            NodeType::FullConePrivateNode,
695        )?;
696
697        print_duration(start.elapsed());
698        Ok(())
699    }
700    pub fn provision_symmetric_nat_gateway(
701        &self,
702        options: &ProvisionOptions,
703        private_node_inventory: &PrivateNodeProvisionInventory,
704    ) -> Result<()> {
705        let start = Instant::now();
706        for vm in &private_node_inventory.symmetric_nat_gateway_vms {
707            println!(
708                "Checking SSH availability for Symmetric NAT Gateway: {}",
709                vm.public_ip_addr
710            );
711            self.ssh_client
712                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
713                .map_err(|e| {
714                    println!(
715                        "Failed to establish SSH connection to Symmetric NAT Gateway: {}",
716                        e
717                    );
718                    e
719                })?;
720        }
721
722        let private_node_ip_map = private_node_inventory
723            .symmetric_private_node_and_gateway_map()?
724            .into_iter()
725            .map(|(k, v)| (v.name.clone(), k.private_ip_addr))
726            .collect::<HashMap<String, IpAddr>>();
727
728        if private_node_ip_map.is_empty() {
729            println!("There are no Symmetric private node VM available to be routed through the Symmetric NAT Gateway");
730            return Err(Error::EmptyInventory(
731                AnsibleInventoryType::SymmetricPrivateNodes,
732            ));
733        }
734
735        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
736            &options.name,
737            private_node_ip_map,
738            "symmetric",
739        );
740        debug!("Provisioning Symmetric NAT Gateway with vars: {vars}");
741        self.ansible_runner.run_playbook(
742            AnsiblePlaybook::SymmetricNatGateway,
743            AnsibleInventoryType::SymmetricNatGateway,
744            Some(vars),
745        )?;
746
747        print_duration(start.elapsed());
748        Ok(())
749    }
750
751    pub fn provision_nodes(
752        &self,
753        options: &ProvisionOptions,
754        initial_contact_peer: Option<String>,
755        initial_network_contacts_url: Option<String>,
756        node_type: NodeType,
757    ) -> Result<()> {
758        let start = Instant::now();
759        let mut home_network_flag = false;
760        let (inventory_type, node_count) = match &node_type {
761            NodeType::FullConePrivateNode => {
762                home_network_flag = true;
763                (
764                    node_type.to_ansible_inventory_type(),
765                    options.full_cone_private_node_count,
766                )
767            }
768            // use provision_genesis_node fn
769            NodeType::Generic => (node_type.to_ansible_inventory_type(), options.node_count),
770            NodeType::Genesis => return Err(Error::InvalidNodeType(node_type)),
771            NodeType::PeerCache => (
772                node_type.to_ansible_inventory_type(),
773                options.peer_cache_node_count,
774            ),
775            NodeType::SymmetricPrivateNode => {
776                home_network_flag = true;
777                (
778                    node_type.to_ansible_inventory_type(),
779                    options.symmetric_private_node_count,
780                )
781            }
782        };
783
784        // For a new deployment, it's quite probable that SSH is available, because this part occurs
785        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
786        // check that SSH is available before proceeding.
787        println!("Obtaining IP addresses for nodes...");
788        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
789
790        println!("Waiting for SSH availability on {node_type:?} nodes...");
791        for vm in inventory.iter() {
792            println!(
793                "Checking SSH availability for {}: {}",
794                vm.name, vm.public_ip_addr
795            );
796            self.ssh_client
797                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
798                .map_err(|e| {
799                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
800                    e
801                })?;
802        }
803
804        println!("SSH is available on all nodes. Proceeding with provisioning...");
805
806        let playbook = match node_type {
807            NodeType::Generic => AnsiblePlaybook::Nodes,
808            NodeType::PeerCache => AnsiblePlaybook::PeerCacheNodes,
809            NodeType::FullConePrivateNode => AnsiblePlaybook::Nodes,
810            NodeType::SymmetricPrivateNode => AnsiblePlaybook::Nodes,
811            _ => return Err(Error::InvalidNodeType(node_type.clone())),
812        };
813        self.ansible_runner.run_playbook(
814            playbook,
815            inventory_type,
816            Some(extra_vars::build_node_extra_vars_doc(
817                &self.cloud_provider.to_string(),
818                options,
819                node_type.clone(),
820                initial_contact_peer,
821                initial_network_contacts_url,
822                node_count,
823                options.evm_network.clone(),
824                home_network_flag,
825            )?),
826        )?;
827
828        print_duration(start.elapsed());
829        Ok(())
830    }
831
832    pub fn provision_symmetric_private_nodes(
833        &self,
834        options: &mut ProvisionOptions,
835        initial_contact_peer: Option<String>,
836        initial_network_contacts_url: Option<String>,
837        private_node_inventory: &PrivateNodeProvisionInventory,
838    ) -> Result<()> {
839        let start = Instant::now();
840        self.print_ansible_run_banner("Provision Symmetric Private Node Config");
841
842        generate_symmetric_private_node_static_environment_inventory(
843            &options.name,
844            &options.output_inventory_dir_path,
845            &private_node_inventory.symmetric_private_node_vms,
846            &private_node_inventory.symmetric_nat_gateway_vms,
847            &self.ssh_client.private_key_path,
848        )
849        .inspect_err(|err| {
850            error!("Failed to generate symmetric private node static inv with err: {err:?}")
851        })?;
852
853        self.ssh_client.set_symmetric_nat_routed_vms(
854            &private_node_inventory.symmetric_private_node_vms,
855            &private_node_inventory.symmetric_nat_gateway_vms,
856        )?;
857
858        let inventory_type = AnsibleInventoryType::SymmetricPrivateNodes;
859
860        // For a new deployment, it's quite probable that SSH is available, because this part occurs
861        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
862        // check that SSH is available before proceeding.
863        println!("Obtaining IP addresses for nodes...");
864        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
865
866        println!("Waiting for SSH availability on Symmetric Private nodes...");
867        for vm in inventory.iter() {
868            println!(
869                "Checking SSH availability for {}: {}",
870                vm.name, vm.public_ip_addr
871            );
872            self.ssh_client
873                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
874                .map_err(|e| {
875                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
876                    e
877                })?;
878        }
879
880        println!("SSH is available on all nodes. Proceeding with provisioning...");
881
882        self.ansible_runner.run_playbook(
883            AnsiblePlaybook::PrivateNodeConfig,
884            inventory_type,
885            Some(
886                extra_vars::build_symmetric_private_node_config_extra_vars_doc(
887                    private_node_inventory,
888                )?,
889            ),
890        )?;
891
892        println!("Provisioned Symmetric Private Node Config");
893        print_duration(start.elapsed());
894
895        self.provision_nodes(
896            options,
897            initial_contact_peer,
898            initial_network_contacts_url,
899            NodeType::SymmetricPrivateNode,
900        )?;
901
902        Ok(())
903    }
904
905    pub async fn provision_uploaders(
906        &self,
907        options: &ProvisionOptions,
908        genesis_multiaddr: Option<String>,
909        genesis_network_contacts_url: Option<String>,
910    ) -> Result<()> {
911        let start = Instant::now();
912
913        let sk_map = self
914            .deposit_funds_to_uploaders(&FundingOptions {
915                evm_data_payments_address: options.evm_data_payments_address.clone(),
916                evm_network: options.evm_network.clone(),
917                evm_payment_token_address: options.evm_payment_token_address.clone(),
918                evm_rpc_url: options.evm_rpc_url.clone(),
919                funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
920                gas_amount: options.gas_amount,
921                token_amount: options.token_amount,
922                uploaders_count: options.uploaders_count,
923            })
924            .await?;
925
926        println!("Running ansible against uploader machine to start the uploader script.");
927        debug!("Running ansible against uploader machine to start the uploader script.");
928
929        self.ansible_runner.run_playbook(
930            AnsiblePlaybook::Uploaders,
931            AnsibleInventoryType::Uploaders,
932            Some(extra_vars::build_uploaders_extra_vars_doc(
933                &self.cloud_provider.to_string(),
934                options,
935                genesis_multiaddr,
936                genesis_network_contacts_url,
937                &sk_map,
938            )?),
939        )?;
940        print_duration(start.elapsed());
941        Ok(())
942    }
943
944    pub fn start_nodes(
945        &self,
946        environment_name: &str,
947        interval: Duration,
948        node_type: Option<NodeType>,
949        custom_inventory: Option<Vec<VirtualMachine>>,
950    ) -> Result<()> {
951        let mut extra_vars = ExtraVarsDocBuilder::default();
952        extra_vars.add_variable("interval", &interval.as_millis().to_string());
953
954        if let Some(node_type) = node_type {
955            println!("Running the start nodes playbook for {node_type:?} nodes");
956            self.ansible_runner.run_playbook(
957                AnsiblePlaybook::StartNodes,
958                node_type.to_ansible_inventory_type(),
959                Some(extra_vars.build()),
960            )?;
961            return Ok(());
962        }
963
964        if let Some(custom_inventory) = custom_inventory {
965            println!("Running the start nodes playbook with a custom inventory");
966            generate_custom_environment_inventory(
967                &custom_inventory,
968                environment_name,
969                &self.ansible_runner.working_directory_path.join("inventory"),
970            )?;
971            self.ansible_runner.run_playbook(
972                AnsiblePlaybook::StartNodes,
973                AnsibleInventoryType::Custom,
974                Some(extra_vars.build()),
975            )?;
976            return Ok(());
977        }
978
979        println!("Running the start nodes playbook for all node types");
980        for node_inv_type in AnsibleInventoryType::iter_node_type() {
981            self.ansible_runner.run_playbook(
982                AnsiblePlaybook::StartNodes,
983                node_inv_type,
984                Some(extra_vars.build()),
985            )?;
986        }
987        Ok(())
988    }
989
990    pub fn status(&self) -> Result<()> {
991        for node_inv_type in AnsibleInventoryType::iter_node_type() {
992            self.ansible_runner
993                .run_playbook(AnsiblePlaybook::Status, node_inv_type, None)?;
994        }
995        Ok(())
996    }
997
998    pub fn start_telegraf(
999        &self,
1000        environment_name: &str,
1001        node_type: Option<NodeType>,
1002        custom_inventory: Option<Vec<VirtualMachine>>,
1003    ) -> Result<()> {
1004        if let Some(node_type) = node_type {
1005            println!("Running the start telegraf playbook for {node_type:?} nodes");
1006            self.ansible_runner.run_playbook(
1007                AnsiblePlaybook::StartTelegraf,
1008                node_type.to_ansible_inventory_type(),
1009                None,
1010            )?;
1011            return Ok(());
1012        }
1013
1014        if let Some(custom_inventory) = custom_inventory {
1015            println!("Running the start telegraf playbook with a custom inventory");
1016            generate_custom_environment_inventory(
1017                &custom_inventory,
1018                environment_name,
1019                &self.ansible_runner.working_directory_path.join("inventory"),
1020            )?;
1021            self.ansible_runner.run_playbook(
1022                AnsiblePlaybook::StartTelegraf,
1023                AnsibleInventoryType::Custom,
1024                None,
1025            )?;
1026            return Ok(());
1027        }
1028
1029        println!("Running the start telegraf playbook for all node types");
1030        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1031            self.ansible_runner.run_playbook(
1032                AnsiblePlaybook::StartTelegraf,
1033                node_inv_type,
1034                None,
1035            )?;
1036        }
1037
1038        Ok(())
1039    }
1040
1041    pub fn stop_nodes(
1042        &self,
1043        environment_name: &str,
1044        interval: Duration,
1045        node_type: Option<NodeType>,
1046        custom_inventory: Option<Vec<VirtualMachine>>,
1047        delay: Option<u64>,
1048        service_names: Option<Vec<String>>,
1049    ) -> Result<()> {
1050        let mut extra_vars = ExtraVarsDocBuilder::default();
1051        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1052        if let Some(delay) = delay {
1053            extra_vars.add_variable("delay", &delay.to_string());
1054        }
1055        if let Some(service_names) = service_names {
1056            extra_vars.add_list_variable("service_names", service_names);
1057        }
1058        let extra_vars = extra_vars.build();
1059
1060        if let Some(node_type) = node_type {
1061            println!("Running the stop nodes playbook for {node_type:?} nodes");
1062            self.ansible_runner.run_playbook(
1063                AnsiblePlaybook::StopNodes,
1064                node_type.to_ansible_inventory_type(),
1065                Some(extra_vars),
1066            )?;
1067            return Ok(());
1068        }
1069
1070        if let Some(custom_inventory) = custom_inventory {
1071            println!("Running the stop nodes playbook with a custom inventory");
1072            generate_custom_environment_inventory(
1073                &custom_inventory,
1074                environment_name,
1075                &self.ansible_runner.working_directory_path.join("inventory"),
1076            )?;
1077            self.ansible_runner.run_playbook(
1078                AnsiblePlaybook::StopNodes,
1079                AnsibleInventoryType::Custom,
1080                Some(extra_vars),
1081            )?;
1082            return Ok(());
1083        }
1084
1085        println!("Running the stop nodes playbook for all node types");
1086        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1087            self.ansible_runner.run_playbook(
1088                AnsiblePlaybook::StopNodes,
1089                node_inv_type,
1090                Some(extra_vars.clone()),
1091            )?;
1092        }
1093
1094        Ok(())
1095    }
1096
1097    pub fn stop_telegraf(
1098        &self,
1099        environment_name: &str,
1100        node_type: Option<NodeType>,
1101        custom_inventory: Option<Vec<VirtualMachine>>,
1102    ) -> Result<()> {
1103        if let Some(node_type) = node_type {
1104            println!("Running the stop telegraf playbook for {node_type:?} nodes");
1105            self.ansible_runner.run_playbook(
1106                AnsiblePlaybook::StopTelegraf,
1107                node_type.to_ansible_inventory_type(),
1108                None,
1109            )?;
1110            return Ok(());
1111        }
1112
1113        if let Some(custom_inventory) = custom_inventory {
1114            println!("Running the stop telegraf playbook with a custom inventory");
1115            generate_custom_environment_inventory(
1116                &custom_inventory,
1117                environment_name,
1118                &self.ansible_runner.working_directory_path.join("inventory"),
1119            )?;
1120            self.ansible_runner.run_playbook(
1121                AnsiblePlaybook::StopTelegraf,
1122                AnsibleInventoryType::Custom,
1123                None,
1124            )?;
1125            return Ok(());
1126        }
1127
1128        println!("Running the stop telegraf playbook for all node types");
1129        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1130            self.ansible_runner
1131                .run_playbook(AnsiblePlaybook::StopTelegraf, node_inv_type, None)?;
1132        }
1133
1134        Ok(())
1135    }
1136
1137    pub fn upgrade_node_telegraf(&self, name: &str) -> Result<()> {
1138        self.ansible_runner.run_playbook(
1139            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1140            AnsibleInventoryType::PeerCacheNodes,
1141            Some(extra_vars::build_node_telegraf_upgrade(
1142                name,
1143                &NodeType::PeerCache,
1144            )?),
1145        )?;
1146        self.ansible_runner.run_playbook(
1147            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1148            AnsibleInventoryType::Nodes,
1149            Some(extra_vars::build_node_telegraf_upgrade(
1150                name,
1151                &NodeType::Generic,
1152            )?),
1153        )?;
1154
1155        self.ansible_runner.run_playbook(
1156            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1157            AnsibleInventoryType::SymmetricPrivateNodes,
1158            Some(extra_vars::build_node_telegraf_upgrade(
1159                name,
1160                &NodeType::SymmetricPrivateNode,
1161            )?),
1162        )?;
1163
1164        self.ansible_runner.run_playbook(
1165            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1166            AnsibleInventoryType::FullConePrivateNodes,
1167            Some(extra_vars::build_node_telegraf_upgrade(
1168                name,
1169                &NodeType::FullConePrivateNode,
1170            )?),
1171        )?;
1172        Ok(())
1173    }
1174
1175    pub fn upgrade_uploader_telegraf(&self, name: &str) -> Result<()> {
1176        self.ansible_runner.run_playbook(
1177            AnsiblePlaybook::UpgradeUploaderTelegrafConfig,
1178            AnsibleInventoryType::Uploaders,
1179            Some(extra_vars::build_uploader_telegraf_upgrade(name)?),
1180        )?;
1181        Ok(())
1182    }
1183
1184    pub fn upgrade_nodes(&self, options: &UpgradeOptions) -> Result<()> {
1185        if let Some(custom_inventory) = &options.custom_inventory {
1186            println!("Running the UpgradeNodes with a custom inventory");
1187            generate_custom_environment_inventory(
1188                custom_inventory,
1189                &options.name,
1190                &self.ansible_runner.working_directory_path.join("inventory"),
1191            )?;
1192            match self.ansible_runner.run_playbook(
1193                AnsiblePlaybook::UpgradeNodes,
1194                AnsibleInventoryType::Custom,
1195                Some(options.get_ansible_vars()),
1196            ) {
1197                Ok(()) => println!("All nodes were successfully upgraded"),
1198                Err(_) => {
1199                    println!("WARNING: some nodes may not have been upgraded or restarted");
1200                }
1201            }
1202            return Ok(());
1203        }
1204
1205        if let Some(node_type) = &options.node_type {
1206            println!("Running the UpgradeNodes playbook for {node_type:?} nodes");
1207            match self.ansible_runner.run_playbook(
1208                AnsiblePlaybook::UpgradeNodes,
1209                node_type.to_ansible_inventory_type(),
1210                Some(options.get_ansible_vars()),
1211            ) {
1212                Ok(()) => println!("All {node_type:?} nodes were successfully upgraded"),
1213                Err(_) => {
1214                    println!(
1215                        "WARNING: some {node_type:?} nodes may not have been upgraded or restarted"
1216                    );
1217                }
1218            }
1219            return Ok(());
1220        }
1221
1222        println!("Running the UpgradeNodes playbook for all node types");
1223
1224        match self.ansible_runner.run_playbook(
1225            AnsiblePlaybook::UpgradeNodes,
1226            AnsibleInventoryType::PeerCacheNodes,
1227            Some(options.get_ansible_vars()),
1228        ) {
1229            Ok(()) => println!("All Peer Cache nodes were successfully upgraded"),
1230            Err(_) => {
1231                println!("WARNING: some Peer Cacche nodes may not have been upgraded or restarted");
1232            }
1233        }
1234        match self.ansible_runner.run_playbook(
1235            AnsiblePlaybook::UpgradeNodes,
1236            AnsibleInventoryType::Nodes,
1237            Some(options.get_ansible_vars()),
1238        ) {
1239            Ok(()) => println!("All generic nodes were successfully upgraded"),
1240            Err(_) => {
1241                println!("WARNING: some nodes may not have been upgraded or restarted");
1242            }
1243        }
1244        match self.ansible_runner.run_playbook(
1245            AnsiblePlaybook::UpgradeNodes,
1246            AnsibleInventoryType::SymmetricPrivateNodes,
1247            Some(options.get_ansible_vars()),
1248        ) {
1249            Ok(()) => println!("All private nodes were successfully upgraded"),
1250            Err(_) => {
1251                println!("WARNING: some nodes may not have been upgraded or restarted");
1252            }
1253        }
1254        // Don't use AnsibleInventoryType::iter_node_type() here, because the genesis node should be upgraded last
1255        match self.ansible_runner.run_playbook(
1256            AnsiblePlaybook::UpgradeNodes,
1257            AnsibleInventoryType::Genesis,
1258            Some(options.get_ansible_vars()),
1259        ) {
1260            Ok(()) => println!("The genesis nodes was successfully upgraded"),
1261            Err(_) => {
1262                println!("WARNING: the genesis node may not have been upgraded or restarted");
1263            }
1264        }
1265        Ok(())
1266    }
1267
1268    pub fn upgrade_antctl(
1269        &self,
1270        environment_name: &str,
1271        version: &Version,
1272        node_type: Option<NodeType>,
1273        custom_inventory: Option<Vec<VirtualMachine>>,
1274    ) -> Result<()> {
1275        let mut extra_vars = ExtraVarsDocBuilder::default();
1276        extra_vars.add_variable("version", &version.to_string());
1277
1278        if let Some(node_type) = node_type {
1279            println!("Running the upgrade safenode-manager playbook for {node_type:?} nodes");
1280            self.ansible_runner.run_playbook(
1281                AnsiblePlaybook::UpgradeAntctl,
1282                node_type.to_ansible_inventory_type(),
1283                Some(extra_vars.build()),
1284            )?;
1285            return Ok(());
1286        }
1287
1288        if let Some(custom_inventory) = custom_inventory {
1289            println!("Running the upgrade safenode-manager playbook with a custom inventory");
1290            generate_custom_environment_inventory(
1291                &custom_inventory,
1292                environment_name,
1293                &self.ansible_runner.working_directory_path.join("inventory"),
1294            )?;
1295            self.ansible_runner.run_playbook(
1296                AnsiblePlaybook::UpgradeAntctl,
1297                AnsibleInventoryType::Custom,
1298                Some(extra_vars.build()),
1299            )?;
1300            return Ok(());
1301        }
1302
1303        println!("Running the upgrade safenode-manager playbook for all node types");
1304        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1305            self.ansible_runner.run_playbook(
1306                AnsiblePlaybook::UpgradeAntctl,
1307                node_inv_type,
1308                Some(extra_vars.build()),
1309            )?;
1310        }
1311
1312        Ok(())
1313    }
1314
1315    pub fn print_ansible_run_banner(&self, s: &str) {
1316        let ansible_run_msg = "Ansible Run: ";
1317        let line = "=".repeat(s.len() + ansible_run_msg.len());
1318        println!("{}\n{}{}\n{}", line, ansible_run_msg, s, line);
1319    }
1320}
sn_testnet_deploy/ansible/provisioning.rs

sn_testnet_deploy/ansible/
provisioning.rs
