sn_testnet_deploy/ansible/
provisioning.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use super::{
8    extra_vars::ExtraVarsDocBuilder,
9    inventory::{
10        generate_full_cone_private_node_static_environment_inventory,
11        generate_symmetric_private_node_static_environment_inventory,
12    },
13    AnsibleInventoryType, AnsiblePlaybook, AnsibleRunner,
14};
15use crate::{
16    ansible::inventory::{
17        generate_custom_environment_inventory,
18        generate_full_cone_nat_gateway_static_environment_inventory,
19    },
20    bootstrap::BootstrapOptions,
21    deploy::DeployOptions,
22    error::{Error, Result},
23    funding::FundingOptions,
24    inventory::{DeploymentNodeRegistries, VirtualMachine},
25    print_duration, run_external_command, BinaryOption, CloudProvider, EvmNetwork, LogFormat,
26    NodeType, SshClient, UpgradeOptions,
27};
28use ant_service_management::NodeRegistry;
29use evmlib::common::U256;
30use log::{debug, error, trace};
31use semver::Version;
32use serde::{Deserialize, Serialize};
33use std::{
34    collections::HashMap,
35    net::{IpAddr, SocketAddr},
36    path::PathBuf,
37    time::{Duration, Instant},
38};
39use walkdir::WalkDir;
40
41use crate::ansible::extra_vars;
42
43pub const DEFAULT_BETA_ENCRYPTION_KEY: &str =
44    "49113d2083f57a976076adbe85decb75115820de1e6e74b47e0429338cef124a";
45
46#[derive(Clone, Serialize, Deserialize)]
47pub struct ProvisionOptions {
48    /// The safe version is also in the binary option, but only for an initial deployment.
49    /// For the upscale, it needs to be provided explicitly, because currently it is not
50    /// recorded in the inventory.
51    pub ant_version: Option<String>,
52    pub binary_option: BinaryOption,
53    pub chunk_size: Option<u64>,
54    pub downloaders_count: u16,
55    pub env_variables: Option<Vec<(String, String)>>,
56    pub evm_data_payments_address: Option<String>,
57    pub evm_network: EvmNetwork,
58    pub evm_payment_token_address: Option<String>,
59    pub evm_rpc_url: Option<String>,
60    pub full_cone_private_node_count: u16,
61    pub funding_wallet_secret_key: Option<String>,
62    pub gas_amount: Option<U256>,
63    pub interval: Duration,
64    pub log_format: Option<LogFormat>,
65    pub logstash_details: Option<(String, Vec<SocketAddr>)>,
66    pub max_archived_log_files: u16,
67    pub max_log_files: u16,
68    pub name: String,
69    pub network_id: Option<u8>,
70    pub node_count: u16,
71    pub output_inventory_dir_path: PathBuf,
72    pub peer_cache_node_count: u16,
73    pub symmetric_private_node_count: u16,
74    pub public_rpc: bool,
75    pub rewards_address: String,
76    pub token_amount: Option<U256>,
77    pub uploaders_count: Option<u16>,
78}
79
80/// These are obtained by running the inventory playbook
81#[derive(Clone, Debug)]
82pub struct PrivateNodeProvisionInventory {
83    pub full_cone_nat_gateway_vms: Vec<VirtualMachine>,
84    pub full_cone_private_node_vms: Vec<VirtualMachine>,
85    pub symmetric_nat_gateway_vms: Vec<VirtualMachine>,
86    pub symmetric_private_node_vms: Vec<VirtualMachine>,
87}
88
89impl PrivateNodeProvisionInventory {
90    pub fn new(
91        provisioner: &AnsibleProvisioner,
92        full_cone_private_node_vm_count: Option<u16>,
93        symmetric_private_node_vm_count: Option<u16>,
94    ) -> Result<Self> {
95        // All the environment types set private_node_vm count to >0 if not specified.
96        let should_provision_full_cone_private_nodes = full_cone_private_node_vm_count
97            .map(|count| count > 0)
98            .unwrap_or(true);
99        let should_provision_symmetric_private_nodes = symmetric_private_node_vm_count
100            .map(|count| count > 0)
101            .unwrap_or(true);
102
103        let mut inventory = Self {
104            full_cone_nat_gateway_vms: Default::default(),
105            full_cone_private_node_vms: Default::default(),
106            symmetric_nat_gateway_vms: Default::default(),
107            symmetric_private_node_vms: Default::default(),
108        };
109
110        if should_provision_full_cone_private_nodes {
111            let full_cone_private_node_vms = provisioner
112                .ansible_runner
113                .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)
114                .inspect_err(|err| {
115                    println!("Failed to obtain the inventory of Full Cone private node: {err:?}");
116                })?;
117
118            let full_cone_nat_gateway_inventory = provisioner
119                .ansible_runner
120                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)
121                .inspect_err(|err| {
122                    println!("Failed to get Full Cone NAT Gateway inventory {err:?}");
123                })?;
124
125            if full_cone_nat_gateway_inventory.len() != full_cone_private_node_vms.len() {
126                println!("The number of Full Cone private nodes does not match the number of Full Cone NAT Gateway VMs");
127                return Err(Error::VmCountMismatch(
128                    Some(AnsibleInventoryType::FullConePrivateNodes),
129                    Some(AnsibleInventoryType::FullConeNatGateway),
130                ));
131            }
132
133            inventory.full_cone_private_node_vms = full_cone_private_node_vms;
134            inventory.full_cone_nat_gateway_vms = full_cone_nat_gateway_inventory;
135        }
136
137        if should_provision_symmetric_private_nodes {
138            let symmetric_private_node_vms = provisioner
139                .ansible_runner
140                .get_inventory(AnsibleInventoryType::SymmetricPrivateNodes, true)
141                .inspect_err(|err| {
142                    println!("Failed to obtain the inventory of Symmetric private node: {err:?}");
143                })?;
144
145            let symmetric_nat_gateway_inventory = provisioner
146                .ansible_runner
147                .get_inventory(AnsibleInventoryType::SymmetricNatGateway, true)
148                .inspect_err(|err| {
149                    println!("Failed to get Symmetric NAT Gateway inventory {err:?}");
150                })?;
151
152            if symmetric_nat_gateway_inventory.len() != symmetric_private_node_vms.len() {
153                println!("The number of Symmetric private nodes does not match the number of Symmetric NAT Gateway VMs");
154                return Err(Error::VmCountMismatch(
155                    Some(AnsibleInventoryType::SymmetricPrivateNodes),
156                    Some(AnsibleInventoryType::SymmetricNatGateway),
157                ));
158            }
159
160            inventory.symmetric_private_node_vms = symmetric_private_node_vms;
161            inventory.symmetric_nat_gateway_vms = symmetric_nat_gateway_inventory;
162        }
163
164        Ok(inventory)
165    }
166
167    pub fn should_provision_full_cone_private_nodes(&self) -> bool {
168        !self.full_cone_private_node_vms.is_empty()
169    }
170
171    pub fn should_provision_symmetric_private_nodes(&self) -> bool {
172        !self.symmetric_private_node_vms.is_empty()
173    }
174
175    pub fn symmetric_private_node_and_gateway_map(
176        &self,
177    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
178        Self::match_private_node_vm_and_gateway_vm(
179            &self.symmetric_private_node_vms,
180            &self.symmetric_nat_gateway_vms,
181        )
182    }
183
184    pub fn full_cone_private_node_and_gateway_map(
185        &self,
186    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
187        Self::match_private_node_vm_and_gateway_vm(
188            &self.full_cone_private_node_vms,
189            &self.full_cone_nat_gateway_vms,
190        )
191    }
192
193    pub fn match_private_node_vm_and_gateway_vm(
194        private_node_vms: &[VirtualMachine],
195        nat_gateway_vms: &[VirtualMachine],
196    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
197        if private_node_vms.len() != nat_gateway_vms.len() {
198            println!(
199            "The number of private node VMs ({}) does not match the number of NAT Gateway VMs ({})",
200            private_node_vms.len(),
201            nat_gateway_vms.len()
202        );
203            error!("The number of private node VMs does not match the number of NAT Gateway VMs: Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}");
204            return Err(Error::VmCountMismatch(None, None));
205        }
206
207        let mut map = HashMap::new();
208        for private_vm in private_node_vms {
209            let nat_gateway = nat_gateway_vms
210                .iter()
211                .find(|vm| {
212                    let private_node_name = private_vm.name.split('-').last().unwrap();
213                    let nat_gateway_name = vm.name.split('-').last().unwrap();
214                    private_node_name == nat_gateway_name
215                })
216                .ok_or_else(|| {
217                    println!(
218                        "Failed to find a matching NAT Gateway for private node: {}",
219                        private_vm.name
220                    );
221                    error!("Failed to find a matching NAT Gateway for private node: {}. Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}", private_vm.name);
222                    Error::VmCountMismatch(None, None)
223                })?;
224
225            let _ = map.insert(private_vm.clone(), nat_gateway.clone());
226        }
227
228        Ok(map)
229    }
230}
231
232impl From<BootstrapOptions> for ProvisionOptions {
233    fn from(bootstrap_options: BootstrapOptions) -> Self {
234        ProvisionOptions {
235            ant_version: None,
236            binary_option: bootstrap_options.binary_option,
237            chunk_size: bootstrap_options.chunk_size,
238            downloaders_count: 0,
239            env_variables: bootstrap_options.env_variables,
240            evm_data_payments_address: bootstrap_options.evm_data_payments_address,
241            evm_network: bootstrap_options.evm_network,
242            evm_payment_token_address: bootstrap_options.evm_payment_token_address,
243            evm_rpc_url: bootstrap_options.evm_rpc_url,
244            full_cone_private_node_count: bootstrap_options.full_cone_private_node_count,
245            funding_wallet_secret_key: None,
246            gas_amount: None,
247            interval: bootstrap_options.interval,
248            log_format: bootstrap_options.log_format,
249            logstash_details: None,
250            max_archived_log_files: bootstrap_options.max_archived_log_files,
251            max_log_files: bootstrap_options.max_log_files,
252            name: bootstrap_options.name,
253            network_id: bootstrap_options.network_id,
254            node_count: bootstrap_options.node_count,
255            output_inventory_dir_path: bootstrap_options.output_inventory_dir_path,
256            peer_cache_node_count: 0,
257            symmetric_private_node_count: bootstrap_options.symmetric_private_node_count,
258            public_rpc: false,
259            rewards_address: bootstrap_options.rewards_address,
260            token_amount: None,
261            uploaders_count: None,
262        }
263    }
264}
265
266impl From<DeployOptions> for ProvisionOptions {
267    fn from(deploy_options: DeployOptions) -> Self {
268        ProvisionOptions {
269            ant_version: None,
270            binary_option: deploy_options.binary_option,
271            chunk_size: deploy_options.chunk_size,
272            downloaders_count: deploy_options.downloaders_count,
273            env_variables: deploy_options.env_variables,
274            evm_data_payments_address: deploy_options.evm_data_payments_address,
275            evm_network: deploy_options.evm_network,
276            evm_payment_token_address: deploy_options.evm_payment_token_address,
277            evm_rpc_url: deploy_options.evm_rpc_url,
278            full_cone_private_node_count: deploy_options.full_cone_private_node_count,
279            funding_wallet_secret_key: deploy_options.funding_wallet_secret_key,
280            gas_amount: deploy_options.initial_gas,
281            interval: deploy_options.interval,
282            log_format: deploy_options.log_format,
283            logstash_details: deploy_options.logstash_details,
284            max_archived_log_files: deploy_options.max_archived_log_files,
285            max_log_files: deploy_options.max_log_files,
286            name: deploy_options.name,
287            network_id: deploy_options.network_id,
288            node_count: deploy_options.node_count,
289            output_inventory_dir_path: deploy_options.output_inventory_dir_path,
290            peer_cache_node_count: deploy_options.peer_cache_node_count,
291            symmetric_private_node_count: deploy_options.symmetric_private_node_count,
292            public_rpc: deploy_options.public_rpc,
293            rewards_address: deploy_options.rewards_address,
294            token_amount: deploy_options.initial_tokens,
295            uploaders_count: Some(deploy_options.uploaders_count),
296        }
297    }
298}
299
300#[derive(Clone)]
301pub struct AnsibleProvisioner {
302    pub ansible_runner: AnsibleRunner,
303    pub cloud_provider: CloudProvider,
304    pub ssh_client: SshClient,
305}
306
307impl AnsibleProvisioner {
308    pub fn new(
309        ansible_runner: AnsibleRunner,
310        cloud_provider: CloudProvider,
311        ssh_client: SshClient,
312    ) -> Self {
313        Self {
314            ansible_runner,
315            cloud_provider,
316            ssh_client,
317        }
318    }
319
320    pub fn build_safe_network_binaries(&self, options: &ProvisionOptions) -> Result<()> {
321        let start = Instant::now();
322        println!("Obtaining IP address for build VM...");
323        let build_inventory = self
324            .ansible_runner
325            .get_inventory(AnsibleInventoryType::Build, true)?;
326        let build_ip = build_inventory[0].public_ip_addr;
327        self.ssh_client
328            .wait_for_ssh_availability(&build_ip, &self.cloud_provider.get_ssh_user())?;
329
330        println!("Running ansible against build VM...");
331        let extra_vars = extra_vars::build_binaries_extra_vars_doc(options)?;
332        self.ansible_runner.run_playbook(
333            AnsiblePlaybook::Build,
334            AnsibleInventoryType::Build,
335            Some(extra_vars),
336        )?;
337        print_duration(start.elapsed());
338        Ok(())
339    }
340
341    pub fn cleanup_node_logs(&self, setup_cron: bool) -> Result<()> {
342        for node_inv_type in AnsibleInventoryType::iter_node_type() {
343            self.ansible_runner.run_playbook(
344                AnsiblePlaybook::CleanupLogs,
345                node_inv_type,
346                Some(format!("{{ \"setup_cron\": \"{setup_cron}\" }}")),
347            )?;
348        }
349
350        Ok(())
351    }
352
353    pub fn copy_logs(&self, name: &str, resources_only: bool) -> Result<()> {
354        for node_inv_type in AnsibleInventoryType::iter_node_type() {
355            self.ansible_runner.run_playbook(
356                AnsiblePlaybook::CopyLogs,
357                node_inv_type,
358                Some(format!(
359                    "{{ \"env_name\": \"{name}\", \"resources_only\" : \"{resources_only}\" }}"
360                )),
361            )?;
362        }
363        Ok(())
364    }
365
366    pub fn get_all_node_inventory(&self) -> Result<Vec<VirtualMachine>> {
367        let mut all_node_inventory = Vec::new();
368        for node_inv_type in AnsibleInventoryType::iter_node_type() {
369            all_node_inventory.extend(self.ansible_runner.get_inventory(node_inv_type, false)?);
370        }
371
372        Ok(all_node_inventory)
373    }
374
375    pub fn get_symmetric_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
376        self.ansible_runner
377            .get_inventory(AnsibleInventoryType::SymmetricNatGateway, false)
378    }
379
380    pub fn get_full_cone_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
381        self.ansible_runner
382            .get_inventory(AnsibleInventoryType::FullConeNatGateway, false)
383    }
384
385    pub fn get_node_registries(
386        &self,
387        inventory_type: &AnsibleInventoryType,
388    ) -> Result<DeploymentNodeRegistries> {
389        debug!("Fetching node manager inventory for {inventory_type:?}");
390        let temp_dir_path = tempfile::tempdir()?.into_path();
391        let temp_dir_json = serde_json::to_string(&temp_dir_path)?;
392
393        self.ansible_runner.run_playbook(
394            AnsiblePlaybook::AntCtlInventory,
395            *inventory_type,
396            Some(format!("{{ \"dest\": {temp_dir_json} }}")),
397        )?;
398
399        let node_registry_paths = WalkDir::new(temp_dir_path)
400            .into_iter()
401            .flatten()
402            .filter_map(|entry| {
403                if entry.file_type().is_file()
404                    && entry.path().extension().is_some_and(|ext| ext == "json")
405                {
406                    // tempdir/<testnet_name>-node/var/safenode-manager/node_registry.json
407                    let mut vm_name = entry.path().to_path_buf();
408                    trace!("Found file with json extension: {vm_name:?}");
409                    vm_name.pop();
410                    vm_name.pop();
411                    vm_name.pop();
412                    // Extract the <testnet_name>-node string
413                    trace!("Extracting the vm name from the path");
414                    let vm_name = vm_name.file_name()?.to_str()?;
415                    trace!("Extracted vm name from path: {vm_name}");
416                    Some((vm_name.to_string(), entry.path().to_path_buf()))
417                } else {
418                    None
419                }
420            })
421            .collect::<Vec<(String, PathBuf)>>();
422
423        let mut node_registries = Vec::new();
424        let mut failed_vms = Vec::new();
425        for (vm_name, file_path) in node_registry_paths {
426            match NodeRegistry::load(&file_path) {
427                Ok(node_registry) => node_registries.push((vm_name.clone(), node_registry)),
428                Err(_) => failed_vms.push(vm_name.clone()),
429            }
430        }
431
432        let deployment_registries = DeploymentNodeRegistries {
433            inventory_type: *inventory_type,
434            retrieved_registries: node_registries,
435            failed_vms,
436        };
437        Ok(deployment_registries)
438    }
439
440    pub fn provision_evm_nodes(&self, options: &ProvisionOptions) -> Result<()> {
441        let start = Instant::now();
442        println!("Obtaining IP address for EVM nodes...");
443        let evm_node_inventory = self
444            .ansible_runner
445            .get_inventory(AnsibleInventoryType::EvmNodes, true)?;
446        let evm_node_ip = evm_node_inventory[0].public_ip_addr;
447        self.ssh_client
448            .wait_for_ssh_availability(&evm_node_ip, &self.cloud_provider.get_ssh_user())?;
449
450        println!("Running ansible against EVM nodes...");
451        self.ansible_runner.run_playbook(
452            AnsiblePlaybook::EvmNodes,
453            AnsibleInventoryType::EvmNodes,
454            Some(extra_vars::build_evm_nodes_extra_vars_doc(
455                &options.name,
456                &self.cloud_provider,
457            )),
458        )?;
459        print_duration(start.elapsed());
460        Ok(())
461    }
462
463    pub fn provision_genesis_node(&self, options: &ProvisionOptions) -> Result<()> {
464        let start = Instant::now();
465        let genesis_inventory = self
466            .ansible_runner
467            .get_inventory(AnsibleInventoryType::Genesis, true)?;
468        let genesis_ip = genesis_inventory[0].public_ip_addr;
469        self.ssh_client
470            .wait_for_ssh_availability(&genesis_ip, &self.cloud_provider.get_ssh_user())?;
471        self.ansible_runner.run_playbook(
472            AnsiblePlaybook::Genesis,
473            AnsibleInventoryType::Genesis,
474            Some(extra_vars::build_node_extra_vars_doc(
475                &self.cloud_provider.to_string(),
476                options,
477                NodeType::Genesis,
478                None,
479                None,
480                1,
481                options.evm_network.clone(),
482                false,
483            )?),
484        )?;
485
486        print_duration(start.elapsed());
487
488        Ok(())
489    }
490
491    pub fn provision_full_cone(
492        &self,
493        options: &ProvisionOptions,
494        initial_contact_peer: Option<String>,
495        initial_network_contacts_url: Option<String>,
496        private_node_inventory: PrivateNodeProvisionInventory,
497        new_full_cone_nat_gateway_new_vms_for_upscale: Option<Vec<VirtualMachine>>,
498    ) -> Result<()> {
499        // Step 1 of Full Cone NAT Gateway
500        let start = Instant::now();
501        self.print_ansible_run_banner("Provision Full Cone NAT Gateway - Step 1");
502
503        for vm in new_full_cone_nat_gateway_new_vms_for_upscale
504            .as_ref()
505            .unwrap_or(&private_node_inventory.full_cone_nat_gateway_vms)
506            .iter()
507        {
508            println!(
509                "Checking SSH availability for Full Cone NAT Gateway: {}",
510                vm.public_ip_addr
511            );
512            self.ssh_client
513                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
514                .map_err(|e| {
515                    println!(
516                        "Failed to establish SSH connection to Full Cone NAT Gateway: {}",
517                        e
518                    );
519                    e
520                })?;
521        }
522
523        let mut modified_private_node_inventory = private_node_inventory.clone();
524
525        // If we are upscaling, then we cannot access the gateway VMs which are already deployed.
526        if let Some(new_full_cone_nat_gateway_new_vms_for_upscale) =
527            &new_full_cone_nat_gateway_new_vms_for_upscale
528        {
529            debug!("Removing existing full cone NAT Gateway and private node VMs from the inventory. Old inventory: {modified_private_node_inventory:?}");
530            let mut names_to_keep = Vec::new();
531
532            for vm in new_full_cone_nat_gateway_new_vms_for_upscale.iter() {
533                let nat_gateway_name = vm.name.split('-').last().unwrap();
534                names_to_keep.push(nat_gateway_name);
535            }
536
537            modified_private_node_inventory
538                .full_cone_nat_gateway_vms
539                .retain(|vm| {
540                    let nat_gateway_name = vm.name.split('-').last().unwrap();
541                    names_to_keep.contains(&nat_gateway_name)
542                });
543            modified_private_node_inventory
544                .full_cone_private_node_vms
545                .retain(|vm| {
546                    let nat_gateway_name = vm.name.split('-').last().unwrap();
547                    names_to_keep.contains(&nat_gateway_name)
548                });
549            debug!("New inventory after removing existing full cone NAT Gateway and private node VMs: {modified_private_node_inventory:?}");
550        }
551
552        if modified_private_node_inventory
553            .full_cone_nat_gateway_vms
554            .is_empty()
555        {
556            error!("There are no full cone NAT Gateway VMs available to upscale");
557            return Ok(());
558        }
559
560        let private_node_ip_map = modified_private_node_inventory
561            .full_cone_private_node_and_gateway_map()?
562            .into_iter()
563            .map(|(k, v)| {
564                let gateway_name = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
565                    debug!("Upscaling, using public IP address for gateway name");
566                    v.public_ip_addr.to_string()
567                } else {
568                    v.name.clone()
569                };
570                (gateway_name, k.private_ip_addr)
571            })
572            .collect::<HashMap<String, IpAddr>>();
573
574        if private_node_ip_map.is_empty() {
575            println!("There are no full cone private node VM available to be routed through the full cone NAT Gateway");
576            return Err(Error::EmptyInventory(
577                AnsibleInventoryType::FullConePrivateNodes,
578            ));
579        }
580
581        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
582            &options.name,
583            private_node_ip_map.clone(),
584            "step1",
585        );
586        debug!("Provisioning Full Cone NAT Gateway - Step 1 with vars: {vars}");
587        let gateway_inventory = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
588            debug!("Upscaling, using static inventory for full cone nat gateway.");
589            generate_full_cone_nat_gateway_static_environment_inventory(
590                &modified_private_node_inventory.full_cone_nat_gateway_vms,
591                &options.name,
592                &options.output_inventory_dir_path,
593            )?;
594
595            AnsibleInventoryType::FullConeNatGatewayStatic
596        } else {
597            AnsibleInventoryType::FullConeNatGateway
598        };
599        self.ansible_runner.run_playbook(
600            AnsiblePlaybook::FullConeNatGateway,
601            gateway_inventory,
602            Some(vars),
603        )?;
604
605        // setup private node config
606        self.print_ansible_run_banner("Provisioning Full Cone Private Node Config");
607
608        generate_full_cone_private_node_static_environment_inventory(
609            &options.name,
610            &options.output_inventory_dir_path,
611            &private_node_inventory.full_cone_private_node_vms,
612            &private_node_inventory.full_cone_nat_gateway_vms,
613            &self.ssh_client.private_key_path,
614        )
615        .inspect_err(|err| {
616            error!("Failed to generate full cone private node static inv with err: {err:?}")
617        })?;
618
619        // For a new deployment, it's quite probable that SSH is available, because this part occurs
620        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
621        // check that SSH is available before proceeding.
622        println!("Obtaining IP addresses for nodes...");
623        let inventory = self
624            .ansible_runner
625            .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)?;
626
627        println!("Waiting for SSH availability on Symmetric Private nodes...");
628        for vm in inventory.iter() {
629            println!(
630                "Checking SSH availability for {}: {}",
631                vm.name, vm.public_ip_addr
632            );
633            self.ssh_client
634                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
635                .map_err(|e| {
636                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
637                    e
638                })?;
639        }
640
641        println!("SSH is available on all nodes. Proceeding with provisioning...");
642
643        self.ansible_runner.run_playbook(
644            AnsiblePlaybook::PrivateNodeConfig,
645            AnsibleInventoryType::FullConePrivateNodes,
646            Some(
647                extra_vars::build_full_cone_private_node_config_extra_vars_docs(
648                    &private_node_inventory,
649                )?,
650            ),
651        )?;
652
653        // Step 2 of Full Cone NAT Gateway
654
655        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
656            &options.name,
657            private_node_ip_map,
658            "step2",
659        );
660
661        self.print_ansible_run_banner("Provisioning Full Cone NAT Gateway - Step 2");
662        debug!("Provisioning Full Cone NAT Gateway - Step 2 with vars: {vars}");
663        self.ansible_runner.run_playbook(
664            AnsiblePlaybook::FullConeNatGateway,
665            gateway_inventory,
666            Some(vars),
667        )?;
668
669        // provision the nodes
670
671        let home_dir = std::env::var("HOME").inspect_err(|err| {
672            println!("Failed to get home directory with error: {err:?}",);
673        })?;
674        let known_hosts_path = format!("{}/.ssh/known_hosts", home_dir);
675        debug!("Cleaning up known hosts file at {known_hosts_path} ");
676        run_external_command(
677            PathBuf::from("rm"),
678            std::env::current_dir()?,
679            vec![known_hosts_path],
680            false,
681            false,
682        )?;
683
684        self.print_ansible_run_banner("Provision Full Cone Private Nodes");
685
686        self.ssh_client.set_full_cone_nat_routed_vms(
687            &private_node_inventory.full_cone_private_node_vms,
688            &private_node_inventory.full_cone_nat_gateway_vms,
689        )?;
690
691        self.provision_nodes(
692            options,
693            initial_contact_peer,
694            initial_network_contacts_url,
695            NodeType::FullConePrivateNode,
696        )?;
697
698        print_duration(start.elapsed());
699        Ok(())
700    }
701    pub fn provision_symmetric_nat_gateway(
702        &self,
703        options: &ProvisionOptions,
704        private_node_inventory: &PrivateNodeProvisionInventory,
705    ) -> Result<()> {
706        let start = Instant::now();
707        for vm in &private_node_inventory.symmetric_nat_gateway_vms {
708            println!(
709                "Checking SSH availability for Symmetric NAT Gateway: {}",
710                vm.public_ip_addr
711            );
712            self.ssh_client
713                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
714                .map_err(|e| {
715                    println!(
716                        "Failed to establish SSH connection to Symmetric NAT Gateway: {}",
717                        e
718                    );
719                    e
720                })?;
721        }
722
723        let private_node_ip_map = private_node_inventory
724            .symmetric_private_node_and_gateway_map()?
725            .into_iter()
726            .map(|(k, v)| (v.name.clone(), k.private_ip_addr))
727            .collect::<HashMap<String, IpAddr>>();
728
729        if private_node_ip_map.is_empty() {
730            println!("There are no Symmetric private node VM available to be routed through the Symmetric NAT Gateway");
731            return Err(Error::EmptyInventory(
732                AnsibleInventoryType::SymmetricPrivateNodes,
733            ));
734        }
735
736        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
737            &options.name,
738            private_node_ip_map,
739            "symmetric",
740        );
741        debug!("Provisioning Symmetric NAT Gateway with vars: {vars}");
742        self.ansible_runner.run_playbook(
743            AnsiblePlaybook::SymmetricNatGateway,
744            AnsibleInventoryType::SymmetricNatGateway,
745            Some(vars),
746        )?;
747
748        print_duration(start.elapsed());
749        Ok(())
750    }
751
752    pub fn provision_nodes(
753        &self,
754        options: &ProvisionOptions,
755        initial_contact_peer: Option<String>,
756        initial_network_contacts_url: Option<String>,
757        node_type: NodeType,
758    ) -> Result<()> {
759        let start = Instant::now();
760        let mut home_network_flag = false;
761        let (inventory_type, node_count) = match &node_type {
762            NodeType::FullConePrivateNode => {
763                home_network_flag = true;
764                (
765                    node_type.to_ansible_inventory_type(),
766                    options.full_cone_private_node_count,
767                )
768            }
769            // use provision_genesis_node fn
770            NodeType::Generic => (node_type.to_ansible_inventory_type(), options.node_count),
771            NodeType::Genesis => return Err(Error::InvalidNodeType(node_type)),
772            NodeType::PeerCache => (
773                node_type.to_ansible_inventory_type(),
774                options.peer_cache_node_count,
775            ),
776            NodeType::SymmetricPrivateNode => {
777                home_network_flag = true;
778                (
779                    node_type.to_ansible_inventory_type(),
780                    options.symmetric_private_node_count,
781                )
782            }
783        };
784
785        // For a new deployment, it's quite probable that SSH is available, because this part occurs
786        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
787        // check that SSH is available before proceeding.
788        println!("Obtaining IP addresses for nodes...");
789        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
790
791        println!("Waiting for SSH availability on {node_type:?} nodes...");
792        for vm in inventory.iter() {
793            println!(
794                "Checking SSH availability for {}: {}",
795                vm.name, vm.public_ip_addr
796            );
797            self.ssh_client
798                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
799                .map_err(|e| {
800                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
801                    e
802                })?;
803        }
804
805        println!("SSH is available on all nodes. Proceeding with provisioning...");
806
807        let playbook = match node_type {
808            NodeType::Generic => AnsiblePlaybook::Nodes,
809            NodeType::PeerCache => AnsiblePlaybook::PeerCacheNodes,
810            NodeType::FullConePrivateNode => AnsiblePlaybook::Nodes,
811            NodeType::SymmetricPrivateNode => AnsiblePlaybook::Nodes,
812            _ => return Err(Error::InvalidNodeType(node_type.clone())),
813        };
814        self.ansible_runner.run_playbook(
815            playbook,
816            inventory_type,
817            Some(extra_vars::build_node_extra_vars_doc(
818                &self.cloud_provider.to_string(),
819                options,
820                node_type.clone(),
821                initial_contact_peer,
822                initial_network_contacts_url,
823                node_count,
824                options.evm_network.clone(),
825                home_network_flag,
826            )?),
827        )?;
828
829        print_duration(start.elapsed());
830        Ok(())
831    }
832
833    pub fn provision_symmetric_private_nodes(
834        &self,
835        options: &mut ProvisionOptions,
836        initial_contact_peer: Option<String>,
837        initial_network_contacts_url: Option<String>,
838        private_node_inventory: &PrivateNodeProvisionInventory,
839    ) -> Result<()> {
840        let start = Instant::now();
841        self.print_ansible_run_banner("Provision Symmetric Private Node Config");
842
843        generate_symmetric_private_node_static_environment_inventory(
844            &options.name,
845            &options.output_inventory_dir_path,
846            &private_node_inventory.symmetric_private_node_vms,
847            &private_node_inventory.symmetric_nat_gateway_vms,
848            &self.ssh_client.private_key_path,
849        )
850        .inspect_err(|err| {
851            error!("Failed to generate symmetric private node static inv with err: {err:?}")
852        })?;
853
854        self.ssh_client.set_symmetric_nat_routed_vms(
855            &private_node_inventory.symmetric_private_node_vms,
856            &private_node_inventory.symmetric_nat_gateway_vms,
857        )?;
858
859        let inventory_type = AnsibleInventoryType::SymmetricPrivateNodes;
860
861        // For a new deployment, it's quite probable that SSH is available, because this part occurs
862        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
863        // check that SSH is available before proceeding.
864        println!("Obtaining IP addresses for nodes...");
865        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
866
867        println!("Waiting for SSH availability on Symmetric Private nodes...");
868        for vm in inventory.iter() {
869            println!(
870                "Checking SSH availability for {}: {}",
871                vm.name, vm.public_ip_addr
872            );
873            self.ssh_client
874                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
875                .map_err(|e| {
876                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
877                    e
878                })?;
879        }
880
881        println!("SSH is available on all nodes. Proceeding with provisioning...");
882
883        self.ansible_runner.run_playbook(
884            AnsiblePlaybook::PrivateNodeConfig,
885            inventory_type,
886            Some(
887                extra_vars::build_symmetric_private_node_config_extra_vars_doc(
888                    private_node_inventory,
889                )?,
890            ),
891        )?;
892
893        println!("Provisioned Symmetric Private Node Config");
894        print_duration(start.elapsed());
895
896        self.provision_nodes(
897            options,
898            initial_contact_peer,
899            initial_network_contacts_url,
900            NodeType::SymmetricPrivateNode,
901        )?;
902
903        Ok(())
904    }
905
906    pub async fn provision_uploaders(
907        &self,
908        options: &ProvisionOptions,
909        genesis_multiaddr: Option<String>,
910        genesis_network_contacts_url: Option<String>,
911    ) -> Result<()> {
912        let start = Instant::now();
913
914        let sk_map = self
915            .deposit_funds_to_uploaders(&FundingOptions {
916                evm_data_payments_address: options.evm_data_payments_address.clone(),
917                evm_network: options.evm_network.clone(),
918                evm_payment_token_address: options.evm_payment_token_address.clone(),
919                evm_rpc_url: options.evm_rpc_url.clone(),
920                funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
921                gas_amount: options.gas_amount,
922                token_amount: options.token_amount,
923                uploaders_count: options.uploaders_count,
924            })
925            .await?;
926
927        println!("Running ansible against uploader machine to start the uploader script.");
928        debug!("Running ansible against uploader machine to start the uploader script.");
929
930        self.ansible_runner.run_playbook(
931            AnsiblePlaybook::Uploaders,
932            AnsibleInventoryType::Uploaders,
933            Some(extra_vars::build_uploaders_extra_vars_doc(
934                &self.cloud_provider.to_string(),
935                options,
936                genesis_multiaddr,
937                genesis_network_contacts_url,
938                &sk_map,
939            )?),
940        )?;
941        print_duration(start.elapsed());
942        Ok(())
943    }
944
945    pub fn start_nodes(
946        &self,
947        environment_name: &str,
948        interval: Duration,
949        node_type: Option<NodeType>,
950        custom_inventory: Option<Vec<VirtualMachine>>,
951    ) -> Result<()> {
952        let mut extra_vars = ExtraVarsDocBuilder::default();
953        extra_vars.add_variable("interval", &interval.as_millis().to_string());
954
955        if let Some(node_type) = node_type {
956            println!("Running the start nodes playbook for {node_type:?} nodes");
957            self.ansible_runner.run_playbook(
958                AnsiblePlaybook::StartNodes,
959                node_type.to_ansible_inventory_type(),
960                Some(extra_vars.build()),
961            )?;
962            return Ok(());
963        }
964
965        if let Some(custom_inventory) = custom_inventory {
966            println!("Running the start nodes playbook with a custom inventory");
967            generate_custom_environment_inventory(
968                &custom_inventory,
969                environment_name,
970                &self.ansible_runner.working_directory_path.join("inventory"),
971            )?;
972            self.ansible_runner.run_playbook(
973                AnsiblePlaybook::StartNodes,
974                AnsibleInventoryType::Custom,
975                Some(extra_vars.build()),
976            )?;
977            return Ok(());
978        }
979
980        println!("Running the start nodes playbook for all node types");
981        for node_inv_type in AnsibleInventoryType::iter_node_type() {
982            self.ansible_runner.run_playbook(
983                AnsiblePlaybook::StartNodes,
984                node_inv_type,
985                Some(extra_vars.build()),
986            )?;
987        }
988        Ok(())
989    }
990
991    pub fn status(&self) -> Result<()> {
992        for node_inv_type in AnsibleInventoryType::iter_node_type() {
993            self.ansible_runner
994                .run_playbook(AnsiblePlaybook::Status, node_inv_type, None)?;
995        }
996        Ok(())
997    }
998
999    pub fn start_telegraf(
1000        &self,
1001        environment_name: &str,
1002        node_type: Option<NodeType>,
1003        custom_inventory: Option<Vec<VirtualMachine>>,
1004    ) -> Result<()> {
1005        if let Some(node_type) = node_type {
1006            println!("Running the start telegraf playbook for {node_type:?} nodes");
1007            self.ansible_runner.run_playbook(
1008                AnsiblePlaybook::StartTelegraf,
1009                node_type.to_ansible_inventory_type(),
1010                None,
1011            )?;
1012            return Ok(());
1013        }
1014
1015        if let Some(custom_inventory) = custom_inventory {
1016            println!("Running the start telegraf playbook with a custom inventory");
1017            generate_custom_environment_inventory(
1018                &custom_inventory,
1019                environment_name,
1020                &self.ansible_runner.working_directory_path.join("inventory"),
1021            )?;
1022            self.ansible_runner.run_playbook(
1023                AnsiblePlaybook::StartTelegraf,
1024                AnsibleInventoryType::Custom,
1025                None,
1026            )?;
1027            return Ok(());
1028        }
1029
1030        println!("Running the start telegraf playbook for all node types");
1031        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1032            self.ansible_runner.run_playbook(
1033                AnsiblePlaybook::StartTelegraf,
1034                node_inv_type,
1035                None,
1036            )?;
1037        }
1038
1039        Ok(())
1040    }
1041
1042    pub fn stop_nodes(
1043        &self,
1044        environment_name: &str,
1045        interval: Duration,
1046        node_type: Option<NodeType>,
1047        custom_inventory: Option<Vec<VirtualMachine>>,
1048        delay: Option<u64>,
1049        service_names: Option<Vec<String>>,
1050    ) -> Result<()> {
1051        let mut extra_vars = ExtraVarsDocBuilder::default();
1052        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1053        if let Some(delay) = delay {
1054            extra_vars.add_variable("delay", &delay.to_string());
1055        }
1056        if let Some(service_names) = service_names {
1057            extra_vars.add_list_variable("service_names", service_names);
1058        }
1059        let extra_vars = extra_vars.build();
1060
1061        if let Some(node_type) = node_type {
1062            println!("Running the stop nodes playbook for {node_type:?} nodes");
1063            self.ansible_runner.run_playbook(
1064                AnsiblePlaybook::StopNodes,
1065                node_type.to_ansible_inventory_type(),
1066                Some(extra_vars),
1067            )?;
1068            return Ok(());
1069        }
1070
1071        if let Some(custom_inventory) = custom_inventory {
1072            println!("Running the stop nodes playbook with a custom inventory");
1073            generate_custom_environment_inventory(
1074                &custom_inventory,
1075                environment_name,
1076                &self.ansible_runner.working_directory_path.join("inventory"),
1077            )?;
1078            self.ansible_runner.run_playbook(
1079                AnsiblePlaybook::StopNodes,
1080                AnsibleInventoryType::Custom,
1081                Some(extra_vars),
1082            )?;
1083            return Ok(());
1084        }
1085
1086        println!("Running the stop nodes playbook for all node types");
1087        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1088            self.ansible_runner.run_playbook(
1089                AnsiblePlaybook::StopNodes,
1090                node_inv_type,
1091                Some(extra_vars.clone()),
1092            )?;
1093        }
1094
1095        Ok(())
1096    }
1097
1098    pub fn stop_telegraf(
1099        &self,
1100        environment_name: &str,
1101        node_type: Option<NodeType>,
1102        custom_inventory: Option<Vec<VirtualMachine>>,
1103    ) -> Result<()> {
1104        if let Some(node_type) = node_type {
1105            println!("Running the stop telegraf playbook for {node_type:?} nodes");
1106            self.ansible_runner.run_playbook(
1107                AnsiblePlaybook::StopTelegraf,
1108                node_type.to_ansible_inventory_type(),
1109                None,
1110            )?;
1111            return Ok(());
1112        }
1113
1114        if let Some(custom_inventory) = custom_inventory {
1115            println!("Running the stop telegraf playbook with a custom inventory");
1116            generate_custom_environment_inventory(
1117                &custom_inventory,
1118                environment_name,
1119                &self.ansible_runner.working_directory_path.join("inventory"),
1120            )?;
1121            self.ansible_runner.run_playbook(
1122                AnsiblePlaybook::StopTelegraf,
1123                AnsibleInventoryType::Custom,
1124                None,
1125            )?;
1126            return Ok(());
1127        }
1128
1129        println!("Running the stop telegraf playbook for all node types");
1130        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1131            self.ansible_runner
1132                .run_playbook(AnsiblePlaybook::StopTelegraf, node_inv_type, None)?;
1133        }
1134
1135        Ok(())
1136    }
1137
1138    pub fn upgrade_node_telegraf(&self, name: &str) -> Result<()> {
1139        self.ansible_runner.run_playbook(
1140            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1141            AnsibleInventoryType::PeerCacheNodes,
1142            Some(extra_vars::build_node_telegraf_upgrade(
1143                name,
1144                &NodeType::PeerCache,
1145            )?),
1146        )?;
1147        self.ansible_runner.run_playbook(
1148            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1149            AnsibleInventoryType::Nodes,
1150            Some(extra_vars::build_node_telegraf_upgrade(
1151                name,
1152                &NodeType::Generic,
1153            )?),
1154        )?;
1155
1156        self.ansible_runner.run_playbook(
1157            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1158            AnsibleInventoryType::SymmetricPrivateNodes,
1159            Some(extra_vars::build_node_telegraf_upgrade(
1160                name,
1161                &NodeType::SymmetricPrivateNode,
1162            )?),
1163        )?;
1164
1165        self.ansible_runner.run_playbook(
1166            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1167            AnsibleInventoryType::FullConePrivateNodes,
1168            Some(extra_vars::build_node_telegraf_upgrade(
1169                name,
1170                &NodeType::FullConePrivateNode,
1171            )?),
1172        )?;
1173        Ok(())
1174    }
1175
1176    pub fn upgrade_uploader_telegraf(&self, name: &str) -> Result<()> {
1177        self.ansible_runner.run_playbook(
1178            AnsiblePlaybook::UpgradeUploaderTelegrafConfig,
1179            AnsibleInventoryType::Uploaders,
1180            Some(extra_vars::build_uploader_telegraf_upgrade(name)?),
1181        )?;
1182        Ok(())
1183    }
1184
1185    pub fn upgrade_nodes(&self, options: &UpgradeOptions) -> Result<()> {
1186        if let Some(custom_inventory) = &options.custom_inventory {
1187            println!("Running the UpgradeNodes with a custom inventory");
1188            generate_custom_environment_inventory(
1189                custom_inventory,
1190                &options.name,
1191                &self.ansible_runner.working_directory_path.join("inventory"),
1192            )?;
1193            match self.ansible_runner.run_playbook(
1194                AnsiblePlaybook::UpgradeNodes,
1195                AnsibleInventoryType::Custom,
1196                Some(options.get_ansible_vars()),
1197            ) {
1198                Ok(()) => println!("All nodes were successfully upgraded"),
1199                Err(_) => {
1200                    println!("WARNING: some nodes may not have been upgraded or restarted");
1201                }
1202            }
1203            return Ok(());
1204        }
1205
1206        if let Some(node_type) = &options.node_type {
1207            println!("Running the UpgradeNodes playbook for {node_type:?} nodes");
1208            match self.ansible_runner.run_playbook(
1209                AnsiblePlaybook::UpgradeNodes,
1210                node_type.to_ansible_inventory_type(),
1211                Some(options.get_ansible_vars()),
1212            ) {
1213                Ok(()) => println!("All {node_type:?} nodes were successfully upgraded"),
1214                Err(_) => {
1215                    println!(
1216                        "WARNING: some {node_type:?} nodes may not have been upgraded or restarted"
1217                    );
1218                }
1219            }
1220            return Ok(());
1221        }
1222
1223        println!("Running the UpgradeNodes playbook for all node types");
1224
1225        match self.ansible_runner.run_playbook(
1226            AnsiblePlaybook::UpgradeNodes,
1227            AnsibleInventoryType::PeerCacheNodes,
1228            Some(options.get_ansible_vars()),
1229        ) {
1230            Ok(()) => println!("All Peer Cache nodes were successfully upgraded"),
1231            Err(_) => {
1232                println!("WARNING: some Peer Cacche nodes may not have been upgraded or restarted");
1233            }
1234        }
1235        match self.ansible_runner.run_playbook(
1236            AnsiblePlaybook::UpgradeNodes,
1237            AnsibleInventoryType::Nodes,
1238            Some(options.get_ansible_vars()),
1239        ) {
1240            Ok(()) => println!("All generic nodes were successfully upgraded"),
1241            Err(_) => {
1242                println!("WARNING: some nodes may not have been upgraded or restarted");
1243            }
1244        }
1245        match self.ansible_runner.run_playbook(
1246            AnsiblePlaybook::UpgradeNodes,
1247            AnsibleInventoryType::SymmetricPrivateNodes,
1248            Some(options.get_ansible_vars()),
1249        ) {
1250            Ok(()) => println!("All private nodes were successfully upgraded"),
1251            Err(_) => {
1252                println!("WARNING: some nodes may not have been upgraded or restarted");
1253            }
1254        }
1255        // Don't use AnsibleInventoryType::iter_node_type() here, because the genesis node should be upgraded last
1256        match self.ansible_runner.run_playbook(
1257            AnsiblePlaybook::UpgradeNodes,
1258            AnsibleInventoryType::Genesis,
1259            Some(options.get_ansible_vars()),
1260        ) {
1261            Ok(()) => println!("The genesis nodes was successfully upgraded"),
1262            Err(_) => {
1263                println!("WARNING: the genesis node may not have been upgraded or restarted");
1264            }
1265        }
1266        Ok(())
1267    }
1268
1269    pub fn upgrade_antctl(
1270        &self,
1271        environment_name: &str,
1272        version: &Version,
1273        node_type: Option<NodeType>,
1274        custom_inventory: Option<Vec<VirtualMachine>>,
1275    ) -> Result<()> {
1276        let mut extra_vars = ExtraVarsDocBuilder::default();
1277        extra_vars.add_variable("version", &version.to_string());
1278
1279        if let Some(node_type) = node_type {
1280            println!("Running the upgrade safenode-manager playbook for {node_type:?} nodes");
1281            self.ansible_runner.run_playbook(
1282                AnsiblePlaybook::UpgradeAntctl,
1283                node_type.to_ansible_inventory_type(),
1284                Some(extra_vars.build()),
1285            )?;
1286            return Ok(());
1287        }
1288
1289        if let Some(custom_inventory) = custom_inventory {
1290            println!("Running the upgrade safenode-manager playbook with a custom inventory");
1291            generate_custom_environment_inventory(
1292                &custom_inventory,
1293                environment_name,
1294                &self.ansible_runner.working_directory_path.join("inventory"),
1295            )?;
1296            self.ansible_runner.run_playbook(
1297                AnsiblePlaybook::UpgradeAntctl,
1298                AnsibleInventoryType::Custom,
1299                Some(extra_vars.build()),
1300            )?;
1301            return Ok(());
1302        }
1303
1304        println!("Running the upgrade safenode-manager playbook for all node types");
1305        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1306            self.ansible_runner.run_playbook(
1307                AnsiblePlaybook::UpgradeAntctl,
1308                node_inv_type,
1309                Some(extra_vars.build()),
1310            )?;
1311        }
1312
1313        Ok(())
1314    }
1315
1316    pub fn print_ansible_run_banner(&self, s: &str) {
1317        let ansible_run_msg = "Ansible Run: ";
1318        let line = "=".repeat(s.len() + ansible_run_msg.len());
1319        println!("{}\n{}{}\n{}", line, ansible_run_msg, s, line);
1320    }
1321}
sn_testnet_deploy/ansible/provisioning.rs

sn_testnet_deploy/ansible/
provisioning.rs
