sn_testnet_deploy/ansible/
provisioning.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use super::{
8    extra_vars::ExtraVarsDocBuilder,
9    inventory::{
10        generate_full_cone_private_node_static_environment_inventory,
11        generate_symmetric_private_node_static_environment_inventory,
12    },
13    AnsibleInventoryType, AnsiblePlaybook, AnsibleRunner,
14};
15use crate::{
16    ansible::inventory::{
17        generate_custom_environment_inventory,
18        generate_full_cone_nat_gateway_static_environment_inventory,
19    },
20    bootstrap::BootstrapOptions,
21    deploy::DeployOptions,
22    error::{Error, Result},
23    funding::FundingOptions,
24    inventory::{DeploymentNodeRegistries, VirtualMachine},
25    print_duration, run_external_command, BinaryOption, CloudProvider, EvmNetwork, LogFormat,
26    NodeType, SshClient, UpgradeOptions,
27};
28use ant_service_management::NodeRegistry;
29use evmlib::common::U256;
30use log::{debug, error, trace};
31use semver::Version;
32use std::{
33    collections::HashMap,
34    net::{IpAddr, SocketAddr},
35    path::PathBuf,
36    time::{Duration, Instant},
37};
38use walkdir::WalkDir;
39
40use crate::ansible::extra_vars;
41
42pub const DEFAULT_BETA_ENCRYPTION_KEY: &str =
43    "49113d2083f57a976076adbe85decb75115820de1e6e74b47e0429338cef124a";
44
45#[derive(Clone)]
46pub struct ProvisionOptions {
47    /// The safe version is also in the binary option, but only for an initial deployment.
48    /// For the upscale, it needs to be provided explicitly, because currently it is not
49    /// recorded in the inventory.
50    pub ant_version: Option<String>,
51    pub binary_option: BinaryOption,
52    pub chunk_size: Option<u64>,
53    pub downloaders_count: u16,
54    pub env_variables: Option<Vec<(String, String)>>,
55    pub evm_data_payments_address: Option<String>,
56    pub evm_network: EvmNetwork,
57    pub evm_payment_token_address: Option<String>,
58    pub evm_rpc_url: Option<String>,
59    pub full_cone_private_node_count: u16,
60    pub funding_wallet_secret_key: Option<String>,
61    pub gas_amount: Option<U256>,
62    pub interval: Duration,
63    pub log_format: Option<LogFormat>,
64    pub logstash_details: Option<(String, Vec<SocketAddr>)>,
65    pub max_archived_log_files: u16,
66    pub max_log_files: u16,
67    pub name: String,
68    pub network_id: Option<u8>,
69    pub node_count: u16,
70    pub output_inventory_dir_path: PathBuf,
71    pub peer_cache_node_count: u16,
72    pub symmetric_private_node_count: u16,
73    pub public_rpc: bool,
74    pub rewards_address: String,
75    pub token_amount: Option<U256>,
76    pub uploaders_count: Option<u16>,
77}
78
79/// These are obtained by running the inventory playbook
80#[derive(Clone, Debug)]
81pub struct PrivateNodeProvisionInventory {
82    pub full_cone_nat_gateway_vms: Vec<VirtualMachine>,
83    pub full_cone_private_node_vms: Vec<VirtualMachine>,
84    pub symmetric_nat_gateway_vms: Vec<VirtualMachine>,
85    pub symmetric_private_node_vms: Vec<VirtualMachine>,
86}
87
88impl PrivateNodeProvisionInventory {
89    pub fn new(
90        provisioner: &AnsibleProvisioner,
91        full_cone_private_node_vm_count: Option<u16>,
92        symmetric_private_node_vm_count: Option<u16>,
93    ) -> Result<Self> {
94        // All the environment types set private_node_vm count to >0 if not specified.
95        let should_provision_full_cone_private_nodes = full_cone_private_node_vm_count
96            .map(|count| count > 0)
97            .unwrap_or(true);
98        let should_provision_symmetric_private_nodes = symmetric_private_node_vm_count
99            .map(|count| count > 0)
100            .unwrap_or(true);
101
102        let mut inventory = Self {
103            full_cone_nat_gateway_vms: Default::default(),
104            full_cone_private_node_vms: Default::default(),
105            symmetric_nat_gateway_vms: Default::default(),
106            symmetric_private_node_vms: Default::default(),
107        };
108
109        if should_provision_full_cone_private_nodes {
110            let full_cone_private_node_vms = provisioner
111                .ansible_runner
112                .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)
113                .inspect_err(|err| {
114                    println!("Failed to obtain the inventory of Full Cone private node: {err:?}");
115                })?;
116
117            let full_cone_nat_gateway_inventory = provisioner
118                .ansible_runner
119                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)
120                .inspect_err(|err| {
121                    println!("Failed to get Full Cone NAT Gateway inventory {err:?}");
122                })?;
123
124            if full_cone_nat_gateway_inventory.len() != full_cone_private_node_vms.len() {
125                println!("The number of Full Cone private nodes does not match the number of Full Cone NAT Gateway VMs");
126                return Err(Error::VmCountMismatch(
127                    Some(AnsibleInventoryType::FullConePrivateNodes),
128                    Some(AnsibleInventoryType::FullConeNatGateway),
129                ));
130            }
131
132            inventory.full_cone_private_node_vms = full_cone_private_node_vms;
133            inventory.full_cone_nat_gateway_vms = full_cone_nat_gateway_inventory;
134        }
135
136        if should_provision_symmetric_private_nodes {
137            let symmetric_private_node_vms = provisioner
138                .ansible_runner
139                .get_inventory(AnsibleInventoryType::SymmetricPrivateNodes, true)
140                .inspect_err(|err| {
141                    println!("Failed to obtain the inventory of Symmetric private node: {err:?}");
142                })?;
143
144            let symmetric_nat_gateway_inventory = provisioner
145                .ansible_runner
146                .get_inventory(AnsibleInventoryType::SymmetricNatGateway, true)
147                .inspect_err(|err| {
148                    println!("Failed to get Symmetric NAT Gateway inventory {err:?}");
149                })?;
150
151            if symmetric_nat_gateway_inventory.len() != symmetric_private_node_vms.len() {
152                println!("The number of Symmetric private nodes does not match the number of Symmetric NAT Gateway VMs");
153                return Err(Error::VmCountMismatch(
154                    Some(AnsibleInventoryType::SymmetricPrivateNodes),
155                    Some(AnsibleInventoryType::SymmetricNatGateway),
156                ));
157            }
158
159            inventory.symmetric_private_node_vms = symmetric_private_node_vms;
160            inventory.symmetric_nat_gateway_vms = symmetric_nat_gateway_inventory;
161        }
162
163        Ok(inventory)
164    }
165
166    pub fn should_provision_full_cone_private_nodes(&self) -> bool {
167        !self.full_cone_private_node_vms.is_empty()
168    }
169
170    pub fn should_provision_symmetric_private_nodes(&self) -> bool {
171        !self.symmetric_private_node_vms.is_empty()
172    }
173
174    pub fn symmetric_private_node_and_gateway_map(
175        &self,
176    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
177        Self::match_private_node_vm_and_gateway_vm(
178            &self.symmetric_private_node_vms,
179            &self.symmetric_nat_gateway_vms,
180        )
181    }
182
183    pub fn full_cone_private_node_and_gateway_map(
184        &self,
185    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
186        Self::match_private_node_vm_and_gateway_vm(
187            &self.full_cone_private_node_vms,
188            &self.full_cone_nat_gateway_vms,
189        )
190    }
191
192    pub fn match_private_node_vm_and_gateway_vm(
193        private_node_vms: &[VirtualMachine],
194        nat_gateway_vms: &[VirtualMachine],
195    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
196        if private_node_vms.len() != nat_gateway_vms.len() {
197            println!(
198            "The number of private node VMs ({}) does not match the number of NAT Gateway VMs ({})",
199            private_node_vms.len(),
200            nat_gateway_vms.len()
201        );
202            error!("The number of private node VMs does not match the number of NAT Gateway VMs: Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}");
203            return Err(Error::VmCountMismatch(None, None));
204        }
205
206        let mut map = HashMap::new();
207        for private_vm in private_node_vms {
208            let nat_gateway = nat_gateway_vms
209                .iter()
210                .find(|vm| {
211                    let private_node_name = private_vm.name.split('-').last().unwrap();
212                    let nat_gateway_name = vm.name.split('-').last().unwrap();
213                    private_node_name == nat_gateway_name
214                })
215                .ok_or_else(|| {
216                    println!(
217                        "Failed to find a matching NAT Gateway for private node: {}",
218                        private_vm.name
219                    );
220                    error!("Failed to find a matching NAT Gateway for private node: {}. Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}", private_vm.name);
221                    Error::VmCountMismatch(None, None)
222                })?;
223
224            let _ = map.insert(private_vm.clone(), nat_gateway.clone());
225        }
226
227        Ok(map)
228    }
229}
230
231impl From<BootstrapOptions> for ProvisionOptions {
232    fn from(bootstrap_options: BootstrapOptions) -> Self {
233        ProvisionOptions {
234            ant_version: None,
235            binary_option: bootstrap_options.binary_option,
236            chunk_size: bootstrap_options.chunk_size,
237            downloaders_count: 0,
238            env_variables: bootstrap_options.env_variables,
239            evm_data_payments_address: bootstrap_options.evm_data_payments_address,
240            evm_network: bootstrap_options.evm_network,
241            evm_payment_token_address: bootstrap_options.evm_payment_token_address,
242            evm_rpc_url: bootstrap_options.evm_rpc_url,
243            full_cone_private_node_count: bootstrap_options.full_cone_private_node_count,
244            funding_wallet_secret_key: None,
245            gas_amount: None,
246            interval: bootstrap_options.interval,
247            log_format: bootstrap_options.log_format,
248            logstash_details: None,
249            max_archived_log_files: bootstrap_options.max_archived_log_files,
250            max_log_files: bootstrap_options.max_log_files,
251            name: bootstrap_options.name,
252            network_id: bootstrap_options.network_id,
253            node_count: bootstrap_options.node_count,
254            output_inventory_dir_path: bootstrap_options.output_inventory_dir_path,
255            peer_cache_node_count: 0,
256            symmetric_private_node_count: bootstrap_options.symmetric_private_node_count,
257            public_rpc: false,
258            rewards_address: bootstrap_options.rewards_address,
259            token_amount: None,
260            uploaders_count: None,
261        }
262    }
263}
264
265impl From<DeployOptions> for ProvisionOptions {
266    fn from(deploy_options: DeployOptions) -> Self {
267        ProvisionOptions {
268            ant_version: None,
269            binary_option: deploy_options.binary_option,
270            chunk_size: deploy_options.chunk_size,
271            downloaders_count: deploy_options.downloaders_count,
272            env_variables: deploy_options.env_variables,
273            evm_data_payments_address: deploy_options.evm_data_payments_address,
274            evm_network: deploy_options.evm_network,
275            evm_payment_token_address: deploy_options.evm_payment_token_address,
276            evm_rpc_url: deploy_options.evm_rpc_url,
277            full_cone_private_node_count: deploy_options.full_cone_private_node_count,
278            funding_wallet_secret_key: deploy_options.funding_wallet_secret_key,
279            gas_amount: deploy_options.initial_gas,
280            interval: deploy_options.interval,
281            log_format: deploy_options.log_format,
282            logstash_details: deploy_options.logstash_details,
283            max_archived_log_files: deploy_options.max_archived_log_files,
284            max_log_files: deploy_options.max_log_files,
285            name: deploy_options.name,
286            network_id: deploy_options.network_id,
287            node_count: deploy_options.node_count,
288            output_inventory_dir_path: deploy_options.output_inventory_dir_path,
289            peer_cache_node_count: deploy_options.peer_cache_node_count,
290            symmetric_private_node_count: deploy_options.symmetric_private_node_count,
291            public_rpc: deploy_options.public_rpc,
292            rewards_address: deploy_options.rewards_address,
293            token_amount: deploy_options.initial_tokens,
294            uploaders_count: Some(deploy_options.uploaders_count),
295        }
296    }
297}
298
299#[derive(Clone)]
300pub struct AnsibleProvisioner {
301    pub ansible_runner: AnsibleRunner,
302    pub cloud_provider: CloudProvider,
303    pub ssh_client: SshClient,
304}
305
306impl AnsibleProvisioner {
307    pub fn new(
308        ansible_runner: AnsibleRunner,
309        cloud_provider: CloudProvider,
310        ssh_client: SshClient,
311    ) -> Self {
312        Self {
313            ansible_runner,
314            cloud_provider,
315            ssh_client,
316        }
317    }
318
319    pub fn build_safe_network_binaries(&self, options: &ProvisionOptions) -> Result<()> {
320        let start = Instant::now();
321        println!("Obtaining IP address for build VM...");
322        let build_inventory = self
323            .ansible_runner
324            .get_inventory(AnsibleInventoryType::Build, true)?;
325        let build_ip = build_inventory[0].public_ip_addr;
326        self.ssh_client
327            .wait_for_ssh_availability(&build_ip, &self.cloud_provider.get_ssh_user())?;
328
329        println!("Running ansible against build VM...");
330        let extra_vars = extra_vars::build_binaries_extra_vars_doc(options)?;
331        self.ansible_runner.run_playbook(
332            AnsiblePlaybook::Build,
333            AnsibleInventoryType::Build,
334            Some(extra_vars),
335        )?;
336        print_duration(start.elapsed());
337        Ok(())
338    }
339
340    pub fn cleanup_node_logs(&self, setup_cron: bool) -> Result<()> {
341        for node_inv_type in AnsibleInventoryType::iter_node_type() {
342            self.ansible_runner.run_playbook(
343                AnsiblePlaybook::CleanupLogs,
344                node_inv_type,
345                Some(format!("{{ \"setup_cron\": \"{setup_cron}\" }}")),
346            )?;
347        }
348
349        Ok(())
350    }
351
352    pub fn copy_logs(&self, name: &str, resources_only: bool) -> Result<()> {
353        for node_inv_type in AnsibleInventoryType::iter_node_type() {
354            self.ansible_runner.run_playbook(
355                AnsiblePlaybook::CopyLogs,
356                node_inv_type,
357                Some(format!(
358                    "{{ \"env_name\": \"{name}\", \"resources_only\" : \"{resources_only}\" }}"
359                )),
360            )?;
361        }
362        Ok(())
363    }
364
365    pub fn get_all_node_inventory(&self) -> Result<Vec<VirtualMachine>> {
366        let mut all_node_inventory = Vec::new();
367        for node_inv_type in AnsibleInventoryType::iter_node_type() {
368            all_node_inventory.extend(self.ansible_runner.get_inventory(node_inv_type, false)?);
369        }
370
371        Ok(all_node_inventory)
372    }
373
374    pub fn get_symmetric_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
375        self.ansible_runner
376            .get_inventory(AnsibleInventoryType::SymmetricNatGateway, false)
377    }
378
379    pub fn get_full_cone_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
380        self.ansible_runner
381            .get_inventory(AnsibleInventoryType::FullConeNatGateway, false)
382    }
383
384    pub fn get_node_registries(
385        &self,
386        inventory_type: &AnsibleInventoryType,
387    ) -> Result<DeploymentNodeRegistries> {
388        debug!("Fetching node manager inventory for {inventory_type:?}");
389        let temp_dir_path = tempfile::tempdir()?.into_path();
390        let temp_dir_json = serde_json::to_string(&temp_dir_path)?;
391
392        self.ansible_runner.run_playbook(
393            AnsiblePlaybook::AntCtlInventory,
394            *inventory_type,
395            Some(format!("{{ \"dest\": {temp_dir_json} }}")),
396        )?;
397
398        let node_registry_paths = WalkDir::new(temp_dir_path)
399            .into_iter()
400            .flatten()
401            .filter_map(|entry| {
402                if entry.file_type().is_file()
403                    && entry.path().extension().is_some_and(|ext| ext == "json")
404                {
405                    // tempdir/<testnet_name>-node/var/safenode-manager/node_registry.json
406                    let mut vm_name = entry.path().to_path_buf();
407                    trace!("Found file with json extension: {vm_name:?}");
408                    vm_name.pop();
409                    vm_name.pop();
410                    vm_name.pop();
411                    // Extract the <testnet_name>-node string
412                    trace!("Extracting the vm name from the path");
413                    let vm_name = vm_name.file_name()?.to_str()?;
414                    trace!("Extracted vm name from path: {vm_name}");
415                    Some((vm_name.to_string(), entry.path().to_path_buf()))
416                } else {
417                    None
418                }
419            })
420            .collect::<Vec<(String, PathBuf)>>();
421
422        let mut node_registries = Vec::new();
423        let mut failed_vms = Vec::new();
424        for (vm_name, file_path) in node_registry_paths {
425            match NodeRegistry::load(&file_path) {
426                Ok(node_registry) => node_registries.push((vm_name.clone(), node_registry)),
427                Err(_) => failed_vms.push(vm_name.clone()),
428            }
429        }
430
431        let deployment_registries = DeploymentNodeRegistries {
432            inventory_type: *inventory_type,
433            retrieved_registries: node_registries,
434            failed_vms,
435        };
436        Ok(deployment_registries)
437    }
438
439    pub fn provision_evm_nodes(&self, options: &ProvisionOptions) -> Result<()> {
440        let start = Instant::now();
441        println!("Obtaining IP address for EVM nodes...");
442        let evm_node_inventory = self
443            .ansible_runner
444            .get_inventory(AnsibleInventoryType::EvmNodes, true)?;
445        let evm_node_ip = evm_node_inventory[0].public_ip_addr;
446        self.ssh_client
447            .wait_for_ssh_availability(&evm_node_ip, &self.cloud_provider.get_ssh_user())?;
448
449        println!("Running ansible against EVM nodes...");
450        self.ansible_runner.run_playbook(
451            AnsiblePlaybook::EvmNodes,
452            AnsibleInventoryType::EvmNodes,
453            Some(extra_vars::build_evm_nodes_extra_vars_doc(
454                &options.name,
455                &self.cloud_provider,
456            )),
457        )?;
458        print_duration(start.elapsed());
459        Ok(())
460    }
461
462    pub fn provision_genesis_node(&self, options: &ProvisionOptions) -> Result<()> {
463        let start = Instant::now();
464        let genesis_inventory = self
465            .ansible_runner
466            .get_inventory(AnsibleInventoryType::Genesis, true)?;
467        let genesis_ip = genesis_inventory[0].public_ip_addr;
468        self.ssh_client
469            .wait_for_ssh_availability(&genesis_ip, &self.cloud_provider.get_ssh_user())?;
470        self.ansible_runner.run_playbook(
471            AnsiblePlaybook::Genesis,
472            AnsibleInventoryType::Genesis,
473            Some(extra_vars::build_node_extra_vars_doc(
474                &self.cloud_provider.to_string(),
475                options,
476                NodeType::Genesis,
477                None,
478                None,
479                1,
480                options.evm_network.clone(),
481                false,
482            )?),
483        )?;
484
485        print_duration(start.elapsed());
486
487        Ok(())
488    }
489
490    pub fn provision_full_cone(
491        &self,
492        options: &ProvisionOptions,
493        initial_contact_peer: Option<String>,
494        initial_network_contacts_url: Option<String>,
495        private_node_inventory: PrivateNodeProvisionInventory,
496        new_full_cone_nat_gateway_new_vms_for_upscale: Option<Vec<VirtualMachine>>,
497    ) -> Result<()> {
498        // Step 1 of Full Cone NAT Gateway
499        let start = Instant::now();
500        self.print_ansible_run_banner("Provision Full Cone NAT Gateway - Step 1");
501
502        for vm in new_full_cone_nat_gateway_new_vms_for_upscale
503            .as_ref()
504            .unwrap_or(&private_node_inventory.full_cone_nat_gateway_vms)
505            .iter()
506        {
507            println!(
508                "Checking SSH availability for Full Cone NAT Gateway: {}",
509                vm.public_ip_addr
510            );
511            self.ssh_client
512                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
513                .map_err(|e| {
514                    println!(
515                        "Failed to establish SSH connection to Full Cone NAT Gateway: {}",
516                        e
517                    );
518                    e
519                })?;
520        }
521
522        let mut modified_private_node_inventory = private_node_inventory.clone();
523
524        // If we are upscaling, then we cannot access the gateway VMs which are already deployed.
525        if let Some(new_full_cone_nat_gateway_new_vms_for_upscale) =
526            &new_full_cone_nat_gateway_new_vms_for_upscale
527        {
528            debug!("Removing existing full cone NAT Gateway and private node VMs from the inventory. Old inventory: {modified_private_node_inventory:?}");
529            let mut names_to_keep = Vec::new();
530
531            for vm in new_full_cone_nat_gateway_new_vms_for_upscale.iter() {
532                let nat_gateway_name = vm.name.split('-').last().unwrap();
533                names_to_keep.push(nat_gateway_name);
534            }
535
536            modified_private_node_inventory
537                .full_cone_nat_gateway_vms
538                .retain(|vm| {
539                    let nat_gateway_name = vm.name.split('-').last().unwrap();
540                    names_to_keep.contains(&nat_gateway_name)
541                });
542            modified_private_node_inventory
543                .full_cone_private_node_vms
544                .retain(|vm| {
545                    let nat_gateway_name = vm.name.split('-').last().unwrap();
546                    names_to_keep.contains(&nat_gateway_name)
547                });
548            debug!("New inventory after removing existing full cone NAT Gateway and private node VMs: {modified_private_node_inventory:?}");
549        }
550
551        if modified_private_node_inventory
552            .full_cone_nat_gateway_vms
553            .is_empty()
554        {
555            error!("There are no full cone NAT Gateway VMs available to upscale");
556            return Ok(());
557        }
558
559        let private_node_ip_map = modified_private_node_inventory
560            .full_cone_private_node_and_gateway_map()?
561            .into_iter()
562            .map(|(k, v)| {
563                let gateway_name = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
564                    debug!("Upscaling, using public IP address for gateway name");
565                    v.public_ip_addr.to_string()
566                } else {
567                    v.name.clone()
568                };
569                (gateway_name, k.private_ip_addr)
570            })
571            .collect::<HashMap<String, IpAddr>>();
572
573        if private_node_ip_map.is_empty() {
574            println!("There are no full cone private node VM available to be routed through the full cone NAT Gateway");
575            return Err(Error::EmptyInventory(
576                AnsibleInventoryType::FullConePrivateNodes,
577            ));
578        }
579
580        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
581            &options.name,
582            private_node_ip_map.clone(),
583            "step1",
584        );
585        debug!("Provisioning Full Cone NAT Gateway - Step 1 with vars: {vars}");
586        let gateway_inventory = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
587            debug!("Upscaling, using static inventory for full cone nat gateway.");
588            generate_full_cone_nat_gateway_static_environment_inventory(
589                &modified_private_node_inventory.full_cone_nat_gateway_vms,
590                &options.name,
591                &options.output_inventory_dir_path,
592            )?;
593
594            AnsibleInventoryType::FullConeNatGatewayStatic
595        } else {
596            AnsibleInventoryType::FullConeNatGateway
597        };
598        self.ansible_runner.run_playbook(
599            AnsiblePlaybook::FullConeNatGateway,
600            gateway_inventory,
601            Some(vars),
602        )?;
603
604        // setup private node config
605        self.print_ansible_run_banner("Provisioning Full Cone Private Node Config");
606
607        generate_full_cone_private_node_static_environment_inventory(
608            &options.name,
609            &options.output_inventory_dir_path,
610            &private_node_inventory.full_cone_private_node_vms,
611            &private_node_inventory.full_cone_nat_gateway_vms,
612            &self.ssh_client.private_key_path,
613        )
614        .inspect_err(|err| {
615            error!("Failed to generate full cone private node static inv with err: {err:?}")
616        })?;
617
618        // For a new deployment, it's quite probable that SSH is available, because this part occurs
619        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
620        // check that SSH is available before proceeding.
621        println!("Obtaining IP addresses for nodes...");
622        let inventory = self
623            .ansible_runner
624            .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)?;
625
626        println!("Waiting for SSH availability on Symmetric Private nodes...");
627        for vm in inventory.iter() {
628            println!(
629                "Checking SSH availability for {}: {}",
630                vm.name, vm.public_ip_addr
631            );
632            self.ssh_client
633                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
634                .map_err(|e| {
635                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
636                    e
637                })?;
638        }
639
640        println!("SSH is available on all nodes. Proceeding with provisioning...");
641
642        self.ansible_runner.run_playbook(
643            AnsiblePlaybook::PrivateNodeConfig,
644            AnsibleInventoryType::FullConePrivateNodes,
645            Some(
646                extra_vars::build_full_cone_private_node_config_extra_vars_docs(
647                    &private_node_inventory,
648                )?,
649            ),
650        )?;
651
652        // Step 2 of Full Cone NAT Gateway
653
654        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
655            &options.name,
656            private_node_ip_map,
657            "step2",
658        );
659
660        self.print_ansible_run_banner("Provisioning Full Cone NAT Gateway - Step 2");
661        debug!("Provisioning Full Cone NAT Gateway - Step 2 with vars: {vars}");
662        self.ansible_runner.run_playbook(
663            AnsiblePlaybook::FullConeNatGateway,
664            gateway_inventory,
665            Some(vars),
666        )?;
667
668        // provision the nodes
669
670        let home_dir = std::env::var("HOME").inspect_err(|err| {
671            println!("Failed to get home directory with error: {err:?}",);
672        })?;
673        let known_hosts_path = format!("{}/.ssh/known_hosts", home_dir);
674        debug!("Cleaning up known hosts file at {known_hosts_path} ");
675        run_external_command(
676            PathBuf::from("rm"),
677            std::env::current_dir()?,
678            vec![known_hosts_path],
679            false,
680            false,
681        )?;
682
683        self.print_ansible_run_banner("Provision Full Cone Private Nodes");
684
685        self.ssh_client.set_full_cone_nat_routed_vms(
686            &private_node_inventory.full_cone_private_node_vms,
687            &private_node_inventory.full_cone_nat_gateway_vms,
688        )?;
689
690        self.provision_nodes(
691            options,
692            initial_contact_peer,
693            initial_network_contacts_url,
694            NodeType::FullConePrivateNode,
695        )?;
696
697        print_duration(start.elapsed());
698        Ok(())
699    }
700    pub fn provision_symmetric_nat_gateway(
701        &self,
702        options: &ProvisionOptions,
703        private_node_inventory: &PrivateNodeProvisionInventory,
704    ) -> Result<()> {
705        let start = Instant::now();
706        for vm in &private_node_inventory.symmetric_nat_gateway_vms {
707            println!(
708                "Checking SSH availability for Symmetric NAT Gateway: {}",
709                vm.public_ip_addr
710            );
711            self.ssh_client
712                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
713                .map_err(|e| {
714                    println!(
715                        "Failed to establish SSH connection to Symmetric NAT Gateway: {}",
716                        e
717                    );
718                    e
719                })?;
720        }
721
722        let private_node_ip_map = private_node_inventory
723            .symmetric_private_node_and_gateway_map()?
724            .into_iter()
725            .map(|(k, v)| (v.name.clone(), k.private_ip_addr))
726            .collect::<HashMap<String, IpAddr>>();
727
728        if private_node_ip_map.is_empty() {
729            println!("There are no Symmetric private node VM available to be routed through the Symmetric NAT Gateway");
730            return Err(Error::EmptyInventory(
731                AnsibleInventoryType::SymmetricPrivateNodes,
732            ));
733        }
734
735        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
736            &options.name,
737            private_node_ip_map,
738            "symmetric",
739        );
740        debug!("Provisioning Symmetric NAT Gateway with vars: {vars}");
741        self.ansible_runner.run_playbook(
742            AnsiblePlaybook::SymmetricNatGateway,
743            AnsibleInventoryType::SymmetricNatGateway,
744            Some(vars),
745        )?;
746
747        print_duration(start.elapsed());
748        Ok(())
749    }
750
751    pub fn provision_nodes(
752        &self,
753        options: &ProvisionOptions,
754        initial_contact_peer: Option<String>,
755        initial_network_contacts_url: Option<String>,
756        node_type: NodeType,
757    ) -> Result<()> {
758        let start = Instant::now();
759        let mut home_network_flag = false;
760        let (inventory_type, node_count) = match &node_type {
761            NodeType::FullConePrivateNode => {
762                home_network_flag = true;
763                (
764                    node_type.to_ansible_inventory_type(),
765                    options.full_cone_private_node_count,
766                )
767            }
768            // use provision_genesis_node fn
769            NodeType::Generic => (node_type.to_ansible_inventory_type(), options.node_count),
770            NodeType::Genesis => return Err(Error::InvalidNodeType(node_type)),
771            NodeType::PeerCache => (
772                node_type.to_ansible_inventory_type(),
773                options.peer_cache_node_count,
774            ),
775            NodeType::SymmetricPrivateNode => {
776                home_network_flag = true;
777                (
778                    node_type.to_ansible_inventory_type(),
779                    options.symmetric_private_node_count,
780                )
781            }
782        };
783
784        // For a new deployment, it's quite probable that SSH is available, because this part occurs
785        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
786        // check that SSH is available before proceeding.
787        println!("Obtaining IP addresses for nodes...");
788        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
789
790        println!("Waiting for SSH availability on {node_type:?} nodes...");
791        for vm in inventory.iter() {
792            println!(
793                "Checking SSH availability for {}: {}",
794                vm.name, vm.public_ip_addr
795            );
796            self.ssh_client
797                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
798                .map_err(|e| {
799                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
800                    e
801                })?;
802        }
803
804        println!("SSH is available on all nodes. Proceeding with provisioning...");
805
806        self.ansible_runner.run_playbook(
807            AnsiblePlaybook::Nodes,
808            inventory_type,
809            Some(extra_vars::build_node_extra_vars_doc(
810                &self.cloud_provider.to_string(),
811                options,
812                node_type.clone(),
813                initial_contact_peer,
814                initial_network_contacts_url,
815                node_count,
816                options.evm_network.clone(),
817                home_network_flag,
818            )?),
819        )?;
820
821        print_duration(start.elapsed());
822        Ok(())
823    }
824
825    pub fn provision_symmetric_private_nodes(
826        &self,
827        options: &mut ProvisionOptions,
828        initial_contact_peer: Option<String>,
829        initial_network_contacts_url: Option<String>,
830        private_node_inventory: &PrivateNodeProvisionInventory,
831    ) -> Result<()> {
832        let start = Instant::now();
833        self.print_ansible_run_banner("Provision Symmetric Private Node Config");
834
835        generate_symmetric_private_node_static_environment_inventory(
836            &options.name,
837            &options.output_inventory_dir_path,
838            &private_node_inventory.symmetric_private_node_vms,
839            &private_node_inventory.symmetric_nat_gateway_vms,
840            &self.ssh_client.private_key_path,
841        )
842        .inspect_err(|err| {
843            error!("Failed to generate symmetric private node static inv with err: {err:?}")
844        })?;
845
846        self.ssh_client.set_symmetric_nat_routed_vms(
847            &private_node_inventory.symmetric_private_node_vms,
848            &private_node_inventory.symmetric_nat_gateway_vms,
849        )?;
850
851        let inventory_type = AnsibleInventoryType::SymmetricPrivateNodes;
852
853        // For a new deployment, it's quite probable that SSH is available, because this part occurs
854        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
855        // check that SSH is available before proceeding.
856        println!("Obtaining IP addresses for nodes...");
857        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
858
859        println!("Waiting for SSH availability on Symmetric Private nodes...");
860        for vm in inventory.iter() {
861            println!(
862                "Checking SSH availability for {}: {}",
863                vm.name, vm.public_ip_addr
864            );
865            self.ssh_client
866                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
867                .map_err(|e| {
868                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
869                    e
870                })?;
871        }
872
873        println!("SSH is available on all nodes. Proceeding with provisioning...");
874
875        self.ansible_runner.run_playbook(
876            AnsiblePlaybook::PrivateNodeConfig,
877            inventory_type,
878            Some(
879                extra_vars::build_symmetric_private_node_config_extra_vars_doc(
880                    private_node_inventory,
881                )?,
882            ),
883        )?;
884
885        println!("Provisioned Symmetric Private Node Config");
886        print_duration(start.elapsed());
887
888        self.provision_nodes(
889            options,
890            initial_contact_peer,
891            initial_network_contacts_url,
892            NodeType::SymmetricPrivateNode,
893        )?;
894
895        Ok(())
896    }
897
898    pub async fn provision_uploaders(
899        &self,
900        options: &ProvisionOptions,
901        genesis_multiaddr: Option<String>,
902        genesis_network_contacts_url: Option<String>,
903    ) -> Result<()> {
904        let start = Instant::now();
905
906        let sk_map = self
907            .deposit_funds_to_uploaders(&FundingOptions {
908                evm_data_payments_address: options.evm_data_payments_address.clone(),
909                evm_network: options.evm_network.clone(),
910                evm_payment_token_address: options.evm_payment_token_address.clone(),
911                evm_rpc_url: options.evm_rpc_url.clone(),
912                funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
913                gas_amount: options.gas_amount,
914                token_amount: options.token_amount,
915                uploaders_count: options.uploaders_count,
916            })
917            .await?;
918
919        println!("Running ansible against uploader machine to start the uploader script.");
920        debug!("Running ansible against uploader machine to start the uploader script.");
921
922        self.ansible_runner.run_playbook(
923            AnsiblePlaybook::Uploaders,
924            AnsibleInventoryType::Uploaders,
925            Some(extra_vars::build_uploaders_extra_vars_doc(
926                &self.cloud_provider.to_string(),
927                options,
928                genesis_multiaddr,
929                genesis_network_contacts_url,
930                &sk_map,
931            )?),
932        )?;
933        print_duration(start.elapsed());
934        Ok(())
935    }
936
937    pub fn start_nodes(
938        &self,
939        environment_name: &str,
940        interval: Duration,
941        node_type: Option<NodeType>,
942        custom_inventory: Option<Vec<VirtualMachine>>,
943    ) -> Result<()> {
944        let mut extra_vars = ExtraVarsDocBuilder::default();
945        extra_vars.add_variable("interval", &interval.as_millis().to_string());
946
947        if let Some(node_type) = node_type {
948            println!("Running the start nodes playbook for {node_type:?} nodes");
949            self.ansible_runner.run_playbook(
950                AnsiblePlaybook::StartNodes,
951                node_type.to_ansible_inventory_type(),
952                Some(extra_vars.build()),
953            )?;
954            return Ok(());
955        }
956
957        if let Some(custom_inventory) = custom_inventory {
958            println!("Running the start nodes playbook with a custom inventory");
959            generate_custom_environment_inventory(
960                &custom_inventory,
961                environment_name,
962                &self.ansible_runner.working_directory_path.join("inventory"),
963            )?;
964            self.ansible_runner.run_playbook(
965                AnsiblePlaybook::StartNodes,
966                AnsibleInventoryType::Custom,
967                Some(extra_vars.build()),
968            )?;
969            return Ok(());
970        }
971
972        println!("Running the start nodes playbook for all node types");
973        for node_inv_type in AnsibleInventoryType::iter_node_type() {
974            self.ansible_runner.run_playbook(
975                AnsiblePlaybook::StartNodes,
976                node_inv_type,
977                Some(extra_vars.build()),
978            )?;
979        }
980        Ok(())
981    }
982
983    pub fn status(&self) -> Result<()> {
984        for node_inv_type in AnsibleInventoryType::iter_node_type() {
985            self.ansible_runner
986                .run_playbook(AnsiblePlaybook::Status, node_inv_type, None)?;
987        }
988        Ok(())
989    }
990
991    pub fn start_telegraf(
992        &self,
993        environment_name: &str,
994        node_type: Option<NodeType>,
995        custom_inventory: Option<Vec<VirtualMachine>>,
996    ) -> Result<()> {
997        if let Some(node_type) = node_type {
998            println!("Running the start telegraf playbook for {node_type:?} nodes");
999            self.ansible_runner.run_playbook(
1000                AnsiblePlaybook::StartTelegraf,
1001                node_type.to_ansible_inventory_type(),
1002                None,
1003            )?;
1004            return Ok(());
1005        }
1006
1007        if let Some(custom_inventory) = custom_inventory {
1008            println!("Running the start telegraf playbook with a custom inventory");
1009            generate_custom_environment_inventory(
1010                &custom_inventory,
1011                environment_name,
1012                &self.ansible_runner.working_directory_path.join("inventory"),
1013            )?;
1014            self.ansible_runner.run_playbook(
1015                AnsiblePlaybook::StartTelegraf,
1016                AnsibleInventoryType::Custom,
1017                None,
1018            )?;
1019            return Ok(());
1020        }
1021
1022        println!("Running the start telegraf playbook for all node types");
1023        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1024            self.ansible_runner.run_playbook(
1025                AnsiblePlaybook::StartTelegraf,
1026                node_inv_type,
1027                None,
1028            )?;
1029        }
1030
1031        Ok(())
1032    }
1033
1034    pub fn stop_nodes(
1035        &self,
1036        environment_name: &str,
1037        interval: Duration,
1038        node_type: Option<NodeType>,
1039        custom_inventory: Option<Vec<VirtualMachine>>,
1040        delay: Option<u64>,
1041    ) -> Result<()> {
1042        let mut extra_vars = ExtraVarsDocBuilder::default();
1043        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1044        if let Some(delay) = delay {
1045            extra_vars.add_variable("delay", &delay.to_string());
1046        }
1047        let extra_vars = extra_vars.build();
1048
1049        if let Some(node_type) = node_type {
1050            println!("Running the stop nodes playbook for {node_type:?} nodes");
1051            self.ansible_runner.run_playbook(
1052                AnsiblePlaybook::StopNodes,
1053                node_type.to_ansible_inventory_type(),
1054                Some(extra_vars),
1055            )?;
1056            return Ok(());
1057        }
1058
1059        if let Some(custom_inventory) = custom_inventory {
1060            println!("Running the stop nodes playbook with a custom inventory");
1061            generate_custom_environment_inventory(
1062                &custom_inventory,
1063                environment_name,
1064                &self.ansible_runner.working_directory_path.join("inventory"),
1065            )?;
1066            self.ansible_runner.run_playbook(
1067                AnsiblePlaybook::StopNodes,
1068                AnsibleInventoryType::Custom,
1069                Some(extra_vars),
1070            )?;
1071            return Ok(());
1072        }
1073
1074        println!("Running the stop nodes playbook for all node types");
1075        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1076            self.ansible_runner.run_playbook(
1077                AnsiblePlaybook::StopNodes,
1078                node_inv_type,
1079                Some(extra_vars.clone()),
1080            )?;
1081        }
1082
1083        Ok(())
1084    }
1085
1086    pub fn stop_telegraf(
1087        &self,
1088        environment_name: &str,
1089        node_type: Option<NodeType>,
1090        custom_inventory: Option<Vec<VirtualMachine>>,
1091    ) -> Result<()> {
1092        if let Some(node_type) = node_type {
1093            println!("Running the stop telegraf playbook for {node_type:?} nodes");
1094            self.ansible_runner.run_playbook(
1095                AnsiblePlaybook::StopTelegraf,
1096                node_type.to_ansible_inventory_type(),
1097                None,
1098            )?;
1099            return Ok(());
1100        }
1101
1102        if let Some(custom_inventory) = custom_inventory {
1103            println!("Running the stop telegraf playbook with a custom inventory");
1104            generate_custom_environment_inventory(
1105                &custom_inventory,
1106                environment_name,
1107                &self.ansible_runner.working_directory_path.join("inventory"),
1108            )?;
1109            self.ansible_runner.run_playbook(
1110                AnsiblePlaybook::StopTelegraf,
1111                AnsibleInventoryType::Custom,
1112                None,
1113            )?;
1114            return Ok(());
1115        }
1116
1117        println!("Running the stop telegraf playbook for all node types");
1118        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1119            self.ansible_runner
1120                .run_playbook(AnsiblePlaybook::StopTelegraf, node_inv_type, None)?;
1121        }
1122
1123        Ok(())
1124    }
1125
1126    pub fn upgrade_node_telegraf(&self, name: &str) -> Result<()> {
1127        self.ansible_runner.run_playbook(
1128            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1129            AnsibleInventoryType::PeerCacheNodes,
1130            Some(extra_vars::build_node_telegraf_upgrade(
1131                name,
1132                &NodeType::PeerCache,
1133            )?),
1134        )?;
1135        self.ansible_runner.run_playbook(
1136            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1137            AnsibleInventoryType::Nodes,
1138            Some(extra_vars::build_node_telegraf_upgrade(
1139                name,
1140                &NodeType::Generic,
1141            )?),
1142        )?;
1143
1144        self.ansible_runner.run_playbook(
1145            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1146            AnsibleInventoryType::SymmetricPrivateNodes,
1147            Some(extra_vars::build_node_telegraf_upgrade(
1148                name,
1149                &NodeType::SymmetricPrivateNode,
1150            )?),
1151        )?;
1152
1153        self.ansible_runner.run_playbook(
1154            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1155            AnsibleInventoryType::FullConePrivateNodes,
1156            Some(extra_vars::build_node_telegraf_upgrade(
1157                name,
1158                &NodeType::FullConePrivateNode,
1159            )?),
1160        )?;
1161        Ok(())
1162    }
1163
1164    pub fn upgrade_uploader_telegraf(&self, name: &str) -> Result<()> {
1165        self.ansible_runner.run_playbook(
1166            AnsiblePlaybook::UpgradeUploaderTelegrafConfig,
1167            AnsibleInventoryType::Uploaders,
1168            Some(extra_vars::build_uploader_telegraf_upgrade(name)?),
1169        )?;
1170        Ok(())
1171    }
1172
1173    pub fn upgrade_nodes(&self, options: &UpgradeOptions) -> Result<()> {
1174        if let Some(custom_inventory) = &options.custom_inventory {
1175            println!("Running the UpgradeNodes with a custom inventory");
1176            generate_custom_environment_inventory(
1177                custom_inventory,
1178                &options.name,
1179                &self.ansible_runner.working_directory_path.join("inventory"),
1180            )?;
1181            match self.ansible_runner.run_playbook(
1182                AnsiblePlaybook::UpgradeNodes,
1183                AnsibleInventoryType::Custom,
1184                Some(options.get_ansible_vars()),
1185            ) {
1186                Ok(()) => println!("All nodes were successfully upgraded"),
1187                Err(_) => {
1188                    println!("WARNING: some nodes may not have been upgraded or restarted");
1189                }
1190            }
1191            return Ok(());
1192        }
1193
1194        if let Some(node_type) = &options.node_type {
1195            println!("Running the UpgradeNodes playbook for {node_type:?} nodes");
1196            match self.ansible_runner.run_playbook(
1197                AnsiblePlaybook::UpgradeNodes,
1198                node_type.to_ansible_inventory_type(),
1199                Some(options.get_ansible_vars()),
1200            ) {
1201                Ok(()) => println!("All {node_type:?} nodes were successfully upgraded"),
1202                Err(_) => {
1203                    println!(
1204                        "WARNING: some {node_type:?} nodes may not have been upgraded or restarted"
1205                    );
1206                }
1207            }
1208            return Ok(());
1209        }
1210
1211        println!("Running the UpgradeNodes playbook for all node types");
1212
1213        match self.ansible_runner.run_playbook(
1214            AnsiblePlaybook::UpgradeNodes,
1215            AnsibleInventoryType::PeerCacheNodes,
1216            Some(options.get_ansible_vars()),
1217        ) {
1218            Ok(()) => println!("All Peer Cache nodes were successfully upgraded"),
1219            Err(_) => {
1220                println!("WARNING: some Peer Cacche nodes may not have been upgraded or restarted");
1221            }
1222        }
1223        match self.ansible_runner.run_playbook(
1224            AnsiblePlaybook::UpgradeNodes,
1225            AnsibleInventoryType::Nodes,
1226            Some(options.get_ansible_vars()),
1227        ) {
1228            Ok(()) => println!("All generic nodes were successfully upgraded"),
1229            Err(_) => {
1230                println!("WARNING: some nodes may not have been upgraded or restarted");
1231            }
1232        }
1233        match self.ansible_runner.run_playbook(
1234            AnsiblePlaybook::UpgradeNodes,
1235            AnsibleInventoryType::SymmetricPrivateNodes,
1236            Some(options.get_ansible_vars()),
1237        ) {
1238            Ok(()) => println!("All private nodes were successfully upgraded"),
1239            Err(_) => {
1240                println!("WARNING: some nodes may not have been upgraded or restarted");
1241            }
1242        }
1243        // Don't use AnsibleInventoryType::iter_node_type() here, because the genesis node should be upgraded last
1244        match self.ansible_runner.run_playbook(
1245            AnsiblePlaybook::UpgradeNodes,
1246            AnsibleInventoryType::Genesis,
1247            Some(options.get_ansible_vars()),
1248        ) {
1249            Ok(()) => println!("The genesis nodes was successfully upgraded"),
1250            Err(_) => {
1251                println!("WARNING: the genesis node may not have been upgraded or restarted");
1252            }
1253        }
1254        Ok(())
1255    }
1256
1257    pub fn upgrade_antctl(
1258        &self,
1259        environment_name: &str,
1260        version: &Version,
1261        node_type: Option<NodeType>,
1262        custom_inventory: Option<Vec<VirtualMachine>>,
1263    ) -> Result<()> {
1264        let mut extra_vars = ExtraVarsDocBuilder::default();
1265        extra_vars.add_variable("version", &version.to_string());
1266
1267        if let Some(node_type) = node_type {
1268            println!("Running the upgrade safenode-manager playbook for {node_type:?} nodes");
1269            self.ansible_runner.run_playbook(
1270                AnsiblePlaybook::UpgradeAntctl,
1271                node_type.to_ansible_inventory_type(),
1272                Some(extra_vars.build()),
1273            )?;
1274            return Ok(());
1275        }
1276
1277        if let Some(custom_inventory) = custom_inventory {
1278            println!("Running the upgrade safenode-manager playbook with a custom inventory");
1279            generate_custom_environment_inventory(
1280                &custom_inventory,
1281                environment_name,
1282                &self.ansible_runner.working_directory_path.join("inventory"),
1283            )?;
1284            self.ansible_runner.run_playbook(
1285                AnsiblePlaybook::UpgradeAntctl,
1286                AnsibleInventoryType::Custom,
1287                Some(extra_vars.build()),
1288            )?;
1289            return Ok(());
1290        }
1291
1292        println!("Running the upgrade safenode-manager playbook for all node types");
1293        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1294            self.ansible_runner.run_playbook(
1295                AnsiblePlaybook::UpgradeAntctl,
1296                node_inv_type,
1297                Some(extra_vars.build()),
1298            )?;
1299        }
1300
1301        Ok(())
1302    }
1303
1304    pub fn print_ansible_run_banner(&self, s: &str) {
1305        let ansible_run_msg = "Ansible Run: ";
1306        let line = "=".repeat(s.len() + ansible_run_msg.len());
1307        println!("{}\n{}{}\n{}", line, ansible_run_msg, s, line);
1308    }
1309}
sn_testnet_deploy/ansible/provisioning.rs

sn_testnet_deploy/ansible/
provisioning.rs
