sn_testnet_deploy/ansible/
provisioning.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use super::{
8    extra_vars::ExtraVarsDocBuilder,
9    inventory::{
10        generate_full_cone_private_node_static_environment_inventory,
11        generate_symmetric_private_node_static_environment_inventory,
12    },
13    AnsibleInventoryType, AnsiblePlaybook, AnsibleRunner,
14};
15use crate::{
16    ansible::inventory::{
17        generate_custom_environment_inventory,
18        generate_full_cone_nat_gateway_static_environment_inventory,
19    },
20    bootstrap::BootstrapOptions,
21    deploy::DeployOptions,
22    error::{Error, Result},
23    funding::FundingOptions,
24    inventory::{DeploymentNodeRegistries, VirtualMachine},
25    print_duration, run_external_command, BinaryOption, CloudProvider, EvmNetwork, LogFormat,
26    NodeType, SshClient, UpgradeOptions,
27};
28use ant_service_management::NodeRegistry;
29use evmlib::common::U256;
30use log::{debug, error, trace};
31use semver::Version;
32use serde::{Deserialize, Serialize};
33use std::{
34    collections::HashMap,
35    net::{IpAddr, SocketAddr},
36    path::PathBuf,
37    time::{Duration, Instant},
38};
39use walkdir::WalkDir;
40
41use crate::ansible::extra_vars;
42
43pub const DEFAULT_BETA_ENCRYPTION_KEY: &str =
44    "49113d2083f57a976076adbe85decb75115820de1e6e74b47e0429338cef124a";
45
46#[derive(Clone, Serialize, Deserialize)]
47pub struct ProvisionOptions {
48    /// The safe version is also in the binary option, but only for an initial deployment.
49    /// For the upscale, it needs to be provided explicitly, because currently it is not
50    /// recorded in the inventory.
51    pub ant_version: Option<String>,
52    pub binary_option: BinaryOption,
53    pub chunk_size: Option<u64>,
54    pub client_env_variables: Option<Vec<(String, String)>>,
55    pub downloaders_count: u16,
56    pub enable_telegraf: bool,
57    pub evm_data_payments_address: Option<String>,
58    pub evm_network: EvmNetwork,
59    pub evm_payment_token_address: Option<String>,
60    pub evm_rpc_url: Option<String>,
61    pub full_cone_private_node_count: u16,
62    pub funding_wallet_secret_key: Option<String>,
63    pub gas_amount: Option<U256>,
64    pub interval: Duration,
65    pub log_format: Option<LogFormat>,
66    pub logstash_details: Option<(String, Vec<SocketAddr>)>,
67    pub max_archived_log_files: u16,
68    pub max_log_files: u16,
69    pub name: String,
70    pub network_id: Option<u8>,
71    pub node_count: u16,
72    pub node_env_variables: Option<Vec<(String, String)>>,
73    pub output_inventory_dir_path: PathBuf,
74    pub peer_cache_node_count: u16,
75    pub public_rpc: bool,
76    pub rewards_address: String,
77    pub symmetric_private_node_count: u16,
78    pub token_amount: Option<U256>,
79    pub uploaders_count: Option<u16>,
80}
81
82/// These are obtained by running the inventory playbook
83#[derive(Clone, Debug)]
84pub struct PrivateNodeProvisionInventory {
85    pub full_cone_nat_gateway_vms: Vec<VirtualMachine>,
86    pub full_cone_private_node_vms: Vec<VirtualMachine>,
87    pub symmetric_nat_gateway_vms: Vec<VirtualMachine>,
88    pub symmetric_private_node_vms: Vec<VirtualMachine>,
89}
90
91impl PrivateNodeProvisionInventory {
92    pub fn new(
93        provisioner: &AnsibleProvisioner,
94        full_cone_private_node_vm_count: Option<u16>,
95        symmetric_private_node_vm_count: Option<u16>,
96    ) -> Result<Self> {
97        // All the environment types set private_node_vm count to >0 if not specified.
98        let should_provision_full_cone_private_nodes = full_cone_private_node_vm_count
99            .map(|count| count > 0)
100            .unwrap_or(true);
101        let should_provision_symmetric_private_nodes = symmetric_private_node_vm_count
102            .map(|count| count > 0)
103            .unwrap_or(true);
104
105        let mut inventory = Self {
106            full_cone_nat_gateway_vms: Default::default(),
107            full_cone_private_node_vms: Default::default(),
108            symmetric_nat_gateway_vms: Default::default(),
109            symmetric_private_node_vms: Default::default(),
110        };
111
112        if should_provision_full_cone_private_nodes {
113            let full_cone_private_node_vms = provisioner
114                .ansible_runner
115                .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)
116                .inspect_err(|err| {
117                    println!("Failed to obtain the inventory of Full Cone private node: {err:?}");
118                })?;
119
120            let full_cone_nat_gateway_inventory = provisioner
121                .ansible_runner
122                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)
123                .inspect_err(|err| {
124                    println!("Failed to get Full Cone NAT Gateway inventory {err:?}");
125                })?;
126
127            if full_cone_nat_gateway_inventory.len() != full_cone_private_node_vms.len() {
128                println!("The number of Full Cone private nodes does not match the number of Full Cone NAT Gateway VMs");
129                return Err(Error::VmCountMismatch(
130                    Some(AnsibleInventoryType::FullConePrivateNodes),
131                    Some(AnsibleInventoryType::FullConeNatGateway),
132                ));
133            }
134
135            inventory.full_cone_private_node_vms = full_cone_private_node_vms;
136            inventory.full_cone_nat_gateway_vms = full_cone_nat_gateway_inventory;
137        }
138
139        if should_provision_symmetric_private_nodes {
140            let symmetric_private_node_vms = provisioner
141                .ansible_runner
142                .get_inventory(AnsibleInventoryType::SymmetricPrivateNodes, true)
143                .inspect_err(|err| {
144                    println!("Failed to obtain the inventory of Symmetric private node: {err:?}");
145                })?;
146
147            let symmetric_nat_gateway_inventory = provisioner
148                .ansible_runner
149                .get_inventory(AnsibleInventoryType::SymmetricNatGateway, true)
150                .inspect_err(|err| {
151                    println!("Failed to get Symmetric NAT Gateway inventory {err:?}");
152                })?;
153
154            if symmetric_nat_gateway_inventory.len() != symmetric_private_node_vms.len() {
155                println!("The number of Symmetric private nodes does not match the number of Symmetric NAT Gateway VMs");
156                return Err(Error::VmCountMismatch(
157                    Some(AnsibleInventoryType::SymmetricPrivateNodes),
158                    Some(AnsibleInventoryType::SymmetricNatGateway),
159                ));
160            }
161
162            inventory.symmetric_private_node_vms = symmetric_private_node_vms;
163            inventory.symmetric_nat_gateway_vms = symmetric_nat_gateway_inventory;
164        }
165
166        Ok(inventory)
167    }
168
169    pub fn should_provision_full_cone_private_nodes(&self) -> bool {
170        !self.full_cone_private_node_vms.is_empty()
171    }
172
173    pub fn should_provision_symmetric_private_nodes(&self) -> bool {
174        !self.symmetric_private_node_vms.is_empty()
175    }
176
177    pub fn symmetric_private_node_and_gateway_map(
178        &self,
179    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
180        Self::match_private_node_vm_and_gateway_vm(
181            &self.symmetric_private_node_vms,
182            &self.symmetric_nat_gateway_vms,
183        )
184    }
185
186    pub fn full_cone_private_node_and_gateway_map(
187        &self,
188    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
189        Self::match_private_node_vm_and_gateway_vm(
190            &self.full_cone_private_node_vms,
191            &self.full_cone_nat_gateway_vms,
192        )
193    }
194
195    pub fn match_private_node_vm_and_gateway_vm(
196        private_node_vms: &[VirtualMachine],
197        nat_gateway_vms: &[VirtualMachine],
198    ) -> Result<HashMap<VirtualMachine, VirtualMachine>> {
199        if private_node_vms.len() != nat_gateway_vms.len() {
200            println!(
201            "The number of private node VMs ({}) does not match the number of NAT Gateway VMs ({})",
202            private_node_vms.len(),
203            nat_gateway_vms.len()
204        );
205            error!("The number of private node VMs does not match the number of NAT Gateway VMs: Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}");
206            return Err(Error::VmCountMismatch(None, None));
207        }
208
209        let mut map = HashMap::new();
210        for private_vm in private_node_vms {
211            let nat_gateway = nat_gateway_vms
212                .iter()
213                .find(|vm| {
214                    let private_node_name = private_vm.name.split('-').last().unwrap();
215                    let nat_gateway_name = vm.name.split('-').last().unwrap();
216                    private_node_name == nat_gateway_name
217                })
218                .ok_or_else(|| {
219                    println!(
220                        "Failed to find a matching NAT Gateway for private node: {}",
221                        private_vm.name
222                    );
223                    error!("Failed to find a matching NAT Gateway for private node: {}. Private VMs: {private_node_vms:?} Nat gateway VMs: {nat_gateway_vms:?}", private_vm.name);
224                    Error::VmCountMismatch(None, None)
225                })?;
226
227            let _ = map.insert(private_vm.clone(), nat_gateway.clone());
228        }
229
230        Ok(map)
231    }
232}
233
234impl From<BootstrapOptions> for ProvisionOptions {
235    fn from(bootstrap_options: BootstrapOptions) -> Self {
236        ProvisionOptions {
237            ant_version: None,
238            binary_option: bootstrap_options.binary_option,
239            chunk_size: bootstrap_options.chunk_size,
240            downloaders_count: 0,
241            enable_telegraf: true,
242            node_env_variables: bootstrap_options.node_env_variables,
243            evm_data_payments_address: bootstrap_options.evm_data_payments_address,
244            evm_network: bootstrap_options.evm_network,
245            evm_payment_token_address: bootstrap_options.evm_payment_token_address,
246            evm_rpc_url: bootstrap_options.evm_rpc_url,
247            full_cone_private_node_count: bootstrap_options.full_cone_private_node_count,
248            funding_wallet_secret_key: None,
249            gas_amount: None,
250            interval: bootstrap_options.interval,
251            log_format: bootstrap_options.log_format,
252            logstash_details: None,
253            max_archived_log_files: bootstrap_options.max_archived_log_files,
254            max_log_files: bootstrap_options.max_log_files,
255            name: bootstrap_options.name,
256            network_id: bootstrap_options.network_id,
257            node_count: bootstrap_options.node_count,
258            output_inventory_dir_path: bootstrap_options.output_inventory_dir_path,
259            peer_cache_node_count: 0,
260            symmetric_private_node_count: bootstrap_options.symmetric_private_node_count,
261            public_rpc: false,
262            rewards_address: bootstrap_options.rewards_address,
263            token_amount: None,
264            uploaders_count: None,
265            client_env_variables: None,
266        }
267    }
268}
269
270impl From<DeployOptions> for ProvisionOptions {
271    fn from(deploy_options: DeployOptions) -> Self {
272        ProvisionOptions {
273            ant_version: None,
274            binary_option: deploy_options.binary_option,
275            chunk_size: deploy_options.chunk_size,
276            downloaders_count: deploy_options.downloaders_count,
277            enable_telegraf: deploy_options.enable_telegraf,
278            node_env_variables: deploy_options.node_env_variables,
279            evm_data_payments_address: deploy_options.evm_data_payments_address,
280            evm_network: deploy_options.evm_network,
281            evm_payment_token_address: deploy_options.evm_payment_token_address,
282            evm_rpc_url: deploy_options.evm_rpc_url,
283            full_cone_private_node_count: deploy_options.full_cone_private_node_count,
284            funding_wallet_secret_key: deploy_options.funding_wallet_secret_key,
285            gas_amount: deploy_options.initial_gas,
286            interval: deploy_options.interval,
287            log_format: deploy_options.log_format,
288            logstash_details: deploy_options.logstash_details,
289            max_archived_log_files: deploy_options.max_archived_log_files,
290            max_log_files: deploy_options.max_log_files,
291            name: deploy_options.name,
292            network_id: deploy_options.network_id,
293            node_count: deploy_options.node_count,
294            output_inventory_dir_path: deploy_options.output_inventory_dir_path,
295            peer_cache_node_count: deploy_options.peer_cache_node_count,
296            symmetric_private_node_count: deploy_options.symmetric_private_node_count,
297            public_rpc: deploy_options.public_rpc,
298            rewards_address: deploy_options.rewards_address,
299            token_amount: deploy_options.initial_tokens,
300            uploaders_count: Some(deploy_options.uploaders_count),
301            client_env_variables: deploy_options.client_env_variables,
302        }
303    }
304}
305
306#[derive(Clone)]
307pub struct AnsibleProvisioner {
308    pub ansible_runner: AnsibleRunner,
309    pub cloud_provider: CloudProvider,
310    pub ssh_client: SshClient,
311}
312
313impl AnsibleProvisioner {
314    pub fn new(
315        ansible_runner: AnsibleRunner,
316        cloud_provider: CloudProvider,
317        ssh_client: SshClient,
318    ) -> Self {
319        Self {
320            ansible_runner,
321            cloud_provider,
322            ssh_client,
323        }
324    }
325
326    pub fn build_safe_network_binaries(&self, options: &ProvisionOptions) -> Result<()> {
327        let start = Instant::now();
328        println!("Obtaining IP address for build VM...");
329        let build_inventory = self
330            .ansible_runner
331            .get_inventory(AnsibleInventoryType::Build, true)?;
332        let build_ip = build_inventory[0].public_ip_addr;
333        self.ssh_client
334            .wait_for_ssh_availability(&build_ip, &self.cloud_provider.get_ssh_user())?;
335
336        println!("Running ansible against build VM...");
337        let extra_vars = extra_vars::build_binaries_extra_vars_doc(options)?;
338        self.ansible_runner.run_playbook(
339            AnsiblePlaybook::Build,
340            AnsibleInventoryType::Build,
341            Some(extra_vars),
342        )?;
343        print_duration(start.elapsed());
344        Ok(())
345    }
346
347    pub fn cleanup_node_logs(&self, setup_cron: bool) -> Result<()> {
348        for node_inv_type in AnsibleInventoryType::iter_node_type() {
349            self.ansible_runner.run_playbook(
350                AnsiblePlaybook::CleanupLogs,
351                node_inv_type,
352                Some(format!("{{ \"setup_cron\": \"{setup_cron}\" }}")),
353            )?;
354        }
355
356        Ok(())
357    }
358
359    pub fn copy_logs(&self, name: &str, resources_only: bool) -> Result<()> {
360        for node_inv_type in AnsibleInventoryType::iter_node_type() {
361            self.ansible_runner.run_playbook(
362                AnsiblePlaybook::CopyLogs,
363                node_inv_type,
364                Some(format!(
365                    "{{ \"env_name\": \"{name}\", \"resources_only\" : \"{resources_only}\" }}"
366                )),
367            )?;
368        }
369        Ok(())
370    }
371
372    pub fn get_all_node_inventory(&self) -> Result<Vec<VirtualMachine>> {
373        let mut all_node_inventory = Vec::new();
374        for node_inv_type in AnsibleInventoryType::iter_node_type() {
375            all_node_inventory.extend(self.ansible_runner.get_inventory(node_inv_type, false)?);
376        }
377
378        Ok(all_node_inventory)
379    }
380
381    pub fn get_symmetric_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
382        self.ansible_runner
383            .get_inventory(AnsibleInventoryType::SymmetricNatGateway, false)
384    }
385
386    pub fn get_full_cone_nat_gateway_inventory(&self) -> Result<Vec<VirtualMachine>> {
387        self.ansible_runner
388            .get_inventory(AnsibleInventoryType::FullConeNatGateway, false)
389    }
390
391    pub fn get_uploader_inventory(&self) -> Result<Vec<VirtualMachine>> {
392        self.ansible_runner
393            .get_inventory(AnsibleInventoryType::Uploaders, false)
394    }
395
396    pub fn get_node_registries(
397        &self,
398        inventory_type: &AnsibleInventoryType,
399    ) -> Result<DeploymentNodeRegistries> {
400        debug!("Fetching node manager inventory for {inventory_type:?}");
401        let temp_dir_path = tempfile::tempdir()?.into_path();
402        let temp_dir_json = serde_json::to_string(&temp_dir_path)?;
403
404        self.ansible_runner.run_playbook(
405            AnsiblePlaybook::AntCtlInventory,
406            *inventory_type,
407            Some(format!("{{ \"dest\": {temp_dir_json} }}")),
408        )?;
409
410        let node_registry_paths = WalkDir::new(temp_dir_path)
411            .into_iter()
412            .flatten()
413            .filter_map(|entry| {
414                if entry.file_type().is_file()
415                    && entry.path().extension().is_some_and(|ext| ext == "json")
416                {
417                    // tempdir/<testnet_name>-node/var/safenode-manager/node_registry.json
418                    let mut vm_name = entry.path().to_path_buf();
419                    trace!("Found file with json extension: {vm_name:?}");
420                    vm_name.pop();
421                    vm_name.pop();
422                    vm_name.pop();
423                    // Extract the <testnet_name>-node string
424                    trace!("Extracting the vm name from the path");
425                    let vm_name = vm_name.file_name()?.to_str()?;
426                    trace!("Extracted vm name from path: {vm_name}");
427                    Some((vm_name.to_string(), entry.path().to_path_buf()))
428                } else {
429                    None
430                }
431            })
432            .collect::<Vec<(String, PathBuf)>>();
433
434        let mut node_registries = Vec::new();
435        let mut failed_vms = Vec::new();
436        for (vm_name, file_path) in node_registry_paths {
437            match NodeRegistry::load(&file_path) {
438                Ok(node_registry) => node_registries.push((vm_name.clone(), node_registry)),
439                Err(_) => failed_vms.push(vm_name.clone()),
440            }
441        }
442
443        let deployment_registries = DeploymentNodeRegistries {
444            inventory_type: *inventory_type,
445            retrieved_registries: node_registries,
446            failed_vms,
447        };
448        Ok(deployment_registries)
449    }
450
451    pub fn provision_evm_nodes(&self, options: &ProvisionOptions) -> Result<()> {
452        let start = Instant::now();
453        println!("Obtaining IP address for EVM nodes...");
454        let evm_node_inventory = self
455            .ansible_runner
456            .get_inventory(AnsibleInventoryType::EvmNodes, true)?;
457        let evm_node_ip = evm_node_inventory[0].public_ip_addr;
458        self.ssh_client
459            .wait_for_ssh_availability(&evm_node_ip, &self.cloud_provider.get_ssh_user())?;
460
461        println!("Running ansible against EVM nodes...");
462        self.ansible_runner.run_playbook(
463            AnsiblePlaybook::EvmNodes,
464            AnsibleInventoryType::EvmNodes,
465            Some(extra_vars::build_evm_nodes_extra_vars_doc(
466                &options.name,
467                &self.cloud_provider,
468                &options.binary_option,
469            )),
470        )?;
471        print_duration(start.elapsed());
472        Ok(())
473    }
474
475    pub fn provision_genesis_node(&self, options: &ProvisionOptions) -> Result<()> {
476        let start = Instant::now();
477        let genesis_inventory = self
478            .ansible_runner
479            .get_inventory(AnsibleInventoryType::Genesis, true)?;
480        let genesis_ip = genesis_inventory[0].public_ip_addr;
481        self.ssh_client
482            .wait_for_ssh_availability(&genesis_ip, &self.cloud_provider.get_ssh_user())?;
483        self.ansible_runner.run_playbook(
484            AnsiblePlaybook::Genesis,
485            AnsibleInventoryType::Genesis,
486            Some(extra_vars::build_node_extra_vars_doc(
487                &self.cloud_provider.to_string(),
488                options,
489                NodeType::Genesis,
490                None,
491                None,
492                1,
493                options.evm_network.clone(),
494                false,
495            )?),
496        )?;
497
498        print_duration(start.elapsed());
499
500        Ok(())
501    }
502
503    pub fn provision_full_cone(
504        &self,
505        options: &ProvisionOptions,
506        initial_contact_peer: Option<String>,
507        initial_network_contacts_url: Option<String>,
508        private_node_inventory: PrivateNodeProvisionInventory,
509        new_full_cone_nat_gateway_new_vms_for_upscale: Option<Vec<VirtualMachine>>,
510    ) -> Result<()> {
511        // Step 1 of Full Cone NAT Gateway
512        let start = Instant::now();
513        self.print_ansible_run_banner("Provision Full Cone NAT Gateway - Step 1");
514
515        for vm in new_full_cone_nat_gateway_new_vms_for_upscale
516            .as_ref()
517            .unwrap_or(&private_node_inventory.full_cone_nat_gateway_vms)
518            .iter()
519        {
520            println!(
521                "Checking SSH availability for Full Cone NAT Gateway: {}",
522                vm.public_ip_addr
523            );
524            self.ssh_client
525                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
526                .map_err(|e| {
527                    println!(
528                        "Failed to establish SSH connection to Full Cone NAT Gateway: {}",
529                        e
530                    );
531                    e
532                })?;
533        }
534
535        let mut modified_private_node_inventory = private_node_inventory.clone();
536
537        // If we are upscaling, then we cannot access the gateway VMs which are already deployed.
538        if let Some(new_full_cone_nat_gateway_new_vms_for_upscale) =
539            &new_full_cone_nat_gateway_new_vms_for_upscale
540        {
541            debug!("Removing existing full cone NAT Gateway and private node VMs from the inventory. Old inventory: {modified_private_node_inventory:?}");
542            let mut names_to_keep = Vec::new();
543
544            for vm in new_full_cone_nat_gateway_new_vms_for_upscale.iter() {
545                let nat_gateway_name = vm.name.split('-').last().unwrap();
546                names_to_keep.push(nat_gateway_name);
547            }
548
549            modified_private_node_inventory
550                .full_cone_nat_gateway_vms
551                .retain(|vm| {
552                    let nat_gateway_name = vm.name.split('-').last().unwrap();
553                    names_to_keep.contains(&nat_gateway_name)
554                });
555            modified_private_node_inventory
556                .full_cone_private_node_vms
557                .retain(|vm| {
558                    let nat_gateway_name = vm.name.split('-').last().unwrap();
559                    names_to_keep.contains(&nat_gateway_name)
560                });
561            debug!("New inventory after removing existing full cone NAT Gateway and private node VMs: {modified_private_node_inventory:?}");
562        }
563
564        if modified_private_node_inventory
565            .full_cone_nat_gateway_vms
566            .is_empty()
567        {
568            error!("There are no full cone NAT Gateway VMs available to upscale");
569            return Ok(());
570        }
571
572        let private_node_ip_map = modified_private_node_inventory
573            .full_cone_private_node_and_gateway_map()?
574            .into_iter()
575            .map(|(k, v)| {
576                let gateway_name = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
577                    debug!("Upscaling, using public IP address for gateway name");
578                    v.public_ip_addr.to_string()
579                } else {
580                    v.name.clone()
581                };
582                (gateway_name, k.private_ip_addr)
583            })
584            .collect::<HashMap<String, IpAddr>>();
585
586        if private_node_ip_map.is_empty() {
587            println!("There are no full cone private node VM available to be routed through the full cone NAT Gateway");
588            return Err(Error::EmptyInventory(
589                AnsibleInventoryType::FullConePrivateNodes,
590            ));
591        }
592
593        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
594            &options.name,
595            private_node_ip_map.clone(),
596            "step1",
597        );
598        debug!("Provisioning Full Cone NAT Gateway - Step 1 with vars: {vars}");
599        let gateway_inventory = if new_full_cone_nat_gateway_new_vms_for_upscale.is_some() {
600            debug!("Upscaling, using static inventory for full cone nat gateway.");
601            generate_full_cone_nat_gateway_static_environment_inventory(
602                &modified_private_node_inventory.full_cone_nat_gateway_vms,
603                &options.name,
604                &options.output_inventory_dir_path,
605            )?;
606
607            AnsibleInventoryType::FullConeNatGatewayStatic
608        } else {
609            AnsibleInventoryType::FullConeNatGateway
610        };
611        self.ansible_runner.run_playbook(
612            AnsiblePlaybook::FullConeNatGateway,
613            gateway_inventory,
614            Some(vars),
615        )?;
616
617        // setup private node config
618        self.print_ansible_run_banner("Provisioning Full Cone Private Node Config");
619
620        generate_full_cone_private_node_static_environment_inventory(
621            &options.name,
622            &options.output_inventory_dir_path,
623            &private_node_inventory.full_cone_private_node_vms,
624            &private_node_inventory.full_cone_nat_gateway_vms,
625            &self.ssh_client.private_key_path,
626        )
627        .inspect_err(|err| {
628            error!("Failed to generate full cone private node static inv with err: {err:?}")
629        })?;
630
631        // For a new deployment, it's quite probable that SSH is available, because this part occurs
632        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
633        // check that SSH is available before proceeding.
634        println!("Obtaining IP addresses for nodes...");
635        let inventory = self
636            .ansible_runner
637            .get_inventory(AnsibleInventoryType::FullConePrivateNodes, true)?;
638
639        println!("Waiting for SSH availability on Symmetric Private nodes...");
640        for vm in inventory.iter() {
641            println!(
642                "Checking SSH availability for {}: {}",
643                vm.name, vm.public_ip_addr
644            );
645            self.ssh_client
646                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
647                .map_err(|e| {
648                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
649                    e
650                })?;
651        }
652
653        println!("SSH is available on all nodes. Proceeding with provisioning...");
654
655        self.ansible_runner.run_playbook(
656            AnsiblePlaybook::PrivateNodeConfig,
657            AnsibleInventoryType::FullConePrivateNodes,
658            Some(
659                extra_vars::build_full_cone_private_node_config_extra_vars_docs(
660                    &private_node_inventory,
661                )?,
662            ),
663        )?;
664
665        // Step 2 of Full Cone NAT Gateway
666
667        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
668            &options.name,
669            private_node_ip_map,
670            "step2",
671        );
672
673        self.print_ansible_run_banner("Provisioning Full Cone NAT Gateway - Step 2");
674        debug!("Provisioning Full Cone NAT Gateway - Step 2 with vars: {vars}");
675        self.ansible_runner.run_playbook(
676            AnsiblePlaybook::FullConeNatGateway,
677            gateway_inventory,
678            Some(vars),
679        )?;
680
681        // provision the nodes
682
683        let home_dir = std::env::var("HOME").inspect_err(|err| {
684            println!("Failed to get home directory with error: {err:?}",);
685        })?;
686        let known_hosts_path = format!("{}/.ssh/known_hosts", home_dir);
687        debug!("Cleaning up known hosts file at {known_hosts_path} ");
688        run_external_command(
689            PathBuf::from("rm"),
690            std::env::current_dir()?,
691            vec![known_hosts_path],
692            false,
693            false,
694        )?;
695
696        self.print_ansible_run_banner("Provision Full Cone Private Nodes");
697
698        self.ssh_client.set_full_cone_nat_routed_vms(
699            &private_node_inventory.full_cone_private_node_vms,
700            &private_node_inventory.full_cone_nat_gateway_vms,
701        )?;
702
703        self.provision_nodes(
704            options,
705            initial_contact_peer,
706            initial_network_contacts_url,
707            NodeType::FullConePrivateNode,
708        )?;
709
710        print_duration(start.elapsed());
711        Ok(())
712    }
713    pub fn provision_symmetric_nat_gateway(
714        &self,
715        options: &ProvisionOptions,
716        private_node_inventory: &PrivateNodeProvisionInventory,
717    ) -> Result<()> {
718        let start = Instant::now();
719        for vm in &private_node_inventory.symmetric_nat_gateway_vms {
720            println!(
721                "Checking SSH availability for Symmetric NAT Gateway: {}",
722                vm.public_ip_addr
723            );
724            self.ssh_client
725                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
726                .map_err(|e| {
727                    println!(
728                        "Failed to establish SSH connection to Symmetric NAT Gateway: {}",
729                        e
730                    );
731                    e
732                })?;
733        }
734
735        let private_node_ip_map = private_node_inventory
736            .symmetric_private_node_and_gateway_map()?
737            .into_iter()
738            .map(|(k, v)| (v.name.clone(), k.private_ip_addr))
739            .collect::<HashMap<String, IpAddr>>();
740
741        if private_node_ip_map.is_empty() {
742            println!("There are no Symmetric private node VM available to be routed through the Symmetric NAT Gateway");
743            return Err(Error::EmptyInventory(
744                AnsibleInventoryType::SymmetricPrivateNodes,
745            ));
746        }
747
748        let vars = extra_vars::build_nat_gateway_extra_vars_doc(
749            &options.name,
750            private_node_ip_map,
751            "symmetric",
752        );
753        debug!("Provisioning Symmetric NAT Gateway with vars: {vars}");
754        self.ansible_runner.run_playbook(
755            AnsiblePlaybook::SymmetricNatGateway,
756            AnsibleInventoryType::SymmetricNatGateway,
757            Some(vars),
758        )?;
759
760        print_duration(start.elapsed());
761        Ok(())
762    }
763
764    pub fn provision_nodes(
765        &self,
766        options: &ProvisionOptions,
767        initial_contact_peer: Option<String>,
768        initial_network_contacts_url: Option<String>,
769        node_type: NodeType,
770    ) -> Result<()> {
771        let start = Instant::now();
772        let mut relay = false;
773        let (inventory_type, node_count) = match &node_type {
774            NodeType::FullConePrivateNode => {
775                relay = true;
776                (
777                    node_type.to_ansible_inventory_type(),
778                    options.full_cone_private_node_count,
779                )
780            }
781            // use provision_genesis_node fn
782            NodeType::Generic => (node_type.to_ansible_inventory_type(), options.node_count),
783            NodeType::Genesis => return Err(Error::InvalidNodeType(node_type)),
784            NodeType::PeerCache => (
785                node_type.to_ansible_inventory_type(),
786                options.peer_cache_node_count,
787            ),
788            NodeType::SymmetricPrivateNode => {
789                relay = true;
790                (
791                    node_type.to_ansible_inventory_type(),
792                    options.symmetric_private_node_count,
793                )
794            }
795        };
796
797        // For a new deployment, it's quite probable that SSH is available, because this part occurs
798        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
799        // check that SSH is available before proceeding.
800        println!("Obtaining IP addresses for nodes...");
801        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
802
803        println!("Waiting for SSH availability on {node_type:?} nodes...");
804        for vm in inventory.iter() {
805            println!(
806                "Checking SSH availability for {}: {}",
807                vm.name, vm.public_ip_addr
808            );
809            self.ssh_client
810                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
811                .map_err(|e| {
812                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
813                    e
814                })?;
815        }
816
817        println!("SSH is available on all nodes. Proceeding with provisioning...");
818
819        let playbook = match node_type {
820            NodeType::Generic => AnsiblePlaybook::Nodes,
821            NodeType::PeerCache => AnsiblePlaybook::PeerCacheNodes,
822            NodeType::FullConePrivateNode => AnsiblePlaybook::Nodes,
823            NodeType::SymmetricPrivateNode => AnsiblePlaybook::Nodes,
824            _ => return Err(Error::InvalidNodeType(node_type.clone())),
825        };
826        self.ansible_runner.run_playbook(
827            playbook,
828            inventory_type,
829            Some(extra_vars::build_node_extra_vars_doc(
830                &self.cloud_provider.to_string(),
831                options,
832                node_type.clone(),
833                initial_contact_peer,
834                initial_network_contacts_url,
835                node_count,
836                options.evm_network.clone(),
837                relay,
838            )?),
839        )?;
840
841        print_duration(start.elapsed());
842        Ok(())
843    }
844
845    pub fn provision_symmetric_private_nodes(
846        &self,
847        options: &mut ProvisionOptions,
848        initial_contact_peer: Option<String>,
849        initial_network_contacts_url: Option<String>,
850        private_node_inventory: &PrivateNodeProvisionInventory,
851    ) -> Result<()> {
852        let start = Instant::now();
853        self.print_ansible_run_banner("Provision Symmetric Private Node Config");
854
855        generate_symmetric_private_node_static_environment_inventory(
856            &options.name,
857            &options.output_inventory_dir_path,
858            &private_node_inventory.symmetric_private_node_vms,
859            &private_node_inventory.symmetric_nat_gateway_vms,
860            &self.ssh_client.private_key_path,
861        )
862        .inspect_err(|err| {
863            error!("Failed to generate symmetric private node static inv with err: {err:?}")
864        })?;
865
866        self.ssh_client.set_symmetric_nat_routed_vms(
867            &private_node_inventory.symmetric_private_node_vms,
868            &private_node_inventory.symmetric_nat_gateway_vms,
869        )?;
870
871        let inventory_type = AnsibleInventoryType::SymmetricPrivateNodes;
872
873        // For a new deployment, it's quite probable that SSH is available, because this part occurs
874        // after the genesis node has been provisioned. However, for a bootstrap deploy, we need to
875        // check that SSH is available before proceeding.
876        println!("Obtaining IP addresses for nodes...");
877        let inventory = self.ansible_runner.get_inventory(inventory_type, true)?;
878
879        println!("Waiting for SSH availability on Symmetric Private nodes...");
880        for vm in inventory.iter() {
881            println!(
882                "Checking SSH availability for {}: {}",
883                vm.name, vm.public_ip_addr
884            );
885            self.ssh_client
886                .wait_for_ssh_availability(&vm.public_ip_addr, &self.cloud_provider.get_ssh_user())
887                .map_err(|e| {
888                    println!("Failed to establish SSH connection to {}: {}", vm.name, e);
889                    e
890                })?;
891        }
892
893        println!("SSH is available on all nodes. Proceeding with provisioning...");
894
895        self.ansible_runner.run_playbook(
896            AnsiblePlaybook::PrivateNodeConfig,
897            inventory_type,
898            Some(
899                extra_vars::build_symmetric_private_node_config_extra_vars_doc(
900                    private_node_inventory,
901                )?,
902            ),
903        )?;
904
905        println!("Provisioned Symmetric Private Node Config");
906        print_duration(start.elapsed());
907
908        self.provision_nodes(
909            options,
910            initial_contact_peer,
911            initial_network_contacts_url,
912            NodeType::SymmetricPrivateNode,
913        )?;
914
915        Ok(())
916    }
917
918    pub async fn provision_uploaders(
919        &self,
920        options: &ProvisionOptions,
921        genesis_multiaddr: Option<String>,
922        genesis_network_contacts_url: Option<String>,
923    ) -> Result<()> {
924        let start = Instant::now();
925
926        let sk_map = self
927            .deposit_funds_to_uploaders(&FundingOptions {
928                evm_data_payments_address: options.evm_data_payments_address.clone(),
929                evm_network: options.evm_network.clone(),
930                evm_payment_token_address: options.evm_payment_token_address.clone(),
931                evm_rpc_url: options.evm_rpc_url.clone(),
932                funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
933                gas_amount: options.gas_amount,
934                token_amount: options.token_amount,
935                uploaders_count: options.uploaders_count,
936            })
937            .await?;
938
939        println!("Running ansible against uploader machine to start the uploader script.");
940        debug!("Running ansible against uploader machine to start the uploader script.");
941
942        self.ansible_runner.run_playbook(
943            AnsiblePlaybook::Uploaders,
944            AnsibleInventoryType::Uploaders,
945            Some(extra_vars::build_uploaders_extra_vars_doc(
946                &self.cloud_provider.to_string(),
947                options,
948                genesis_multiaddr,
949                genesis_network_contacts_url,
950                &sk_map,
951            )?),
952        )?;
953        print_duration(start.elapsed());
954        Ok(())
955    }
956
957    pub fn start_nodes(
958        &self,
959        environment_name: &str,
960        interval: Duration,
961        node_type: Option<NodeType>,
962        custom_inventory: Option<Vec<VirtualMachine>>,
963    ) -> Result<()> {
964        let mut extra_vars = ExtraVarsDocBuilder::default();
965        extra_vars.add_variable("interval", &interval.as_millis().to_string());
966
967        if let Some(node_type) = node_type {
968            println!("Running the start nodes playbook for {node_type:?} nodes");
969            self.ansible_runner.run_playbook(
970                AnsiblePlaybook::StartNodes,
971                node_type.to_ansible_inventory_type(),
972                Some(extra_vars.build()),
973            )?;
974            return Ok(());
975        }
976
977        if let Some(custom_inventory) = custom_inventory {
978            println!("Running the start nodes playbook with a custom inventory");
979            generate_custom_environment_inventory(
980                &custom_inventory,
981                environment_name,
982                &self.ansible_runner.working_directory_path.join("inventory"),
983            )?;
984            self.ansible_runner.run_playbook(
985                AnsiblePlaybook::StartNodes,
986                AnsibleInventoryType::Custom,
987                Some(extra_vars.build()),
988            )?;
989            return Ok(());
990        }
991
992        println!("Running the start nodes playbook for all node types");
993        for node_inv_type in AnsibleInventoryType::iter_node_type() {
994            self.ansible_runner.run_playbook(
995                AnsiblePlaybook::StartNodes,
996                node_inv_type,
997                Some(extra_vars.build()),
998            )?;
999        }
1000        Ok(())
1001    }
1002
1003    pub fn status(&self) -> Result<()> {
1004        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1005            self.ansible_runner
1006                .run_playbook(AnsiblePlaybook::Status, node_inv_type, None)?;
1007        }
1008        Ok(())
1009    }
1010
1011    pub fn start_telegraf(
1012        &self,
1013        environment_name: &str,
1014        node_type: Option<NodeType>,
1015        custom_inventory: Option<Vec<VirtualMachine>>,
1016    ) -> Result<()> {
1017        if let Some(node_type) = node_type {
1018            println!("Running the start telegraf playbook for {node_type:?} nodes");
1019            self.ansible_runner.run_playbook(
1020                AnsiblePlaybook::StartTelegraf,
1021                node_type.to_ansible_inventory_type(),
1022                None,
1023            )?;
1024            return Ok(());
1025        }
1026
1027        if let Some(custom_inventory) = custom_inventory {
1028            println!("Running the start telegraf playbook with a custom inventory");
1029            generate_custom_environment_inventory(
1030                &custom_inventory,
1031                environment_name,
1032                &self.ansible_runner.working_directory_path.join("inventory"),
1033            )?;
1034            self.ansible_runner.run_playbook(
1035                AnsiblePlaybook::StartTelegraf,
1036                AnsibleInventoryType::Custom,
1037                None,
1038            )?;
1039            return Ok(());
1040        }
1041
1042        println!("Running the start telegraf playbook for all node types");
1043        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1044            self.ansible_runner.run_playbook(
1045                AnsiblePlaybook::StartTelegraf,
1046                node_inv_type,
1047                None,
1048            )?;
1049        }
1050
1051        Ok(())
1052    }
1053
1054    pub fn stop_nodes(
1055        &self,
1056        environment_name: &str,
1057        interval: Duration,
1058        node_type: Option<NodeType>,
1059        custom_inventory: Option<Vec<VirtualMachine>>,
1060        delay: Option<u64>,
1061        service_names: Option<Vec<String>>,
1062    ) -> Result<()> {
1063        let mut extra_vars = ExtraVarsDocBuilder::default();
1064        extra_vars.add_variable("interval", &interval.as_millis().to_string());
1065        if let Some(delay) = delay {
1066            extra_vars.add_variable("delay", &delay.to_string());
1067        }
1068        if let Some(service_names) = service_names {
1069            extra_vars.add_list_variable("service_names", service_names);
1070        }
1071        let extra_vars = extra_vars.build();
1072
1073        if let Some(node_type) = node_type {
1074            println!("Running the stop nodes playbook for {node_type:?} nodes");
1075            self.ansible_runner.run_playbook(
1076                AnsiblePlaybook::StopNodes,
1077                node_type.to_ansible_inventory_type(),
1078                Some(extra_vars),
1079            )?;
1080            return Ok(());
1081        }
1082
1083        if let Some(custom_inventory) = custom_inventory {
1084            println!("Running the stop nodes playbook with a custom inventory");
1085            generate_custom_environment_inventory(
1086                &custom_inventory,
1087                environment_name,
1088                &self.ansible_runner.working_directory_path.join("inventory"),
1089            )?;
1090            self.ansible_runner.run_playbook(
1091                AnsiblePlaybook::StopNodes,
1092                AnsibleInventoryType::Custom,
1093                Some(extra_vars),
1094            )?;
1095            return Ok(());
1096        }
1097
1098        println!("Running the stop nodes playbook for all node types");
1099        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1100            self.ansible_runner.run_playbook(
1101                AnsiblePlaybook::StopNodes,
1102                node_inv_type,
1103                Some(extra_vars.clone()),
1104            )?;
1105        }
1106
1107        Ok(())
1108    }
1109
1110    pub fn stop_telegraf(
1111        &self,
1112        environment_name: &str,
1113        node_type: Option<NodeType>,
1114        custom_inventory: Option<Vec<VirtualMachine>>,
1115    ) -> Result<()> {
1116        if let Some(node_type) = node_type {
1117            println!("Running the stop telegraf playbook for {node_type:?} nodes");
1118            self.ansible_runner.run_playbook(
1119                AnsiblePlaybook::StopTelegraf,
1120                node_type.to_ansible_inventory_type(),
1121                None,
1122            )?;
1123            return Ok(());
1124        }
1125
1126        if let Some(custom_inventory) = custom_inventory {
1127            println!("Running the stop telegraf playbook with a custom inventory");
1128            generate_custom_environment_inventory(
1129                &custom_inventory,
1130                environment_name,
1131                &self.ansible_runner.working_directory_path.join("inventory"),
1132            )?;
1133            self.ansible_runner.run_playbook(
1134                AnsiblePlaybook::StopTelegraf,
1135                AnsibleInventoryType::Custom,
1136                None,
1137            )?;
1138            return Ok(());
1139        }
1140
1141        println!("Running the stop telegraf playbook for all node types");
1142        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1143            self.ansible_runner
1144                .run_playbook(AnsiblePlaybook::StopTelegraf, node_inv_type, None)?;
1145        }
1146
1147        Ok(())
1148    }
1149
1150    pub fn upgrade_node_telegraf(&self, name: &str) -> Result<()> {
1151        self.ansible_runner.run_playbook(
1152            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1153            AnsibleInventoryType::PeerCacheNodes,
1154            Some(extra_vars::build_node_telegraf_upgrade(
1155                name,
1156                &NodeType::PeerCache,
1157            )?),
1158        )?;
1159        self.ansible_runner.run_playbook(
1160            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1161            AnsibleInventoryType::Nodes,
1162            Some(extra_vars::build_node_telegraf_upgrade(
1163                name,
1164                &NodeType::Generic,
1165            )?),
1166        )?;
1167
1168        self.ansible_runner.run_playbook(
1169            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1170            AnsibleInventoryType::SymmetricPrivateNodes,
1171            Some(extra_vars::build_node_telegraf_upgrade(
1172                name,
1173                &NodeType::SymmetricPrivateNode,
1174            )?),
1175        )?;
1176
1177        self.ansible_runner.run_playbook(
1178            AnsiblePlaybook::UpgradeNodeTelegrafConfig,
1179            AnsibleInventoryType::FullConePrivateNodes,
1180            Some(extra_vars::build_node_telegraf_upgrade(
1181                name,
1182                &NodeType::FullConePrivateNode,
1183            )?),
1184        )?;
1185        Ok(())
1186    }
1187
1188    pub fn upgrade_uploader_telegraf(&self, name: &str) -> Result<()> {
1189        self.ansible_runner.run_playbook(
1190            AnsiblePlaybook::UpgradeUploaderTelegrafConfig,
1191            AnsibleInventoryType::Uploaders,
1192            Some(extra_vars::build_uploader_telegraf_upgrade(name)?),
1193        )?;
1194        Ok(())
1195    }
1196
1197    pub fn upgrade_nodes(&self, options: &UpgradeOptions) -> Result<()> {
1198        if let Some(custom_inventory) = &options.custom_inventory {
1199            println!("Running the UpgradeNodes with a custom inventory");
1200            generate_custom_environment_inventory(
1201                custom_inventory,
1202                &options.name,
1203                &self.ansible_runner.working_directory_path.join("inventory"),
1204            )?;
1205            match self.ansible_runner.run_playbook(
1206                AnsiblePlaybook::UpgradeNodes,
1207                AnsibleInventoryType::Custom,
1208                Some(options.get_ansible_vars()),
1209            ) {
1210                Ok(()) => println!("All nodes were successfully upgraded"),
1211                Err(_) => {
1212                    println!("WARNING: some nodes may not have been upgraded or restarted");
1213                }
1214            }
1215            return Ok(());
1216        }
1217
1218        if let Some(node_type) = &options.node_type {
1219            println!("Running the UpgradeNodes playbook for {node_type:?} nodes");
1220            match self.ansible_runner.run_playbook(
1221                AnsiblePlaybook::UpgradeNodes,
1222                node_type.to_ansible_inventory_type(),
1223                Some(options.get_ansible_vars()),
1224            ) {
1225                Ok(()) => println!("All {node_type:?} nodes were successfully upgraded"),
1226                Err(_) => {
1227                    println!(
1228                        "WARNING: some {node_type:?} nodes may not have been upgraded or restarted"
1229                    );
1230                }
1231            }
1232            return Ok(());
1233        }
1234
1235        println!("Running the UpgradeNodes playbook for all node types");
1236
1237        match self.ansible_runner.run_playbook(
1238            AnsiblePlaybook::UpgradeNodes,
1239            AnsibleInventoryType::PeerCacheNodes,
1240            Some(options.get_ansible_vars()),
1241        ) {
1242            Ok(()) => println!("All Peer Cache nodes were successfully upgraded"),
1243            Err(_) => {
1244                println!("WARNING: some Peer Cacche nodes may not have been upgraded or restarted");
1245            }
1246        }
1247        match self.ansible_runner.run_playbook(
1248            AnsiblePlaybook::UpgradeNodes,
1249            AnsibleInventoryType::Nodes,
1250            Some(options.get_ansible_vars()),
1251        ) {
1252            Ok(()) => println!("All generic nodes were successfully upgraded"),
1253            Err(_) => {
1254                println!("WARNING: some nodes may not have been upgraded or restarted");
1255            }
1256        }
1257        match self.ansible_runner.run_playbook(
1258            AnsiblePlaybook::UpgradeNodes,
1259            AnsibleInventoryType::SymmetricPrivateNodes,
1260            Some(options.get_ansible_vars()),
1261        ) {
1262            Ok(()) => println!("All private nodes were successfully upgraded"),
1263            Err(_) => {
1264                println!("WARNING: some nodes may not have been upgraded or restarted");
1265            }
1266        }
1267        // Don't use AnsibleInventoryType::iter_node_type() here, because the genesis node should be upgraded last
1268        match self.ansible_runner.run_playbook(
1269            AnsiblePlaybook::UpgradeNodes,
1270            AnsibleInventoryType::Genesis,
1271            Some(options.get_ansible_vars()),
1272        ) {
1273            Ok(()) => println!("The genesis nodes was successfully upgraded"),
1274            Err(_) => {
1275                println!("WARNING: the genesis node may not have been upgraded or restarted");
1276            }
1277        }
1278        Ok(())
1279    }
1280
1281    pub fn upgrade_antctl(
1282        &self,
1283        environment_name: &str,
1284        version: &Version,
1285        node_type: Option<NodeType>,
1286        custom_inventory: Option<Vec<VirtualMachine>>,
1287    ) -> Result<()> {
1288        let mut extra_vars = ExtraVarsDocBuilder::default();
1289        extra_vars.add_variable("version", &version.to_string());
1290
1291        if let Some(node_type) = node_type {
1292            println!("Running the upgrade safenode-manager playbook for {node_type:?} nodes");
1293            self.ansible_runner.run_playbook(
1294                AnsiblePlaybook::UpgradeAntctl,
1295                node_type.to_ansible_inventory_type(),
1296                Some(extra_vars.build()),
1297            )?;
1298            return Ok(());
1299        }
1300
1301        if let Some(custom_inventory) = custom_inventory {
1302            println!("Running the upgrade safenode-manager playbook with a custom inventory");
1303            generate_custom_environment_inventory(
1304                &custom_inventory,
1305                environment_name,
1306                &self.ansible_runner.working_directory_path.join("inventory"),
1307            )?;
1308            self.ansible_runner.run_playbook(
1309                AnsiblePlaybook::UpgradeAntctl,
1310                AnsibleInventoryType::Custom,
1311                Some(extra_vars.build()),
1312            )?;
1313            return Ok(());
1314        }
1315
1316        println!("Running the upgrade safenode-manager playbook for all node types");
1317        for node_inv_type in AnsibleInventoryType::iter_node_type() {
1318            self.ansible_runner.run_playbook(
1319                AnsiblePlaybook::UpgradeAntctl,
1320                node_inv_type,
1321                Some(extra_vars.build()),
1322            )?;
1323        }
1324
1325        Ok(())
1326    }
1327
1328    pub fn print_ansible_run_banner(&self, s: &str) {
1329        let ansible_run_msg = "Ansible Run: ";
1330        let line = "=".repeat(s.len() + ansible_run_msg.len());
1331        println!("{}\n{}{}\n{}", line, ansible_run_msg, s, line);
1332    }
1333}
sn_testnet_deploy/ansible/provisioning.rs

sn_testnet_deploy/ansible/
provisioning.rs
