sn_testnet_deploy/
upscale.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use crate::{
8    ansible::{
9        inventory::AnsibleInventoryType,
10        provisioning::{PrivateNodeProvisionInventory, ProvisionOptions},
11    },
12    error::{Error, Result},
13    get_anvil_node_data_hardcoded, get_bootstrap_cache_url, get_genesis_multiaddr, get_multiaddr,
14    DeploymentInventory, DeploymentType, EvmNetwork, InfraRunOptions, NodeType, TestnetDeployer,
15};
16use colored::Colorize;
17use evmlib::common::U256;
18use log::debug;
19use std::{collections::HashSet, time::Duration};
20
21#[derive(Clone)]
22pub struct UpscaleOptions {
23    pub ansible_verbose: bool,
24    pub ant_version: Option<String>,
25    pub current_inventory: DeploymentInventory,
26    pub desired_client_vm_count: Option<u16>,
27    pub desired_full_cone_private_node_count: Option<u16>,
28    pub desired_full_cone_private_node_vm_count: Option<u16>,
29    pub desired_node_count: Option<u16>,
30    pub desired_node_vm_count: Option<u16>,
31    pub desired_peer_cache_node_count: Option<u16>,
32    pub desired_peer_cache_node_vm_count: Option<u16>,
33    pub desired_symmetric_private_node_count: Option<u16>,
34    pub desired_symmetric_private_node_vm_count: Option<u16>,
35    pub desired_uploaders_count: Option<u16>,
36    pub funding_wallet_secret_key: Option<String>,
37    pub gas_amount: Option<U256>,
38    pub interval: Duration,
39    pub infra_only: bool,
40    pub max_archived_log_files: u16,
41    pub max_log_files: u16,
42    pub network_dashboard_branch: Option<String>,
43    pub node_env_variables: Option<Vec<(String, String)>>,
44    pub plan: bool,
45    pub public_rpc: bool,
46    pub provision_only: bool,
47    pub start_delayed_verifier: bool,
48    pub start_random_verifier: bool,
49    pub start_performance_verifier: bool,
50    pub token_amount: Option<U256>,
51}
52
53impl TestnetDeployer {
54    pub async fn upscale(&self, options: &UpscaleOptions) -> Result<()> {
55        let is_bootstrap_deploy = matches!(
56            options
57                .current_inventory
58                .environment_details
59                .deployment_type,
60            DeploymentType::Bootstrap
61        );
62
63        if is_bootstrap_deploy
64            && (options.desired_peer_cache_node_count.is_some()
65                || options.desired_peer_cache_node_vm_count.is_some()
66                || options.desired_client_vm_count.is_some())
67        {
68            return Err(Error::InvalidUpscaleOptionsForBootstrapDeployment);
69        }
70
71        let desired_peer_cache_node_vm_count = options
72            .desired_peer_cache_node_vm_count
73            .unwrap_or(options.current_inventory.peer_cache_node_vms.len() as u16);
74        if desired_peer_cache_node_vm_count
75            < options.current_inventory.peer_cache_node_vms.len() as u16
76        {
77            return Err(Error::InvalidUpscaleDesiredPeerCacheVmCount);
78        }
79        debug!("Using {desired_peer_cache_node_vm_count} for desired Peer Cache node VM count");
80
81        let desired_node_vm_count = options
82            .desired_node_vm_count
83            .unwrap_or(options.current_inventory.node_vms.len() as u16);
84        if desired_node_vm_count < options.current_inventory.node_vms.len() as u16 {
85            return Err(Error::InvalidUpscaleDesiredNodeVmCount);
86        }
87        debug!("Using {desired_node_vm_count} for desired node VM count");
88
89        let desired_full_cone_private_node_vm_count = options
90            .desired_full_cone_private_node_vm_count
91            .unwrap_or(options.current_inventory.full_cone_private_node_vms.len() as u16);
92        if desired_full_cone_private_node_vm_count
93            < options.current_inventory.full_cone_private_node_vms.len() as u16
94        {
95            return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeVmCount);
96        }
97        debug!("Using {desired_full_cone_private_node_vm_count} for desired full cone private node VM count");
98
99        let desired_symmetric_private_node_vm_count = options
100            .desired_symmetric_private_node_vm_count
101            .unwrap_or(options.current_inventory.symmetric_private_node_vms.len() as u16);
102        if desired_symmetric_private_node_vm_count
103            < options.current_inventory.symmetric_private_node_vms.len() as u16
104        {
105            return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeVmCount);
106        }
107        debug!("Using {desired_symmetric_private_node_vm_count} for desired full cone private node VM count");
108
109        let desired_client_vm_count = options
110            .desired_client_vm_count
111            .unwrap_or(options.current_inventory.client_vms.len() as u16);
112        if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
113            return Err(Error::InvalidUpscaleDesiredClientVmCount);
114        }
115        debug!("Using {desired_client_vm_count} for desired Client VM count");
116
117        let desired_peer_cache_node_count = options
118            .desired_peer_cache_node_count
119            .unwrap_or(options.current_inventory.peer_cache_node_count() as u16);
120        if desired_peer_cache_node_count < options.current_inventory.peer_cache_node_count() as u16
121        {
122            return Err(Error::InvalidUpscaleDesiredPeerCacheNodeCount);
123        }
124        debug!("Using {desired_peer_cache_node_count} for desired peer cache node count");
125
126        let desired_node_count = options
127            .desired_node_count
128            .unwrap_or(options.current_inventory.node_count() as u16);
129        if desired_node_count < options.current_inventory.node_count() as u16 {
130            return Err(Error::InvalidUpscaleDesiredNodeCount);
131        }
132        debug!("Using {desired_node_count} for desired node count");
133
134        let desired_full_cone_private_node_count = options
135            .desired_full_cone_private_node_count
136            .unwrap_or(options.current_inventory.full_cone_private_node_count() as u16);
137        if desired_full_cone_private_node_count
138            < options.current_inventory.full_cone_private_node_count() as u16
139        {
140            return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeCount);
141        }
142        debug!(
143            "Using {desired_full_cone_private_node_count} for desired full cone private node count"
144        );
145
146        let desired_symmetric_private_node_count = options
147            .desired_symmetric_private_node_count
148            .unwrap_or(options.current_inventory.symmetric_private_node_count() as u16);
149        if desired_symmetric_private_node_count
150            < options.current_inventory.symmetric_private_node_count() as u16
151        {
152            return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeCount);
153        }
154        debug!(
155            "Using {desired_symmetric_private_node_count} for desired symmetric private node count"
156        );
157
158        let mut infra_run_options = InfraRunOptions::generate_existing(
159            &options.current_inventory.name,
160            &options.current_inventory.environment_details.region,
161            &self.terraform_runner,
162            Some(&options.current_inventory.environment_details),
163        )
164        .await?;
165        infra_run_options.peer_cache_node_vm_count = Some(desired_peer_cache_node_vm_count);
166        infra_run_options.node_vm_count = Some(desired_node_vm_count);
167        infra_run_options.full_cone_private_node_vm_count =
168            Some(desired_full_cone_private_node_vm_count);
169        infra_run_options.symmetric_private_node_vm_count =
170            Some(desired_symmetric_private_node_vm_count);
171        infra_run_options.client_vm_count = Some(desired_client_vm_count);
172
173        if options.plan {
174            self.plan(&infra_run_options)?;
175            return Ok(());
176        }
177
178        self.create_or_update_infra(&infra_run_options)
179            .map_err(|err| {
180                println!("Failed to create infra {err:?}");
181                err
182            })?;
183
184        if options.infra_only {
185            return Ok(());
186        }
187
188        let mut provision_options = ProvisionOptions {
189            ant_version: options.ant_version.clone(),
190            binary_option: options.current_inventory.binary_option.clone(),
191            chunk_size: None,
192            chunk_tracker_data_addresses: None,
193            chunk_tracker_services: None,
194            client_env_variables: None,
195            delayed_verifier_batch_size: None,
196            delayed_verifier_quorum_value: None,
197            disable_nodes: false,
198            enable_logging: true,
199            enable_metrics: true,
200            evm_data_payments_address: options
201                .current_inventory
202                .environment_details
203                .evm_details
204                .data_payments_address
205                .clone(),
206            evm_merkle_payments_address: options
207                .current_inventory
208                .environment_details
209                .evm_details
210                .merkle_payments_address
211                .clone(),
212            evm_network: options
213                .current_inventory
214                .environment_details
215                .evm_details
216                .network
217                .clone(),
218            evm_payment_token_address: options
219                .current_inventory
220                .environment_details
221                .evm_details
222                .payment_token_address
223                .clone(),
224            evm_rpc_url: options
225                .current_inventory
226                .environment_details
227                .evm_details
228                .rpc_url
229                .clone(),
230            expected_hash: None,
231            expected_size: None,
232            file_address: None,
233            full_cone_private_node_count: desired_full_cone_private_node_count,
234            funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
235            gas_amount: options.gas_amount,
236            interval: Some(options.interval),
237            log_format: None,
238            max_archived_log_files: options.max_archived_log_files,
239            max_log_files: options.max_log_files,
240            max_uploads: None,
241            merkle: false,
242            name: options.current_inventory.name.clone(),
243            network_id: options.current_inventory.environment_details.network_id,
244            network_dashboard_branch: None,
245            node_count: desired_node_count,
246            node_env_variables: options.node_env_variables.clone(),
247            output_inventory_dir_path: self
248                .working_directory_path
249                .join("ansible")
250                .join("inventory"),
251            peer_cache_node_count: desired_peer_cache_node_count,
252            performance_verifier_batch_size: None,
253            port_restricted_cone_private_node_count: 0,
254            public_rpc: options.public_rpc,
255            random_verifier_batch_size: None,
256            repair_service_count: 0,
257            data_retrieval_service_count: 0,
258            rewards_address: options
259                .current_inventory
260                .environment_details
261                .rewards_address
262                .clone(),
263            scan_frequency: None,
264            single_node_payment: false,
265            sleep_duration: None,
266            start_chunk_trackers: false,
267            start_data_retrieval: false,
268            start_delayed_verifier: options.start_delayed_verifier,
269            start_performance_verifier: options.start_performance_verifier,
270            start_random_verifier: options.start_random_verifier,
271            start_uploaders: false,
272            symmetric_private_node_count: desired_symmetric_private_node_count,
273            token_amount: None,
274            upload_batch_size: None,
275            upload_size: None,
276            uploaders_count: options.desired_uploaders_count,
277            upload_interval: None,
278            upnp_private_node_count: 0,
279            wallet_secret_keys: None,
280        };
281        let mut node_provision_failed = false;
282
283        let (initial_multiaddr, initial_ip_addr) = if is_bootstrap_deploy {
284            get_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client).map_err(
285                |err| {
286                    println!("Failed to get node multiaddr {err:?}");
287                    err
288                },
289            )?
290        } else {
291            get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
292                .map_err(|err| {
293                    println!("Failed to get genesis multiaddr {err:?}");
294                    err
295                })?
296                .ok_or_else(|| Error::GenesisListenAddress)?
297        };
298        let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
299        debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
300
301        if !is_bootstrap_deploy {
302            self.wait_for_ssh_availability_on_new_machines(
303                AnsibleInventoryType::PeerCacheNodes,
304                &options.current_inventory,
305            )?;
306            self.ansible_provisioner
307                .print_ansible_run_banner("Provision Peer Cache Nodes");
308            match self.ansible_provisioner.provision_nodes(
309                &provision_options,
310                Some(initial_multiaddr.clone()),
311                Some(initial_network_contacts_url.clone()),
312                NodeType::PeerCache,
313            ) {
314                Ok(()) => {
315                    println!("Provisioned Peer Cache nodes");
316                }
317                Err(err) => {
318                    log::error!("Failed to provision Peer Cache nodes: {err}");
319                    node_provision_failed = true;
320                }
321            }
322        }
323
324        self.wait_for_ssh_availability_on_new_machines(
325            AnsibleInventoryType::Nodes,
326            &options.current_inventory,
327        )?;
328        self.ansible_provisioner
329            .print_ansible_run_banner("Provision Normal Nodes");
330        match self.ansible_provisioner.provision_nodes(
331            &provision_options,
332            Some(initial_multiaddr.clone()),
333            Some(initial_network_contacts_url.clone()),
334            NodeType::Generic,
335        ) {
336            Ok(()) => {
337                println!("Provisioned normal nodes");
338            }
339            Err(err) => {
340                log::error!("Failed to provision normal nodes: {err}");
341                node_provision_failed = true;
342            }
343        }
344
345        let private_node_inventory = PrivateNodeProvisionInventory::new(
346            &self.ansible_provisioner,
347            Some(desired_full_cone_private_node_vm_count),
348            Some(desired_symmetric_private_node_vm_count),
349            None, // TODO: Add port restricted cone upscale support
350        )?;
351
352        if private_node_inventory.should_provision_full_cone_private_nodes() {
353            let full_cone_nat_gateway_inventory = self
354                .ansible_provisioner
355                .ansible_runner
356                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)?;
357
358            let full_cone_nat_gateway_new_vms: Vec<_> = full_cone_nat_gateway_inventory
359                .into_iter()
360                .filter(|item| {
361                    !options
362                        .current_inventory
363                        .full_cone_nat_gateway_vms
364                        .contains(item)
365                })
366                .collect();
367
368            for vm in full_cone_nat_gateway_new_vms.iter() {
369                self.ssh_client.wait_for_ssh_availability(
370                    &vm.public_ip_addr,
371                    &self.cloud_provider.get_ssh_user(),
372                )?;
373            }
374
375            let full_cone_nat_gateway_new_vms = if full_cone_nat_gateway_new_vms.is_empty() {
376                None
377            } else {
378                debug!("Full Cone NAT Gateway new VMs: {full_cone_nat_gateway_new_vms:?}");
379                Some(full_cone_nat_gateway_new_vms)
380            };
381
382            match self.ansible_provisioner.provision_full_cone(
383                &provision_options,
384                Some(initial_multiaddr.clone()),
385                Some(initial_network_contacts_url.clone()),
386                private_node_inventory.clone(),
387                full_cone_nat_gateway_new_vms,
388            ) {
389                Ok(()) => {
390                    println!("Provisioned Full Cone nodes and Gateway");
391                }
392                Err(err) => {
393                    log::error!("Failed to provision Full Cone nodes and Gateway: {err}");
394                    node_provision_failed = true;
395                }
396            }
397        }
398
399        if private_node_inventory.should_provision_symmetric_private_nodes() {
400            self.wait_for_ssh_availability_on_new_machines(
401                AnsibleInventoryType::SymmetricNatGateway,
402                &options.current_inventory,
403            )?;
404            self.ansible_provisioner
405                .print_ansible_run_banner("Provision Symmetric NAT Gateway");
406            self.ansible_provisioner
407                .provision_symmetric_nat_gateway(&provision_options, &private_node_inventory)
408                .map_err(|err| {
409                    println!("Failed to provision symmetric NAT gateway {err:?}");
410                    err
411                })?;
412
413            self.wait_for_ssh_availability_on_new_machines(
414                AnsibleInventoryType::SymmetricPrivateNodes,
415                &options.current_inventory,
416            )?;
417            self.ansible_provisioner
418                .print_ansible_run_banner("Provision Symmetric Private Nodes");
419            match self.ansible_provisioner.provision_symmetric_private_nodes(
420                &mut provision_options,
421                Some(initial_multiaddr.clone()),
422                Some(initial_network_contacts_url.clone()),
423                &private_node_inventory,
424            ) {
425                Ok(()) => {
426                    println!("Provisioned symmetric private nodes");
427                }
428                Err(err) => {
429                    log::error!("Failed to provision symmetric private nodes: {err}");
430                    node_provision_failed = true;
431                }
432            }
433        }
434
435        let should_provision_uploaders =
436            options.desired_uploaders_count.is_some() || options.desired_client_vm_count.is_some();
437        if should_provision_uploaders {
438            // get anvil funding sk
439            if provision_options.evm_network == EvmNetwork::Anvil {
440                let anvil_node_data =
441                    get_anvil_node_data_hardcoded(&self.ansible_provisioner.ansible_runner)
442                        .map_err(|err| {
443                            println!("Failed to get evm testnet data {err:?}");
444                            err
445                        })?;
446
447                provision_options.funding_wallet_secret_key =
448                    Some(anvil_node_data.deployer_wallet_private_key);
449            }
450
451            self.wait_for_ssh_availability_on_new_machines(
452                AnsibleInventoryType::Clients,
453                &options.current_inventory,
454            )?;
455            let genesis_network_contacts = get_bootstrap_cache_url(&initial_ip_addr);
456            self.ansible_provisioner
457                .print_ansible_run_banner("Provision Clients");
458            self.ansible_provisioner
459                .provision_uploaders(
460                    &provision_options,
461                    Some(initial_multiaddr.clone()),
462                    Some(genesis_network_contacts.clone()),
463                )
464                .await
465                .map_err(|err| {
466                    println!("Failed to provision Clients {err:?}");
467                    err
468                })?;
469        }
470
471        if node_provision_failed {
472            println!();
473            println!("{}", "WARNING!".yellow());
474            println!("Some nodes failed to provision without error.");
475            println!("This usually means a small number of nodes failed to start on a few VMs.");
476            println!("However, most of the time the deployment will still be usable.");
477            println!("See the output from Ansible to determine which VMs had failures.");
478        }
479
480        Ok(())
481    }
482
483    pub async fn upscale_clients(&self, options: &UpscaleOptions) -> Result<()> {
484        let is_bootstrap_deploy = matches!(
485            options
486                .current_inventory
487                .environment_details
488                .deployment_type,
489            DeploymentType::Bootstrap
490        );
491
492        if is_bootstrap_deploy {
493            return Err(Error::InvalidClientUpscaleDeploymentType(
494                "bootstrap".to_string(),
495            ));
496        }
497
498        let desired_client_vm_count = options
499            .desired_client_vm_count
500            .unwrap_or(options.current_inventory.client_vms.len() as u16);
501        if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
502            return Err(Error::InvalidUpscaleDesiredClientVmCount);
503        }
504        debug!("Using {desired_client_vm_count} for desired Client VM count");
505
506        let mut infra_run_options = InfraRunOptions::generate_existing(
507            &options.current_inventory.name,
508            &options.current_inventory.environment_details.region,
509            &self.terraform_runner,
510            Some(&options.current_inventory.environment_details),
511        )
512        .await?;
513        infra_run_options.client_vm_count = Some(desired_client_vm_count);
514
515        if options.plan {
516            self.plan(&infra_run_options)?;
517            return Ok(());
518        }
519
520        if !options.provision_only {
521            self.create_or_update_infra(&infra_run_options)
522                .map_err(|err| {
523                    println!("Failed to create infra {err:?}");
524                    err
525                })?;
526        }
527
528        if options.infra_only {
529            return Ok(());
530        }
531
532        let (initial_multiaddr, initial_ip_addr) =
533            get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)?
534                .ok_or_else(|| Error::GenesisListenAddress)?;
535        let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
536        debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
537
538        let provision_options = ProvisionOptions {
539            ant_version: options.ant_version.clone(),
540            binary_option: options.current_inventory.binary_option.clone(),
541            chunk_size: None,
542            chunk_tracker_data_addresses: None,
543            chunk_tracker_services: None,
544            client_env_variables: None,
545            delayed_verifier_batch_size: None,
546            delayed_verifier_quorum_value: None,
547            disable_nodes: false,
548            enable_logging: true,
549            enable_metrics: true,
550            evm_data_payments_address: options
551                .current_inventory
552                .environment_details
553                .evm_details
554                .data_payments_address
555                .clone(),
556            evm_merkle_payments_address: options
557                .current_inventory
558                .environment_details
559                .evm_details
560                .merkle_payments_address
561                .clone(),
562            evm_network: options
563                .current_inventory
564                .environment_details
565                .evm_details
566                .network
567                .clone(),
568            evm_payment_token_address: options
569                .current_inventory
570                .environment_details
571                .evm_details
572                .payment_token_address
573                .clone(),
574            evm_rpc_url: options
575                .current_inventory
576                .environment_details
577                .evm_details
578                .rpc_url
579                .clone(),
580            expected_hash: None,
581            expected_size: None,
582            file_address: None,
583            full_cone_private_node_count: 0,
584            funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
585            gas_amount: options.gas_amount,
586            interval: Some(options.interval),
587            log_format: None,
588            max_archived_log_files: options.max_archived_log_files,
589            max_log_files: options.max_log_files,
590            max_uploads: None,
591            merkle: false,
592            name: options.current_inventory.name.clone(),
593            network_id: options.current_inventory.environment_details.network_id,
594            network_dashboard_branch: None,
595            node_count: 0,
596            node_env_variables: None,
597            output_inventory_dir_path: self
598                .working_directory_path
599                .join("ansible")
600                .join("inventory"),
601            peer_cache_node_count: 0,
602            performance_verifier_batch_size: None,
603            public_rpc: options.public_rpc,
604            random_verifier_batch_size: None,
605            repair_service_count: 0,
606            data_retrieval_service_count: 0,
607            rewards_address: options
608                .current_inventory
609                .environment_details
610                .rewards_address
611                .clone(),
612            scan_frequency: None,
613            single_node_payment: false,
614            sleep_duration: None,
615            start_chunk_trackers: false,
616            start_data_retrieval: false,
617            start_delayed_verifier: options.start_delayed_verifier,
618            start_random_verifier: options.start_random_verifier,
619            start_performance_verifier: options.start_performance_verifier,
620            start_uploaders: false,
621            symmetric_private_node_count: 0,
622            token_amount: options.token_amount,
623            uploaders_count: options.desired_uploaders_count,
624            upload_batch_size: None,
625            upload_size: None,
626            upload_interval: None,
627            upnp_private_node_count: 0,
628            port_restricted_cone_private_node_count: 0,
629            wallet_secret_keys: None,
630        };
631
632        self.wait_for_ssh_availability_on_new_machines(
633            AnsibleInventoryType::Clients,
634            &options.current_inventory,
635        )?;
636        self.ansible_provisioner
637            .print_ansible_run_banner("Provision Clients");
638        self.ansible_provisioner
639            .provision_uploaders(
640                &provision_options,
641                Some(initial_multiaddr),
642                Some(initial_network_contacts_url),
643            )
644            .await
645            .map_err(|err| {
646                println!("Failed to provision clients {err:?}");
647                err
648            })?;
649
650        Ok(())
651    }
652
653    fn wait_for_ssh_availability_on_new_machines(
654        &self,
655        inventory_type: AnsibleInventoryType,
656        current_inventory: &DeploymentInventory,
657    ) -> Result<()> {
658        let inventory = self
659            .ansible_provisioner
660            .ansible_runner
661            .get_inventory(inventory_type, true)?;
662        let old_set: HashSet<_> = match inventory_type {
663            AnsibleInventoryType::Clients => current_inventory
664                .client_vms
665                .iter()
666                .map(|client_vm| &client_vm.vm)
667                .cloned()
668                .collect(),
669            AnsibleInventoryType::PeerCacheNodes => current_inventory
670                .peer_cache_node_vms
671                .iter()
672                .map(|node_vm| &node_vm.vm)
673                .cloned()
674                .collect(),
675            AnsibleInventoryType::Nodes => current_inventory
676                .node_vms
677                .iter()
678                .map(|node_vm| &node_vm.vm)
679                .cloned()
680                .collect(),
681            AnsibleInventoryType::FullConeNatGateway => current_inventory
682                .full_cone_nat_gateway_vms
683                .iter()
684                .cloned()
685                .collect(),
686            AnsibleInventoryType::SymmetricNatGateway => current_inventory
687                .symmetric_nat_gateway_vms
688                .iter()
689                .cloned()
690                .collect(),
691            AnsibleInventoryType::FullConePrivateNodes => current_inventory
692                .full_cone_private_node_vms
693                .iter()
694                .map(|node_vm| &node_vm.vm)
695                .cloned()
696                .collect(),
697            AnsibleInventoryType::SymmetricPrivateNodes => current_inventory
698                .symmetric_private_node_vms
699                .iter()
700                .map(|node_vm| &node_vm.vm)
701                .cloned()
702                .collect(),
703            it => return Err(Error::UpscaleInventoryTypeNotSupported(it.to_string())),
704        };
705        let new_vms: Vec<_> = inventory
706            .into_iter()
707            .filter(|item| !old_set.contains(item))
708            .collect();
709        for vm in new_vms.iter() {
710            self.ssh_client.wait_for_ssh_availability(
711                &vm.public_ip_addr,
712                &self.cloud_provider.get_ssh_user(),
713            )?;
714        }
715        Ok(())
716    }
717}
sn_testnet_deploy/upscale.rs

sn_testnet_deploy/
upscale.rs
