sn_testnet_deploy/
upscale.rs

1// Copyright (c) 2023, MaidSafe.
2// All rights reserved.
3//
4// This SAFE Network Software is licensed under the BSD-3-Clause license.
5// Please see the LICENSE file for more details.
6
7use crate::{
8    ansible::{
9        inventory::AnsibleInventoryType,
10        provisioning::{PrivateNodeProvisionInventory, ProvisionOptions},
11    },
12    error::{Error, Result},
13    get_anvil_node_data_hardcoded, get_bootstrap_cache_url, get_genesis_multiaddr, get_multiaddr,
14    DeploymentInventory, DeploymentType, EvmNetwork, InfraRunOptions, NodeType, TestnetDeployer,
15};
16use colored::Colorize;
17use evmlib::common::U256;
18use log::debug;
19use std::{collections::HashSet, time::Duration};
20
21#[derive(Clone)]
22pub struct UpscaleOptions {
23    pub ansible_verbose: bool,
24    pub ant_version: Option<String>,
25    pub current_inventory: DeploymentInventory,
26    pub desired_client_vm_count: Option<u16>,
27    pub desired_full_cone_private_node_count: Option<u16>,
28    pub desired_full_cone_private_node_vm_count: Option<u16>,
29    pub desired_node_count: Option<u16>,
30    pub desired_node_vm_count: Option<u16>,
31    pub desired_peer_cache_node_count: Option<u16>,
32    pub desired_peer_cache_node_vm_count: Option<u16>,
33    pub desired_symmetric_private_node_count: Option<u16>,
34    pub desired_symmetric_private_node_vm_count: Option<u16>,
35    pub desired_uploaders_count: Option<u16>,
36    pub funding_wallet_secret_key: Option<String>,
37    pub gas_amount: Option<U256>,
38    pub interval: Duration,
39    pub infra_only: bool,
40    pub max_archived_log_files: u16,
41    pub max_log_files: u16,
42    pub network_dashboard_branch: Option<String>,
43    pub node_env_variables: Option<Vec<(String, String)>>,
44    pub plan: bool,
45    pub public_rpc: bool,
46    pub provision_only: bool,
47    pub start_delayed_verifier: bool,
48    pub start_random_verifier: bool,
49    pub start_performance_verifier: bool,
50    pub token_amount: Option<U256>,
51}
52
53impl TestnetDeployer {
54    pub async fn upscale(&self, options: &UpscaleOptions) -> Result<()> {
55        let is_bootstrap_deploy = matches!(
56            options
57                .current_inventory
58                .environment_details
59                .deployment_type,
60            DeploymentType::Bootstrap
61        );
62
63        if is_bootstrap_deploy
64            && (options.desired_peer_cache_node_count.is_some()
65                || options.desired_peer_cache_node_vm_count.is_some()
66                || options.desired_client_vm_count.is_some())
67        {
68            return Err(Error::InvalidUpscaleOptionsForBootstrapDeployment);
69        }
70
71        let desired_peer_cache_node_vm_count = options
72            .desired_peer_cache_node_vm_count
73            .unwrap_or(options.current_inventory.peer_cache_node_vms.len() as u16);
74        if desired_peer_cache_node_vm_count
75            < options.current_inventory.peer_cache_node_vms.len() as u16
76        {
77            return Err(Error::InvalidUpscaleDesiredPeerCacheVmCount);
78        }
79        debug!("Using {desired_peer_cache_node_vm_count} for desired Peer Cache node VM count");
80
81        let desired_node_vm_count = options
82            .desired_node_vm_count
83            .unwrap_or(options.current_inventory.node_vms.len() as u16);
84        if desired_node_vm_count < options.current_inventory.node_vms.len() as u16 {
85            return Err(Error::InvalidUpscaleDesiredNodeVmCount);
86        }
87        debug!("Using {desired_node_vm_count} for desired node VM count");
88
89        let desired_full_cone_private_node_vm_count = options
90            .desired_full_cone_private_node_vm_count
91            .unwrap_or(options.current_inventory.full_cone_private_node_vms.len() as u16);
92        if desired_full_cone_private_node_vm_count
93            < options.current_inventory.full_cone_private_node_vms.len() as u16
94        {
95            return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeVmCount);
96        }
97        debug!("Using {desired_full_cone_private_node_vm_count} for desired full cone private node VM count");
98
99        let desired_symmetric_private_node_vm_count = options
100            .desired_symmetric_private_node_vm_count
101            .unwrap_or(options.current_inventory.symmetric_private_node_vms.len() as u16);
102        if desired_symmetric_private_node_vm_count
103            < options.current_inventory.symmetric_private_node_vms.len() as u16
104        {
105            return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeVmCount);
106        }
107        debug!("Using {desired_symmetric_private_node_vm_count} for desired full cone private node VM count");
108
109        let desired_client_vm_count = options
110            .desired_client_vm_count
111            .unwrap_or(options.current_inventory.client_vms.len() as u16);
112        if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
113            return Err(Error::InvalidUpscaleDesiredClientVmCount);
114        }
115        debug!("Using {desired_client_vm_count} for desired Client VM count");
116
117        let desired_peer_cache_node_count = options
118            .desired_peer_cache_node_count
119            .unwrap_or(options.current_inventory.peer_cache_node_count() as u16);
120        if desired_peer_cache_node_count < options.current_inventory.peer_cache_node_count() as u16
121        {
122            return Err(Error::InvalidUpscaleDesiredPeerCacheNodeCount);
123        }
124        debug!("Using {desired_peer_cache_node_count} for desired peer cache node count");
125
126        let desired_node_count = options
127            .desired_node_count
128            .unwrap_or(options.current_inventory.node_count() as u16);
129        if desired_node_count < options.current_inventory.node_count() as u16 {
130            return Err(Error::InvalidUpscaleDesiredNodeCount);
131        }
132        debug!("Using {desired_node_count} for desired node count");
133
134        let desired_full_cone_private_node_count = options
135            .desired_full_cone_private_node_count
136            .unwrap_or(options.current_inventory.full_cone_private_node_count() as u16);
137        if desired_full_cone_private_node_count
138            < options.current_inventory.full_cone_private_node_count() as u16
139        {
140            return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeCount);
141        }
142        debug!(
143            "Using {desired_full_cone_private_node_count} for desired full cone private node count"
144        );
145
146        let desired_symmetric_private_node_count = options
147            .desired_symmetric_private_node_count
148            .unwrap_or(options.current_inventory.symmetric_private_node_count() as u16);
149        if desired_symmetric_private_node_count
150            < options.current_inventory.symmetric_private_node_count() as u16
151        {
152            return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeCount);
153        }
154        debug!(
155            "Using {desired_symmetric_private_node_count} for desired symmetric private node count"
156        );
157
158        let mut infra_run_options = InfraRunOptions::generate_existing(
159            &options.current_inventory.name,
160            &options.current_inventory.environment_details.region,
161            &self.terraform_runner,
162            Some(&options.current_inventory.environment_details),
163        )
164        .await?;
165        infra_run_options.peer_cache_node_vm_count = Some(desired_peer_cache_node_vm_count);
166        infra_run_options.node_vm_count = Some(desired_node_vm_count);
167        infra_run_options.full_cone_private_node_vm_count =
168            Some(desired_full_cone_private_node_vm_count);
169        infra_run_options.symmetric_private_node_vm_count =
170            Some(desired_symmetric_private_node_vm_count);
171        infra_run_options.client_vm_count = Some(desired_client_vm_count);
172
173        if options.plan {
174            self.plan(&infra_run_options)?;
175            return Ok(());
176        }
177
178        self.create_or_update_infra(&infra_run_options)
179            .map_err(|err| {
180                println!("Failed to create infra {err:?}");
181                err
182            })?;
183
184        if options.infra_only {
185            return Ok(());
186        }
187
188        let mut provision_options = ProvisionOptions {
189            ant_version: options.ant_version.clone(),
190            binary_option: options.current_inventory.binary_option.clone(),
191            chunk_size: None,
192            chunk_tracker_data_addresses: None,
193            chunk_tracker_services: None,
194            client_env_variables: None,
195            delayed_verifier_batch_size: None,
196            delayed_verifier_quorum_value: None,
197            disable_nodes: false,
198            enable_logging: true,
199            enable_metrics: true,
200            evm_data_payments_address: options
201                .current_inventory
202                .environment_details
203                .evm_details
204                .data_payments_address
205                .clone(),
206            evm_network: options
207                .current_inventory
208                .environment_details
209                .evm_details
210                .network
211                .clone(),
212            evm_payment_token_address: options
213                .current_inventory
214                .environment_details
215                .evm_details
216                .payment_token_address
217                .clone(),
218            evm_rpc_url: options
219                .current_inventory
220                .environment_details
221                .evm_details
222                .rpc_url
223                .clone(),
224            expected_hash: None,
225            expected_size: None,
226            file_address: None,
227            full_cone_private_node_count: desired_full_cone_private_node_count,
228            funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
229            gas_amount: options.gas_amount,
230            interval: Some(options.interval),
231            log_format: None,
232            max_archived_log_files: options.max_archived_log_files,
233            max_log_files: options.max_log_files,
234            max_uploads: None,
235            name: options.current_inventory.name.clone(),
236            network_id: options.current_inventory.environment_details.network_id,
237            network_dashboard_branch: None,
238            node_count: desired_node_count,
239            node_env_variables: options.node_env_variables.clone(),
240            output_inventory_dir_path: self
241                .working_directory_path
242                .join("ansible")
243                .join("inventory"),
244            peer_cache_node_count: desired_peer_cache_node_count,
245            performance_verifier_batch_size: None,
246            port_restricted_cone_private_node_count: 0,
247            public_rpc: options.public_rpc,
248            random_verifier_batch_size: None,
249            repair_service_count: 0,
250            data_retrieval_service_count: 0,
251            rewards_address: options
252                .current_inventory
253                .environment_details
254                .rewards_address
255                .clone(),
256            scan_frequency: None,
257            single_node_payment: false,
258            sleep_duration: None,
259            start_chunk_trackers: false,
260            start_data_retrieval: false,
261            start_delayed_verifier: options.start_delayed_verifier,
262            start_performance_verifier: options.start_performance_verifier,
263            start_random_verifier: options.start_random_verifier,
264            start_uploaders: false,
265            symmetric_private_node_count: desired_symmetric_private_node_count,
266            token_amount: None,
267            upload_batch_size: None,
268            upload_size: None,
269            uploaders_count: options.desired_uploaders_count,
270            upload_interval: None,
271            upnp_private_node_count: 0,
272            wallet_secret_keys: None,
273        };
274        let mut node_provision_failed = false;
275
276        let (initial_multiaddr, initial_ip_addr) = if is_bootstrap_deploy {
277            get_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client).map_err(
278                |err| {
279                    println!("Failed to get node multiaddr {err:?}");
280                    err
281                },
282            )?
283        } else {
284            get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
285                .map_err(|err| {
286                    println!("Failed to get genesis multiaddr {err:?}");
287                    err
288                })?
289                .ok_or_else(|| Error::GenesisListenAddress)?
290        };
291        let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
292        debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
293
294        if !is_bootstrap_deploy {
295            self.wait_for_ssh_availability_on_new_machines(
296                AnsibleInventoryType::PeerCacheNodes,
297                &options.current_inventory,
298            )?;
299            self.ansible_provisioner
300                .print_ansible_run_banner("Provision Peer Cache Nodes");
301            match self.ansible_provisioner.provision_nodes(
302                &provision_options,
303                Some(initial_multiaddr.clone()),
304                Some(initial_network_contacts_url.clone()),
305                NodeType::PeerCache,
306            ) {
307                Ok(()) => {
308                    println!("Provisioned Peer Cache nodes");
309                }
310                Err(err) => {
311                    log::error!("Failed to provision Peer Cache nodes: {err}");
312                    node_provision_failed = true;
313                }
314            }
315        }
316
317        self.wait_for_ssh_availability_on_new_machines(
318            AnsibleInventoryType::Nodes,
319            &options.current_inventory,
320        )?;
321        self.ansible_provisioner
322            .print_ansible_run_banner("Provision Normal Nodes");
323        match self.ansible_provisioner.provision_nodes(
324            &provision_options,
325            Some(initial_multiaddr.clone()),
326            Some(initial_network_contacts_url.clone()),
327            NodeType::Generic,
328        ) {
329            Ok(()) => {
330                println!("Provisioned normal nodes");
331            }
332            Err(err) => {
333                log::error!("Failed to provision normal nodes: {err}");
334                node_provision_failed = true;
335            }
336        }
337
338        let private_node_inventory = PrivateNodeProvisionInventory::new(
339            &self.ansible_provisioner,
340            Some(desired_full_cone_private_node_vm_count),
341            Some(desired_symmetric_private_node_vm_count),
342            None, // TODO: Add port restricted cone upscale support
343        )?;
344
345        if private_node_inventory.should_provision_full_cone_private_nodes() {
346            let full_cone_nat_gateway_inventory = self
347                .ansible_provisioner
348                .ansible_runner
349                .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)?;
350
351            let full_cone_nat_gateway_new_vms: Vec<_> = full_cone_nat_gateway_inventory
352                .into_iter()
353                .filter(|item| {
354                    !options
355                        .current_inventory
356                        .full_cone_nat_gateway_vms
357                        .contains(item)
358                })
359                .collect();
360
361            for vm in full_cone_nat_gateway_new_vms.iter() {
362                self.ssh_client.wait_for_ssh_availability(
363                    &vm.public_ip_addr,
364                    &self.cloud_provider.get_ssh_user(),
365                )?;
366            }
367
368            let full_cone_nat_gateway_new_vms = if full_cone_nat_gateway_new_vms.is_empty() {
369                None
370            } else {
371                debug!("Full Cone NAT Gateway new VMs: {full_cone_nat_gateway_new_vms:?}");
372                Some(full_cone_nat_gateway_new_vms)
373            };
374
375            match self.ansible_provisioner.provision_full_cone(
376                &provision_options,
377                Some(initial_multiaddr.clone()),
378                Some(initial_network_contacts_url.clone()),
379                private_node_inventory.clone(),
380                full_cone_nat_gateway_new_vms,
381            ) {
382                Ok(()) => {
383                    println!("Provisioned Full Cone nodes and Gateway");
384                }
385                Err(err) => {
386                    log::error!("Failed to provision Full Cone nodes and Gateway: {err}");
387                    node_provision_failed = true;
388                }
389            }
390        }
391
392        if private_node_inventory.should_provision_symmetric_private_nodes() {
393            self.wait_for_ssh_availability_on_new_machines(
394                AnsibleInventoryType::SymmetricNatGateway,
395                &options.current_inventory,
396            )?;
397            self.ansible_provisioner
398                .print_ansible_run_banner("Provision Symmetric NAT Gateway");
399            self.ansible_provisioner
400                .provision_symmetric_nat_gateway(&provision_options, &private_node_inventory)
401                .map_err(|err| {
402                    println!("Failed to provision symmetric NAT gateway {err:?}");
403                    err
404                })?;
405
406            self.wait_for_ssh_availability_on_new_machines(
407                AnsibleInventoryType::SymmetricPrivateNodes,
408                &options.current_inventory,
409            )?;
410            self.ansible_provisioner
411                .print_ansible_run_banner("Provision Symmetric Private Nodes");
412            match self.ansible_provisioner.provision_symmetric_private_nodes(
413                &mut provision_options,
414                Some(initial_multiaddr.clone()),
415                Some(initial_network_contacts_url.clone()),
416                &private_node_inventory,
417            ) {
418                Ok(()) => {
419                    println!("Provisioned symmetric private nodes");
420                }
421                Err(err) => {
422                    log::error!("Failed to provision symmetric private nodes: {err}");
423                    node_provision_failed = true;
424                }
425            }
426        }
427
428        let should_provision_uploaders =
429            options.desired_uploaders_count.is_some() || options.desired_client_vm_count.is_some();
430        if should_provision_uploaders {
431            // get anvil funding sk
432            if provision_options.evm_network == EvmNetwork::Anvil {
433                let anvil_node_data =
434                    get_anvil_node_data_hardcoded(&self.ansible_provisioner.ansible_runner)
435                        .map_err(|err| {
436                            println!("Failed to get evm testnet data {err:?}");
437                            err
438                        })?;
439
440                provision_options.funding_wallet_secret_key =
441                    Some(anvil_node_data.deployer_wallet_private_key);
442            }
443
444            self.wait_for_ssh_availability_on_new_machines(
445                AnsibleInventoryType::Clients,
446                &options.current_inventory,
447            )?;
448            let genesis_network_contacts = get_bootstrap_cache_url(&initial_ip_addr);
449            self.ansible_provisioner
450                .print_ansible_run_banner("Provision Clients");
451            self.ansible_provisioner
452                .provision_uploaders(
453                    &provision_options,
454                    Some(initial_multiaddr.clone()),
455                    Some(genesis_network_contacts.clone()),
456                )
457                .await
458                .map_err(|err| {
459                    println!("Failed to provision Clients {err:?}");
460                    err
461                })?;
462        }
463
464        if node_provision_failed {
465            println!();
466            println!("{}", "WARNING!".yellow());
467            println!("Some nodes failed to provision without error.");
468            println!("This usually means a small number of nodes failed to start on a few VMs.");
469            println!("However, most of the time the deployment will still be usable.");
470            println!("See the output from Ansible to determine which VMs had failures.");
471        }
472
473        Ok(())
474    }
475
476    pub async fn upscale_clients(&self, options: &UpscaleOptions) -> Result<()> {
477        let is_bootstrap_deploy = matches!(
478            options
479                .current_inventory
480                .environment_details
481                .deployment_type,
482            DeploymentType::Bootstrap
483        );
484
485        if is_bootstrap_deploy {
486            return Err(Error::InvalidClientUpscaleDeploymentType(
487                "bootstrap".to_string(),
488            ));
489        }
490
491        let desired_client_vm_count = options
492            .desired_client_vm_count
493            .unwrap_or(options.current_inventory.client_vms.len() as u16);
494        if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
495            return Err(Error::InvalidUpscaleDesiredClientVmCount);
496        }
497        debug!("Using {desired_client_vm_count} for desired Client VM count");
498
499        let mut infra_run_options = InfraRunOptions::generate_existing(
500            &options.current_inventory.name,
501            &options.current_inventory.environment_details.region,
502            &self.terraform_runner,
503            Some(&options.current_inventory.environment_details),
504        )
505        .await?;
506        infra_run_options.client_vm_count = Some(desired_client_vm_count);
507
508        if options.plan {
509            self.plan(&infra_run_options)?;
510            return Ok(());
511        }
512
513        if !options.provision_only {
514            self.create_or_update_infra(&infra_run_options)
515                .map_err(|err| {
516                    println!("Failed to create infra {err:?}");
517                    err
518                })?;
519        }
520
521        if options.infra_only {
522            return Ok(());
523        }
524
525        let (initial_multiaddr, initial_ip_addr) =
526            get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)?
527                .ok_or_else(|| Error::GenesisListenAddress)?;
528        let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
529        debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
530
531        let provision_options = ProvisionOptions {
532            ant_version: options.ant_version.clone(),
533            binary_option: options.current_inventory.binary_option.clone(),
534            chunk_size: None,
535            chunk_tracker_data_addresses: None,
536            chunk_tracker_services: None,
537            client_env_variables: None,
538            delayed_verifier_batch_size: None,
539            delayed_verifier_quorum_value: None,
540            disable_nodes: false,
541            enable_logging: true,
542            enable_metrics: true,
543            evm_data_payments_address: options
544                .current_inventory
545                .environment_details
546                .evm_details
547                .data_payments_address
548                .clone(),
549            evm_network: options
550                .current_inventory
551                .environment_details
552                .evm_details
553                .network
554                .clone(),
555            evm_payment_token_address: options
556                .current_inventory
557                .environment_details
558                .evm_details
559                .payment_token_address
560                .clone(),
561            evm_rpc_url: options
562                .current_inventory
563                .environment_details
564                .evm_details
565                .rpc_url
566                .clone(),
567            expected_hash: None,
568            expected_size: None,
569            file_address: None,
570            full_cone_private_node_count: 0,
571            funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
572            gas_amount: options.gas_amount,
573            interval: Some(options.interval),
574            log_format: None,
575            max_archived_log_files: options.max_archived_log_files,
576            max_log_files: options.max_log_files,
577            max_uploads: None,
578            name: options.current_inventory.name.clone(),
579            network_id: options.current_inventory.environment_details.network_id,
580            network_dashboard_branch: None,
581            node_count: 0,
582            node_env_variables: None,
583            output_inventory_dir_path: self
584                .working_directory_path
585                .join("ansible")
586                .join("inventory"),
587            peer_cache_node_count: 0,
588            performance_verifier_batch_size: None,
589            public_rpc: options.public_rpc,
590            random_verifier_batch_size: None,
591            repair_service_count: 0,
592            data_retrieval_service_count: 0,
593            rewards_address: options
594                .current_inventory
595                .environment_details
596                .rewards_address
597                .clone(),
598            scan_frequency: None,
599            single_node_payment: false,
600            sleep_duration: None,
601            start_chunk_trackers: false,
602            start_data_retrieval: false,
603            start_delayed_verifier: options.start_delayed_verifier,
604            start_random_verifier: options.start_random_verifier,
605            start_performance_verifier: options.start_performance_verifier,
606            start_uploaders: false,
607            symmetric_private_node_count: 0,
608            token_amount: options.token_amount,
609            uploaders_count: options.desired_uploaders_count,
610            upload_batch_size: None,
611            upload_size: None,
612            upload_interval: None,
613            upnp_private_node_count: 0,
614            port_restricted_cone_private_node_count: 0,
615            wallet_secret_keys: None,
616        };
617
618        self.wait_for_ssh_availability_on_new_machines(
619            AnsibleInventoryType::Clients,
620            &options.current_inventory,
621        )?;
622        self.ansible_provisioner
623            .print_ansible_run_banner("Provision Clients");
624        self.ansible_provisioner
625            .provision_uploaders(
626                &provision_options,
627                Some(initial_multiaddr),
628                Some(initial_network_contacts_url),
629            )
630            .await
631            .map_err(|err| {
632                println!("Failed to provision clients {err:?}");
633                err
634            })?;
635
636        Ok(())
637    }
638
639    fn wait_for_ssh_availability_on_new_machines(
640        &self,
641        inventory_type: AnsibleInventoryType,
642        current_inventory: &DeploymentInventory,
643    ) -> Result<()> {
644        let inventory = self
645            .ansible_provisioner
646            .ansible_runner
647            .get_inventory(inventory_type, true)?;
648        let old_set: HashSet<_> = match inventory_type {
649            AnsibleInventoryType::Clients => current_inventory
650                .client_vms
651                .iter()
652                .map(|client_vm| &client_vm.vm)
653                .cloned()
654                .collect(),
655            AnsibleInventoryType::PeerCacheNodes => current_inventory
656                .peer_cache_node_vms
657                .iter()
658                .map(|node_vm| &node_vm.vm)
659                .cloned()
660                .collect(),
661            AnsibleInventoryType::Nodes => current_inventory
662                .node_vms
663                .iter()
664                .map(|node_vm| &node_vm.vm)
665                .cloned()
666                .collect(),
667            AnsibleInventoryType::FullConeNatGateway => current_inventory
668                .full_cone_nat_gateway_vms
669                .iter()
670                .cloned()
671                .collect(),
672            AnsibleInventoryType::SymmetricNatGateway => current_inventory
673                .symmetric_nat_gateway_vms
674                .iter()
675                .cloned()
676                .collect(),
677            AnsibleInventoryType::FullConePrivateNodes => current_inventory
678                .full_cone_private_node_vms
679                .iter()
680                .map(|node_vm| &node_vm.vm)
681                .cloned()
682                .collect(),
683            AnsibleInventoryType::SymmetricPrivateNodes => current_inventory
684                .symmetric_private_node_vms
685                .iter()
686                .map(|node_vm| &node_vm.vm)
687                .cloned()
688                .collect(),
689            it => return Err(Error::UpscaleInventoryTypeNotSupported(it.to_string())),
690        };
691        let new_vms: Vec<_> = inventory
692            .into_iter()
693            .filter(|item| !old_set.contains(item))
694            .collect();
695        for vm in new_vms.iter() {
696            self.ssh_client.wait_for_ssh_availability(
697                &vm.public_ip_addr,
698                &self.cloud_provider.get_ssh_user(),
699            )?;
700        }
701        Ok(())
702    }
703}
sn_testnet_deploy/upscale.rs

sn_testnet_deploy/
upscale.rs
