1use crate::{
8 ansible::{
9 inventory::AnsibleInventoryType,
10 provisioning::{PrivateNodeProvisionInventory, ProvisionOptions},
11 },
12 error::{Error, Result},
13 get_anvil_node_data, get_bootstrap_cache_url, get_genesis_multiaddr, get_multiaddr,
14 DeploymentInventory, DeploymentType, EvmNetwork, InfraRunOptions, NodeType, TestnetDeployer,
15};
16use colored::Colorize;
17use evmlib::common::U256;
18use log::debug;
19use std::{collections::HashSet, time::Duration};
20
21#[derive(Clone)]
22pub struct UpscaleOptions {
23 pub ansible_verbose: bool,
24 pub ant_version: Option<String>,
25 pub current_inventory: DeploymentInventory,
26 pub desired_client_vm_count: Option<u16>,
27 pub desired_full_cone_private_node_count: Option<u16>,
28 pub desired_full_cone_private_node_vm_count: Option<u16>,
29 pub desired_node_count: Option<u16>,
30 pub desired_node_vm_count: Option<u16>,
31 pub desired_peer_cache_node_count: Option<u16>,
32 pub desired_peer_cache_node_vm_count: Option<u16>,
33 pub desired_symmetric_private_node_count: Option<u16>,
34 pub desired_symmetric_private_node_vm_count: Option<u16>,
35 pub desired_uploaders_count: Option<u16>,
36 pub enable_delayed_verifier: bool,
37 pub enable_random_verifier: bool,
38 pub enable_performance_verifier: bool,
39 pub funding_wallet_secret_key: Option<String>,
40 pub gas_amount: Option<U256>,
41 pub interval: Duration,
42 pub infra_only: bool,
43 pub max_archived_log_files: u16,
44 pub max_log_files: u16,
45 pub network_dashboard_branch: Option<String>,
46 pub node_env_variables: Option<Vec<(String, String)>>,
47 pub plan: bool,
48 pub public_rpc: bool,
49 pub provision_only: bool,
50 pub token_amount: Option<U256>,
51}
52
53impl TestnetDeployer {
54 pub async fn upscale(&self, options: &UpscaleOptions) -> Result<()> {
55 let is_bootstrap_deploy = matches!(
56 options
57 .current_inventory
58 .environment_details
59 .deployment_type,
60 DeploymentType::Bootstrap
61 );
62
63 if is_bootstrap_deploy
64 && (options.desired_peer_cache_node_count.is_some()
65 || options.desired_peer_cache_node_vm_count.is_some()
66 || options.desired_client_vm_count.is_some())
67 {
68 return Err(Error::InvalidUpscaleOptionsForBootstrapDeployment);
69 }
70
71 let desired_peer_cache_node_vm_count = options
72 .desired_peer_cache_node_vm_count
73 .unwrap_or(options.current_inventory.peer_cache_node_vms.len() as u16);
74 if desired_peer_cache_node_vm_count
75 < options.current_inventory.peer_cache_node_vms.len() as u16
76 {
77 return Err(Error::InvalidUpscaleDesiredPeerCacheVmCount);
78 }
79 debug!("Using {desired_peer_cache_node_vm_count} for desired Peer Cache node VM count");
80
81 let desired_node_vm_count = options
82 .desired_node_vm_count
83 .unwrap_or(options.current_inventory.node_vms.len() as u16);
84 if desired_node_vm_count < options.current_inventory.node_vms.len() as u16 {
85 return Err(Error::InvalidUpscaleDesiredNodeVmCount);
86 }
87 debug!("Using {desired_node_vm_count} for desired node VM count");
88
89 let desired_full_cone_private_node_vm_count = options
90 .desired_full_cone_private_node_vm_count
91 .unwrap_or(options.current_inventory.full_cone_private_node_vms.len() as u16);
92 if desired_full_cone_private_node_vm_count
93 < options.current_inventory.full_cone_private_node_vms.len() as u16
94 {
95 return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeVmCount);
96 }
97 debug!("Using {desired_full_cone_private_node_vm_count} for desired full cone private node VM count");
98
99 let desired_symmetric_private_node_vm_count = options
100 .desired_symmetric_private_node_vm_count
101 .unwrap_or(options.current_inventory.symmetric_private_node_vms.len() as u16);
102 if desired_symmetric_private_node_vm_count
103 < options.current_inventory.symmetric_private_node_vms.len() as u16
104 {
105 return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeVmCount);
106 }
107 debug!("Using {desired_symmetric_private_node_vm_count} for desired full cone private node VM count");
108
109 let desired_client_vm_count = options
110 .desired_client_vm_count
111 .unwrap_or(options.current_inventory.client_vms.len() as u16);
112 if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
113 return Err(Error::InvalidUpscaleDesiredClientVmCount);
114 }
115 debug!("Using {desired_client_vm_count} for desired Client VM count");
116
117 let desired_peer_cache_node_count = options
118 .desired_peer_cache_node_count
119 .unwrap_or(options.current_inventory.peer_cache_node_count() as u16);
120 if desired_peer_cache_node_count < options.current_inventory.peer_cache_node_count() as u16
121 {
122 return Err(Error::InvalidUpscaleDesiredPeerCacheNodeCount);
123 }
124 debug!("Using {desired_peer_cache_node_count} for desired peer cache node count");
125
126 let desired_node_count = options
127 .desired_node_count
128 .unwrap_or(options.current_inventory.node_count() as u16);
129 if desired_node_count < options.current_inventory.node_count() as u16 {
130 return Err(Error::InvalidUpscaleDesiredNodeCount);
131 }
132 debug!("Using {desired_node_count} for desired node count");
133
134 let desired_full_cone_private_node_count = options
135 .desired_full_cone_private_node_count
136 .unwrap_or(options.current_inventory.full_cone_private_node_count() as u16);
137 if desired_full_cone_private_node_count
138 < options.current_inventory.full_cone_private_node_count() as u16
139 {
140 return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeCount);
141 }
142 debug!(
143 "Using {desired_full_cone_private_node_count} for desired full cone private node count"
144 );
145
146 let desired_symmetric_private_node_count = options
147 .desired_symmetric_private_node_count
148 .unwrap_or(options.current_inventory.symmetric_private_node_count() as u16);
149 if desired_symmetric_private_node_count
150 < options.current_inventory.symmetric_private_node_count() as u16
151 {
152 return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeCount);
153 }
154 debug!(
155 "Using {desired_symmetric_private_node_count} for desired symmetric private node count"
156 );
157
158 let mut infra_run_options = InfraRunOptions::generate_existing(
159 &options.current_inventory.name,
160 &options.current_inventory.environment_details.region,
161 &self.terraform_runner,
162 Some(&options.current_inventory.environment_details),
163 )
164 .await?;
165 infra_run_options.peer_cache_node_vm_count = Some(desired_peer_cache_node_vm_count);
166 infra_run_options.node_vm_count = Some(desired_node_vm_count);
167 infra_run_options.full_cone_private_node_vm_count =
168 Some(desired_full_cone_private_node_vm_count);
169 infra_run_options.symmetric_private_node_vm_count =
170 Some(desired_symmetric_private_node_vm_count);
171 infra_run_options.client_vm_count = Some(desired_client_vm_count);
172
173 if options.plan {
174 self.plan(&infra_run_options)?;
175 return Ok(());
176 }
177
178 self.create_or_update_infra(&infra_run_options)
179 .map_err(|err| {
180 println!("Failed to create infra {err:?}");
181 err
182 })?;
183
184 if options.infra_only {
185 return Ok(());
186 }
187
188 let mut provision_options = ProvisionOptions {
189 ant_version: options.ant_version.clone(),
190 binary_option: options.current_inventory.binary_option.clone(),
191 chunk_size: None,
192 client_env_variables: None,
193 delayed_verifier_batch_size: None,
194 delayed_verifier_quorum_value: None,
195 enable_delayed_verifier: options.enable_delayed_verifier,
196 enable_performance_verifier: options.enable_performance_verifier,
197 enable_random_verifier: options.enable_random_verifier,
198 enable_telegraf: true,
199 enable_uploaders: true,
200 evm_data_payments_address: options
201 .current_inventory
202 .environment_details
203 .evm_details
204 .data_payments_address
205 .clone(),
206 evm_network: options
207 .current_inventory
208 .environment_details
209 .evm_details
210 .network
211 .clone(),
212 evm_payment_token_address: options
213 .current_inventory
214 .environment_details
215 .evm_details
216 .payment_token_address
217 .clone(),
218 evm_rpc_url: options
219 .current_inventory
220 .environment_details
221 .evm_details
222 .rpc_url
223 .clone(),
224 expected_hash: None,
225 expected_size: None,
226 file_address: None,
227 full_cone_private_node_count: desired_full_cone_private_node_count,
228 funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
229 gas_amount: options.gas_amount,
230 interval: Some(options.interval),
231 log_format: None,
232 max_archived_log_files: options.max_archived_log_files,
233 max_log_files: options.max_log_files,
234 max_uploads: None,
235 name: options.current_inventory.name.clone(),
236 network_id: options.current_inventory.environment_details.network_id,
237 network_dashboard_branch: None,
238 node_count: desired_node_count,
239 node_env_variables: options.node_env_variables.clone(),
240 output_inventory_dir_path: self
241 .working_directory_path
242 .join("ansible")
243 .join("inventory"),
244 peer_cache_node_count: desired_peer_cache_node_count,
245 performance_verifier_batch_size: None,
246 public_rpc: options.public_rpc,
247 random_verifier_batch_size: None,
248 rewards_address: options
249 .current_inventory
250 .environment_details
251 .rewards_address
252 .clone(),
253 sleep_duration: None,
254 symmetric_private_node_count: desired_symmetric_private_node_count,
255 token_amount: None,
256 upload_batch_size: None,
257 upload_size: None,
258 uploaders_count: options.desired_uploaders_count,
259 upload_interval: None,
260 upnp_private_node_count: 0,
261 wallet_secret_keys: None,
262 };
263 let mut node_provision_failed = false;
264
265 let (initial_multiaddr, initial_ip_addr) = if is_bootstrap_deploy {
266 get_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client).map_err(
267 |err| {
268 println!("Failed to get node multiaddr {err:?}");
269 err
270 },
271 )?
272 } else {
273 get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
274 .map_err(|err| {
275 println!("Failed to get genesis multiaddr {err:?}");
276 err
277 })?
278 };
279 let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
280 debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
281
282 if !is_bootstrap_deploy {
283 self.wait_for_ssh_availability_on_new_machines(
284 AnsibleInventoryType::PeerCacheNodes,
285 &options.current_inventory,
286 )?;
287 self.ansible_provisioner
288 .print_ansible_run_banner("Provision Peer Cache Nodes");
289 match self.ansible_provisioner.provision_nodes(
290 &provision_options,
291 Some(initial_multiaddr.clone()),
292 Some(initial_network_contacts_url.clone()),
293 NodeType::PeerCache,
294 ) {
295 Ok(()) => {
296 println!("Provisioned Peer Cache nodes");
297 }
298 Err(err) => {
299 log::error!("Failed to provision Peer Cache nodes: {err}");
300 node_provision_failed = true;
301 }
302 }
303 }
304
305 self.wait_for_ssh_availability_on_new_machines(
306 AnsibleInventoryType::Nodes,
307 &options.current_inventory,
308 )?;
309 self.ansible_provisioner
310 .print_ansible_run_banner("Provision Normal Nodes");
311 match self.ansible_provisioner.provision_nodes(
312 &provision_options,
313 Some(initial_multiaddr.clone()),
314 Some(initial_network_contacts_url.clone()),
315 NodeType::Generic,
316 ) {
317 Ok(()) => {
318 println!("Provisioned normal nodes");
319 }
320 Err(err) => {
321 log::error!("Failed to provision normal nodes: {err}");
322 node_provision_failed = true;
323 }
324 }
325
326 let private_node_inventory = PrivateNodeProvisionInventory::new(
327 &self.ansible_provisioner,
328 Some(desired_full_cone_private_node_vm_count),
329 Some(desired_symmetric_private_node_vm_count),
330 )?;
331
332 if private_node_inventory.should_provision_full_cone_private_nodes() {
333 let full_cone_nat_gateway_inventory = self
334 .ansible_provisioner
335 .ansible_runner
336 .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)?;
337
338 let full_cone_nat_gateway_new_vms: Vec<_> = full_cone_nat_gateway_inventory
339 .into_iter()
340 .filter(|item| {
341 !options
342 .current_inventory
343 .full_cone_nat_gateway_vms
344 .contains(item)
345 })
346 .collect();
347
348 for vm in full_cone_nat_gateway_new_vms.iter() {
349 self.ssh_client.wait_for_ssh_availability(
350 &vm.public_ip_addr,
351 &self.cloud_provider.get_ssh_user(),
352 )?;
353 }
354
355 let full_cone_nat_gateway_new_vms = if full_cone_nat_gateway_new_vms.is_empty() {
356 None
357 } else {
358 debug!("Full Cone NAT Gateway new VMs: {full_cone_nat_gateway_new_vms:?}");
359 Some(full_cone_nat_gateway_new_vms)
360 };
361
362 match self.ansible_provisioner.provision_full_cone(
363 &provision_options,
364 Some(initial_multiaddr.clone()),
365 Some(initial_network_contacts_url.clone()),
366 private_node_inventory.clone(),
367 full_cone_nat_gateway_new_vms,
368 ) {
369 Ok(()) => {
370 println!("Provisioned Full Cone nodes and Gateway");
371 }
372 Err(err) => {
373 log::error!("Failed to provision Full Cone nodes and Gateway: {err}");
374 node_provision_failed = true;
375 }
376 }
377 }
378
379 if private_node_inventory.should_provision_symmetric_private_nodes() {
380 self.wait_for_ssh_availability_on_new_machines(
381 AnsibleInventoryType::SymmetricNatGateway,
382 &options.current_inventory,
383 )?;
384 self.ansible_provisioner
385 .print_ansible_run_banner("Provision Symmetric NAT Gateway");
386 self.ansible_provisioner
387 .provision_symmetric_nat_gateway(&provision_options, &private_node_inventory)
388 .map_err(|err| {
389 println!("Failed to provision symmetric NAT gateway {err:?}");
390 err
391 })?;
392
393 self.wait_for_ssh_availability_on_new_machines(
394 AnsibleInventoryType::SymmetricPrivateNodes,
395 &options.current_inventory,
396 )?;
397 self.ansible_provisioner
398 .print_ansible_run_banner("Provision Symmetric Private Nodes");
399 match self.ansible_provisioner.provision_symmetric_private_nodes(
400 &mut provision_options,
401 Some(initial_multiaddr.clone()),
402 Some(initial_network_contacts_url.clone()),
403 &private_node_inventory,
404 ) {
405 Ok(()) => {
406 println!("Provisioned symmetric private nodes");
407 }
408 Err(err) => {
409 log::error!("Failed to provision symmetric private nodes: {err}");
410 node_provision_failed = true;
411 }
412 }
413 }
414
415 let should_provision_uploaders =
416 options.desired_uploaders_count.is_some() || options.desired_client_vm_count.is_some();
417 if should_provision_uploaders {
418 if provision_options.evm_network == EvmNetwork::Anvil {
420 let anvil_node_data =
421 get_anvil_node_data(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
422 .map_err(|err| {
423 println!("Failed to get evm testnet data {err:?}");
424 err
425 })?;
426
427 provision_options.funding_wallet_secret_key =
428 Some(anvil_node_data.deployer_wallet_private_key);
429 }
430
431 self.wait_for_ssh_availability_on_new_machines(
432 AnsibleInventoryType::Clients,
433 &options.current_inventory,
434 )?;
435 let genesis_network_contacts = get_bootstrap_cache_url(&initial_ip_addr);
436 self.ansible_provisioner
437 .print_ansible_run_banner("Provision Clients");
438 self.ansible_provisioner
439 .provision_uploaders(
440 &provision_options,
441 Some(initial_multiaddr.clone()),
442 Some(genesis_network_contacts.clone()),
443 )
444 .await
445 .map_err(|err| {
446 println!("Failed to provision Clients {err:?}");
447 err
448 })?;
449 }
450
451 if node_provision_failed {
452 println!();
453 println!("{}", "WARNING!".yellow());
454 println!("Some nodes failed to provision without error.");
455 println!("This usually means a small number of nodes failed to start on a few VMs.");
456 println!("However, most of the time the deployment will still be usable.");
457 println!("See the output from Ansible to determine which VMs had failures.");
458 }
459
460 Ok(())
461 }
462
463 pub async fn upscale_clients(&self, options: &UpscaleOptions) -> Result<()> {
464 let is_bootstrap_deploy = matches!(
465 options
466 .current_inventory
467 .environment_details
468 .deployment_type,
469 DeploymentType::Bootstrap
470 );
471
472 if is_bootstrap_deploy {
473 return Err(Error::InvalidClientUpscaleDeploymentType(
474 "bootstrap".to_string(),
475 ));
476 }
477
478 let desired_client_vm_count = options
479 .desired_client_vm_count
480 .unwrap_or(options.current_inventory.client_vms.len() as u16);
481 if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
482 return Err(Error::InvalidUpscaleDesiredClientVmCount);
483 }
484 debug!("Using {desired_client_vm_count} for desired Client VM count");
485
486 let mut infra_run_options = InfraRunOptions::generate_existing(
487 &options.current_inventory.name,
488 &options.current_inventory.environment_details.region,
489 &self.terraform_runner,
490 Some(&options.current_inventory.environment_details),
491 )
492 .await?;
493 infra_run_options.client_vm_count = Some(desired_client_vm_count);
494
495 if options.plan {
496 self.plan(&infra_run_options)?;
497 return Ok(());
498 }
499
500 if !options.provision_only {
501 self.create_or_update_infra(&infra_run_options)
502 .map_err(|err| {
503 println!("Failed to create infra {err:?}");
504 err
505 })?;
506 }
507
508 if options.infra_only {
509 return Ok(());
510 }
511
512 let (initial_multiaddr, initial_ip_addr) =
513 get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
514 .map_err(|err| {
515 println!("Failed to get genesis multiaddr {err:?}");
516 err
517 })?;
518 let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
519 debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
520
521 let provision_options = ProvisionOptions {
522 ant_version: options.ant_version.clone(),
523 binary_option: options.current_inventory.binary_option.clone(),
524 chunk_size: None,
525 client_env_variables: None,
526 delayed_verifier_batch_size: None,
527 delayed_verifier_quorum_value: None,
528 enable_delayed_verifier: options.enable_delayed_verifier,
529 enable_random_verifier: options.enable_random_verifier,
530 enable_performance_verifier: options.enable_performance_verifier,
531 enable_telegraf: true,
532 enable_uploaders: true,
533 evm_data_payments_address: options
534 .current_inventory
535 .environment_details
536 .evm_details
537 .data_payments_address
538 .clone(),
539 evm_network: options
540 .current_inventory
541 .environment_details
542 .evm_details
543 .network
544 .clone(),
545 evm_payment_token_address: options
546 .current_inventory
547 .environment_details
548 .evm_details
549 .payment_token_address
550 .clone(),
551 evm_rpc_url: options
552 .current_inventory
553 .environment_details
554 .evm_details
555 .rpc_url
556 .clone(),
557 expected_hash: None,
558 expected_size: None,
559 file_address: None,
560 full_cone_private_node_count: 0,
561 funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
562 gas_amount: options.gas_amount,
563 interval: Some(options.interval),
564 log_format: None,
565 max_archived_log_files: options.max_archived_log_files,
566 max_log_files: options.max_log_files,
567 max_uploads: None,
568 name: options.current_inventory.name.clone(),
569 network_id: options.current_inventory.environment_details.network_id,
570 network_dashboard_branch: None,
571 node_count: 0,
572 node_env_variables: None,
573 output_inventory_dir_path: self
574 .working_directory_path
575 .join("ansible")
576 .join("inventory"),
577 peer_cache_node_count: 0,
578 performance_verifier_batch_size: None,
579 public_rpc: options.public_rpc,
580 random_verifier_batch_size: None,
581 rewards_address: options
582 .current_inventory
583 .environment_details
584 .rewards_address
585 .clone(),
586 sleep_duration: None,
587 symmetric_private_node_count: 0,
588 token_amount: options.token_amount,
589 uploaders_count: options.desired_uploaders_count,
590 upload_batch_size: None,
591 upload_size: None,
592 upload_interval: None,
593 upnp_private_node_count: 0,
594 wallet_secret_keys: None,
595 };
596
597 self.wait_for_ssh_availability_on_new_machines(
598 AnsibleInventoryType::Clients,
599 &options.current_inventory,
600 )?;
601 self.ansible_provisioner
602 .print_ansible_run_banner("Provision Clients");
603 self.ansible_provisioner
604 .provision_uploaders(
605 &provision_options,
606 Some(initial_multiaddr),
607 Some(initial_network_contacts_url),
608 )
609 .await
610 .map_err(|err| {
611 println!("Failed to provision clients {err:?}");
612 err
613 })?;
614
615 Ok(())
616 }
617
618 fn wait_for_ssh_availability_on_new_machines(
619 &self,
620 inventory_type: AnsibleInventoryType,
621 current_inventory: &DeploymentInventory,
622 ) -> Result<()> {
623 let inventory = self
624 .ansible_provisioner
625 .ansible_runner
626 .get_inventory(inventory_type, true)?;
627 let old_set: HashSet<_> = match inventory_type {
628 AnsibleInventoryType::Clients => current_inventory
629 .client_vms
630 .iter()
631 .map(|client_vm| &client_vm.vm)
632 .cloned()
633 .collect(),
634 AnsibleInventoryType::PeerCacheNodes => current_inventory
635 .peer_cache_node_vms
636 .iter()
637 .map(|node_vm| &node_vm.vm)
638 .cloned()
639 .collect(),
640 AnsibleInventoryType::Nodes => current_inventory
641 .node_vms
642 .iter()
643 .map(|node_vm| &node_vm.vm)
644 .cloned()
645 .collect(),
646 AnsibleInventoryType::FullConeNatGateway => current_inventory
647 .full_cone_nat_gateway_vms
648 .iter()
649 .cloned()
650 .collect(),
651 AnsibleInventoryType::SymmetricNatGateway => current_inventory
652 .symmetric_nat_gateway_vms
653 .iter()
654 .cloned()
655 .collect(),
656 AnsibleInventoryType::FullConePrivateNodes => current_inventory
657 .full_cone_private_node_vms
658 .iter()
659 .map(|node_vm| &node_vm.vm)
660 .cloned()
661 .collect(),
662 AnsibleInventoryType::SymmetricPrivateNodes => current_inventory
663 .symmetric_private_node_vms
664 .iter()
665 .map(|node_vm| &node_vm.vm)
666 .cloned()
667 .collect(),
668 it => return Err(Error::UpscaleInventoryTypeNotSupported(it.to_string())),
669 };
670 let new_vms: Vec<_> = inventory
671 .into_iter()
672 .filter(|item| !old_set.contains(item))
673 .collect();
674 for vm in new_vms.iter() {
675 self.ssh_client.wait_for_ssh_availability(
676 &vm.public_ip_addr,
677 &self.cloud_provider.get_ssh_user(),
678 )?;
679 }
680 Ok(())
681 }
682}