1use crate::{
8 ansible::{
9 inventory::AnsibleInventoryType,
10 provisioning::{PrivateNodeProvisionInventory, ProvisionOptions},
11 },
12 error::{Error, Result},
13 get_anvil_node_data, get_bootstrap_cache_url, get_genesis_multiaddr, get_multiaddr,
14 DeploymentInventory, DeploymentType, EvmNetwork, InfraRunOptions, NodeType, TestnetDeployer,
15};
16use colored::Colorize;
17use evmlib::common::U256;
18use log::debug;
19use std::{collections::HashSet, time::Duration};
20
21#[derive(Clone)]
22pub struct UpscaleOptions {
23 pub ansible_verbose: bool,
24 pub ant_version: Option<String>,
25 pub current_inventory: DeploymentInventory,
26 pub desired_client_vm_count: Option<u16>,
27 pub desired_full_cone_private_node_count: Option<u16>,
28 pub desired_full_cone_private_node_vm_count: Option<u16>,
29 pub desired_node_count: Option<u16>,
30 pub desired_node_vm_count: Option<u16>,
31 pub desired_peer_cache_node_count: Option<u16>,
32 pub desired_peer_cache_node_vm_count: Option<u16>,
33 pub desired_symmetric_private_node_count: Option<u16>,
34 pub desired_symmetric_private_node_vm_count: Option<u16>,
35 pub desired_uploaders_count: Option<u16>,
36 pub enable_delayed_verifier: bool,
37 pub enable_random_verifier: bool,
38 pub enable_performance_verifier: bool,
39 pub funding_wallet_secret_key: Option<String>,
40 pub gas_amount: Option<U256>,
41 pub interval: Duration,
42 pub infra_only: bool,
43 pub max_archived_log_files: u16,
44 pub max_log_files: u16,
45 pub network_dashboard_branch: Option<String>,
46 pub node_env_variables: Option<Vec<(String, String)>>,
47 pub plan: bool,
48 pub public_rpc: bool,
49 pub provision_only: bool,
50 pub token_amount: Option<U256>,
51}
52
53impl TestnetDeployer {
54 pub async fn upscale(&self, options: &UpscaleOptions) -> Result<()> {
55 let is_bootstrap_deploy = matches!(
56 options
57 .current_inventory
58 .environment_details
59 .deployment_type,
60 DeploymentType::Bootstrap
61 );
62
63 if is_bootstrap_deploy
64 && (options.desired_peer_cache_node_count.is_some()
65 || options.desired_peer_cache_node_vm_count.is_some()
66 || options.desired_client_vm_count.is_some())
67 {
68 return Err(Error::InvalidUpscaleOptionsForBootstrapDeployment);
69 }
70
71 let desired_peer_cache_node_vm_count = options
72 .desired_peer_cache_node_vm_count
73 .unwrap_or(options.current_inventory.peer_cache_node_vms.len() as u16);
74 if desired_peer_cache_node_vm_count
75 < options.current_inventory.peer_cache_node_vms.len() as u16
76 {
77 return Err(Error::InvalidUpscaleDesiredPeerCacheVmCount);
78 }
79 debug!("Using {desired_peer_cache_node_vm_count} for desired Peer Cache node VM count");
80
81 let desired_node_vm_count = options
82 .desired_node_vm_count
83 .unwrap_or(options.current_inventory.node_vms.len() as u16);
84 if desired_node_vm_count < options.current_inventory.node_vms.len() as u16 {
85 return Err(Error::InvalidUpscaleDesiredNodeVmCount);
86 }
87 debug!("Using {desired_node_vm_count} for desired node VM count");
88
89 let desired_full_cone_private_node_vm_count = options
90 .desired_full_cone_private_node_vm_count
91 .unwrap_or(options.current_inventory.full_cone_private_node_vms.len() as u16);
92 if desired_full_cone_private_node_vm_count
93 < options.current_inventory.full_cone_private_node_vms.len() as u16
94 {
95 return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeVmCount);
96 }
97 debug!("Using {desired_full_cone_private_node_vm_count} for desired full cone private node VM count");
98
99 let desired_symmetric_private_node_vm_count = options
100 .desired_symmetric_private_node_vm_count
101 .unwrap_or(options.current_inventory.symmetric_private_node_vms.len() as u16);
102 if desired_symmetric_private_node_vm_count
103 < options.current_inventory.symmetric_private_node_vms.len() as u16
104 {
105 return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeVmCount);
106 }
107 debug!("Using {desired_symmetric_private_node_vm_count} for desired full cone private node VM count");
108
109 let desired_client_vm_count = options
110 .desired_client_vm_count
111 .unwrap_or(options.current_inventory.client_vms.len() as u16);
112 if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
113 return Err(Error::InvalidUpscaleDesiredClientVmCount);
114 }
115 debug!("Using {desired_client_vm_count} for desired Client VM count");
116
117 let desired_peer_cache_node_count = options
118 .desired_peer_cache_node_count
119 .unwrap_or(options.current_inventory.peer_cache_node_count() as u16);
120 if desired_peer_cache_node_count < options.current_inventory.peer_cache_node_count() as u16
121 {
122 return Err(Error::InvalidUpscaleDesiredPeerCacheNodeCount);
123 }
124 debug!("Using {desired_peer_cache_node_count} for desired peer cache node count");
125
126 let desired_node_count = options
127 .desired_node_count
128 .unwrap_or(options.current_inventory.node_count() as u16);
129 if desired_node_count < options.current_inventory.node_count() as u16 {
130 return Err(Error::InvalidUpscaleDesiredNodeCount);
131 }
132 debug!("Using {desired_node_count} for desired node count");
133
134 let desired_full_cone_private_node_count = options
135 .desired_full_cone_private_node_count
136 .unwrap_or(options.current_inventory.full_cone_private_node_count() as u16);
137 if desired_full_cone_private_node_count
138 < options.current_inventory.full_cone_private_node_count() as u16
139 {
140 return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeCount);
141 }
142 debug!(
143 "Using {desired_full_cone_private_node_count} for desired full cone private node count"
144 );
145
146 let desired_symmetric_private_node_count = options
147 .desired_symmetric_private_node_count
148 .unwrap_or(options.current_inventory.symmetric_private_node_count() as u16);
149 if desired_symmetric_private_node_count
150 < options.current_inventory.symmetric_private_node_count() as u16
151 {
152 return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeCount);
153 }
154 debug!(
155 "Using {desired_symmetric_private_node_count} for desired symmetric private node count"
156 );
157
158 let mut infra_run_options = InfraRunOptions::generate_existing(
159 &options.current_inventory.name,
160 &options.current_inventory.environment_details.region,
161 &self.terraform_runner,
162 Some(&options.current_inventory.environment_details),
163 )
164 .await?;
165 infra_run_options.peer_cache_node_vm_count = Some(desired_peer_cache_node_vm_count);
166 infra_run_options.node_vm_count = Some(desired_node_vm_count);
167 infra_run_options.full_cone_private_node_vm_count =
168 Some(desired_full_cone_private_node_vm_count);
169 infra_run_options.symmetric_private_node_vm_count =
170 Some(desired_symmetric_private_node_vm_count);
171 infra_run_options.client_vm_count = Some(desired_client_vm_count);
172
173 if options.plan {
174 self.plan(&infra_run_options)?;
175 return Ok(());
176 }
177
178 self.create_or_update_infra(&infra_run_options)
179 .map_err(|err| {
180 println!("Failed to create infra {err:?}");
181 err
182 })?;
183
184 if options.infra_only {
185 return Ok(());
186 }
187
188 let mut provision_options = ProvisionOptions {
189 ant_version: options.ant_version.clone(),
190 binary_option: options.current_inventory.binary_option.clone(),
191 chunk_size: None,
192 client_env_variables: None,
193 delayed_verifier_batch_size: None,
194 delayed_verifier_quorum_value: None,
195 enable_delayed_verifier: options.enable_delayed_verifier,
196 enable_performance_verifier: options.enable_performance_verifier,
197 enable_random_verifier: options.enable_random_verifier,
198 enable_telegraf: true,
199 enable_uploaders: true,
200 evm_data_payments_address: options
201 .current_inventory
202 .environment_details
203 .evm_details
204 .data_payments_address
205 .clone(),
206 evm_network: options
207 .current_inventory
208 .environment_details
209 .evm_details
210 .network
211 .clone(),
212 evm_payment_token_address: options
213 .current_inventory
214 .environment_details
215 .evm_details
216 .payment_token_address
217 .clone(),
218 evm_rpc_url: options
219 .current_inventory
220 .environment_details
221 .evm_details
222 .rpc_url
223 .clone(),
224 expected_hash: None,
225 expected_size: None,
226 file_address: None,
227 full_cone_private_node_count: desired_full_cone_private_node_count,
228 funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
229 gas_amount: options.gas_amount,
230 interval: Some(options.interval),
231 log_format: None,
232 max_archived_log_files: options.max_archived_log_files,
233 max_log_files: options.max_log_files,
234 max_uploads: None,
235 name: options.current_inventory.name.clone(),
236 network_id: options.current_inventory.environment_details.network_id,
237 network_dashboard_branch: None,
238 node_count: desired_node_count,
239 node_env_variables: options.node_env_variables.clone(),
240 output_inventory_dir_path: self
241 .working_directory_path
242 .join("ansible")
243 .join("inventory"),
244 peer_cache_node_count: desired_peer_cache_node_count,
245 performance_verifier_batch_size: None,
246 public_rpc: options.public_rpc,
247 random_verifier_batch_size: None,
248 rewards_address: options
249 .current_inventory
250 .environment_details
251 .rewards_address
252 .clone(),
253 sleep_duration: None,
254 symmetric_private_node_count: desired_symmetric_private_node_count,
255 token_amount: None,
256 upload_size: None,
257 uploaders_count: options.desired_uploaders_count,
258 upload_interval: None,
259 wallet_secret_keys: None,
260 };
261 let mut node_provision_failed = false;
262
263 let (initial_multiaddr, initial_ip_addr) = if is_bootstrap_deploy {
264 get_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client).map_err(
265 |err| {
266 println!("Failed to get node multiaddr {err:?}");
267 err
268 },
269 )?
270 } else {
271 get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
272 .map_err(|err| {
273 println!("Failed to get genesis multiaddr {err:?}");
274 err
275 })?
276 };
277 let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
278 debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
279
280 if !is_bootstrap_deploy {
281 self.wait_for_ssh_availability_on_new_machines(
282 AnsibleInventoryType::PeerCacheNodes,
283 &options.current_inventory,
284 )?;
285 self.ansible_provisioner
286 .print_ansible_run_banner("Provision Peer Cache Nodes");
287 match self.ansible_provisioner.provision_nodes(
288 &provision_options,
289 Some(initial_multiaddr.clone()),
290 Some(initial_network_contacts_url.clone()),
291 NodeType::PeerCache,
292 ) {
293 Ok(()) => {
294 println!("Provisioned Peer Cache nodes");
295 }
296 Err(err) => {
297 log::error!("Failed to provision Peer Cache nodes: {err}");
298 node_provision_failed = true;
299 }
300 }
301 }
302
303 self.wait_for_ssh_availability_on_new_machines(
304 AnsibleInventoryType::Nodes,
305 &options.current_inventory,
306 )?;
307 self.ansible_provisioner
308 .print_ansible_run_banner("Provision Normal Nodes");
309 match self.ansible_provisioner.provision_nodes(
310 &provision_options,
311 Some(initial_multiaddr.clone()),
312 Some(initial_network_contacts_url.clone()),
313 NodeType::Generic,
314 ) {
315 Ok(()) => {
316 println!("Provisioned normal nodes");
317 }
318 Err(err) => {
319 log::error!("Failed to provision normal nodes: {err}");
320 node_provision_failed = true;
321 }
322 }
323
324 let private_node_inventory = PrivateNodeProvisionInventory::new(
325 &self.ansible_provisioner,
326 Some(desired_full_cone_private_node_vm_count),
327 Some(desired_symmetric_private_node_vm_count),
328 )?;
329
330 if private_node_inventory.should_provision_full_cone_private_nodes() {
331 let full_cone_nat_gateway_inventory = self
332 .ansible_provisioner
333 .ansible_runner
334 .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)?;
335
336 let full_cone_nat_gateway_new_vms: Vec<_> = full_cone_nat_gateway_inventory
337 .into_iter()
338 .filter(|item| {
339 !options
340 .current_inventory
341 .full_cone_nat_gateway_vms
342 .contains(item)
343 })
344 .collect();
345
346 for vm in full_cone_nat_gateway_new_vms.iter() {
347 self.ssh_client.wait_for_ssh_availability(
348 &vm.public_ip_addr,
349 &self.cloud_provider.get_ssh_user(),
350 )?;
351 }
352
353 let full_cone_nat_gateway_new_vms = if full_cone_nat_gateway_new_vms.is_empty() {
354 None
355 } else {
356 debug!("Full Cone NAT Gateway new VMs: {full_cone_nat_gateway_new_vms:?}");
357 Some(full_cone_nat_gateway_new_vms)
358 };
359
360 match self.ansible_provisioner.provision_full_cone(
361 &provision_options,
362 Some(initial_multiaddr.clone()),
363 Some(initial_network_contacts_url.clone()),
364 private_node_inventory.clone(),
365 full_cone_nat_gateway_new_vms,
366 ) {
367 Ok(()) => {
368 println!("Provisioned Full Cone nodes and Gateway");
369 }
370 Err(err) => {
371 log::error!("Failed to provision Full Cone nodes and Gateway: {err}");
372 node_provision_failed = true;
373 }
374 }
375 }
376
377 if private_node_inventory.should_provision_symmetric_private_nodes() {
378 self.wait_for_ssh_availability_on_new_machines(
379 AnsibleInventoryType::SymmetricNatGateway,
380 &options.current_inventory,
381 )?;
382 self.ansible_provisioner
383 .print_ansible_run_banner("Provision Symmetric NAT Gateway");
384 self.ansible_provisioner
385 .provision_symmetric_nat_gateway(&provision_options, &private_node_inventory)
386 .map_err(|err| {
387 println!("Failed to provision symmetric NAT gateway {err:?}");
388 err
389 })?;
390
391 self.wait_for_ssh_availability_on_new_machines(
392 AnsibleInventoryType::SymmetricPrivateNodes,
393 &options.current_inventory,
394 )?;
395 self.ansible_provisioner
396 .print_ansible_run_banner("Provision Symmetric Private Nodes");
397 match self.ansible_provisioner.provision_symmetric_private_nodes(
398 &mut provision_options,
399 Some(initial_multiaddr.clone()),
400 Some(initial_network_contacts_url.clone()),
401 &private_node_inventory,
402 ) {
403 Ok(()) => {
404 println!("Provisioned symmetric private nodes");
405 }
406 Err(err) => {
407 log::error!("Failed to provision symmetric private nodes: {err}");
408 node_provision_failed = true;
409 }
410 }
411 }
412
413 let should_provision_uploaders =
414 options.desired_uploaders_count.is_some() || options.desired_client_vm_count.is_some();
415 if should_provision_uploaders {
416 if provision_options.evm_network == EvmNetwork::Anvil {
418 let anvil_node_data =
419 get_anvil_node_data(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
420 .map_err(|err| {
421 println!("Failed to get evm testnet data {err:?}");
422 err
423 })?;
424
425 provision_options.funding_wallet_secret_key =
426 Some(anvil_node_data.deployer_wallet_private_key);
427 }
428
429 self.wait_for_ssh_availability_on_new_machines(
430 AnsibleInventoryType::Clients,
431 &options.current_inventory,
432 )?;
433 let genesis_network_contacts = get_bootstrap_cache_url(&initial_ip_addr);
434 self.ansible_provisioner
435 .print_ansible_run_banner("Provision Clients");
436 self.ansible_provisioner
437 .provision_clients(
438 &provision_options,
439 Some(initial_multiaddr.clone()),
440 Some(genesis_network_contacts.clone()),
441 )
442 .await
443 .map_err(|err| {
444 println!("Failed to provision Clients {err:?}");
445 err
446 })?;
447 }
448
449 if node_provision_failed {
450 println!();
451 println!("{}", "WARNING!".yellow());
452 println!("Some nodes failed to provision without error.");
453 println!("This usually means a small number of nodes failed to start on a few VMs.");
454 println!("However, most of the time the deployment will still be usable.");
455 println!("See the output from Ansible to determine which VMs had failures.");
456 }
457
458 Ok(())
459 }
460
461 pub async fn upscale_clients(&self, options: &UpscaleOptions) -> Result<()> {
462 let is_bootstrap_deploy = matches!(
463 options
464 .current_inventory
465 .environment_details
466 .deployment_type,
467 DeploymentType::Bootstrap
468 );
469
470 if is_bootstrap_deploy {
471 return Err(Error::InvalidClientUpscaleDeploymentType(
472 "bootstrap".to_string(),
473 ));
474 }
475
476 let desired_client_vm_count = options
477 .desired_client_vm_count
478 .unwrap_or(options.current_inventory.client_vms.len() as u16);
479 if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
480 return Err(Error::InvalidUpscaleDesiredClientVmCount);
481 }
482 debug!("Using {desired_client_vm_count} for desired Client VM count");
483
484 let mut infra_run_options = InfraRunOptions::generate_existing(
485 &options.current_inventory.name,
486 &options.current_inventory.environment_details.region,
487 &self.terraform_runner,
488 Some(&options.current_inventory.environment_details),
489 )
490 .await?;
491 infra_run_options.client_vm_count = Some(desired_client_vm_count);
492
493 if options.plan {
494 self.plan(&infra_run_options)?;
495 return Ok(());
496 }
497
498 if !options.provision_only {
499 self.create_or_update_infra(&infra_run_options)
500 .map_err(|err| {
501 println!("Failed to create infra {err:?}");
502 err
503 })?;
504 }
505
506 if options.infra_only {
507 return Ok(());
508 }
509
510 let (initial_multiaddr, initial_ip_addr) =
511 get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
512 .map_err(|err| {
513 println!("Failed to get genesis multiaddr {err:?}");
514 err
515 })?;
516 let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
517 debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
518
519 let provision_options = ProvisionOptions {
520 ant_version: options.ant_version.clone(),
521 binary_option: options.current_inventory.binary_option.clone(),
522 chunk_size: None,
523 client_env_variables: None,
524 delayed_verifier_batch_size: None,
525 delayed_verifier_quorum_value: None,
526 enable_delayed_verifier: options.enable_delayed_verifier,
527 enable_random_verifier: options.enable_random_verifier,
528 enable_performance_verifier: options.enable_performance_verifier,
529 enable_telegraf: true,
530 enable_uploaders: true,
531 evm_data_payments_address: options
532 .current_inventory
533 .environment_details
534 .evm_details
535 .data_payments_address
536 .clone(),
537 evm_network: options
538 .current_inventory
539 .environment_details
540 .evm_details
541 .network
542 .clone(),
543 evm_payment_token_address: options
544 .current_inventory
545 .environment_details
546 .evm_details
547 .payment_token_address
548 .clone(),
549 evm_rpc_url: options
550 .current_inventory
551 .environment_details
552 .evm_details
553 .rpc_url
554 .clone(),
555 expected_hash: None,
556 expected_size: None,
557 file_address: None,
558 full_cone_private_node_count: 0,
559 funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
560 gas_amount: options.gas_amount,
561 interval: Some(options.interval),
562 log_format: None,
563 max_archived_log_files: options.max_archived_log_files,
564 max_log_files: options.max_log_files,
565 max_uploads: None,
566 name: options.current_inventory.name.clone(),
567 network_id: options.current_inventory.environment_details.network_id,
568 network_dashboard_branch: None,
569 node_count: 0,
570 node_env_variables: None,
571 output_inventory_dir_path: self
572 .working_directory_path
573 .join("ansible")
574 .join("inventory"),
575 peer_cache_node_count: 0,
576 performance_verifier_batch_size: None,
577 public_rpc: options.public_rpc,
578 random_verifier_batch_size: None,
579 rewards_address: options
580 .current_inventory
581 .environment_details
582 .rewards_address
583 .clone(),
584 sleep_duration: None,
585 symmetric_private_node_count: 0,
586 token_amount: options.token_amount,
587 uploaders_count: options.desired_uploaders_count,
588 upload_size: None,
589 upload_interval: None,
590 wallet_secret_keys: None,
591 };
592
593 self.wait_for_ssh_availability_on_new_machines(
594 AnsibleInventoryType::Clients,
595 &options.current_inventory,
596 )?;
597 self.ansible_provisioner
598 .print_ansible_run_banner("Provision Clients");
599 self.ansible_provisioner
600 .provision_clients(
601 &provision_options,
602 Some(initial_multiaddr),
603 Some(initial_network_contacts_url),
604 )
605 .await
606 .map_err(|err| {
607 println!("Failed to provision clients {err:?}");
608 err
609 })?;
610
611 Ok(())
612 }
613
614 fn wait_for_ssh_availability_on_new_machines(
615 &self,
616 inventory_type: AnsibleInventoryType,
617 current_inventory: &DeploymentInventory,
618 ) -> Result<()> {
619 let inventory = self
620 .ansible_provisioner
621 .ansible_runner
622 .get_inventory(inventory_type, true)?;
623 let old_set: HashSet<_> = match inventory_type {
624 AnsibleInventoryType::Clients => current_inventory
625 .client_vms
626 .iter()
627 .map(|client_vm| &client_vm.vm)
628 .cloned()
629 .collect(),
630 AnsibleInventoryType::PeerCacheNodes => current_inventory
631 .peer_cache_node_vms
632 .iter()
633 .map(|node_vm| &node_vm.vm)
634 .cloned()
635 .collect(),
636 AnsibleInventoryType::Nodes => current_inventory
637 .node_vms
638 .iter()
639 .map(|node_vm| &node_vm.vm)
640 .cloned()
641 .collect(),
642 AnsibleInventoryType::FullConeNatGateway => current_inventory
643 .full_cone_nat_gateway_vms
644 .iter()
645 .cloned()
646 .collect(),
647 AnsibleInventoryType::SymmetricNatGateway => current_inventory
648 .symmetric_nat_gateway_vms
649 .iter()
650 .cloned()
651 .collect(),
652 AnsibleInventoryType::FullConePrivateNodes => current_inventory
653 .full_cone_private_node_vms
654 .iter()
655 .map(|node_vm| &node_vm.vm)
656 .cloned()
657 .collect(),
658 AnsibleInventoryType::SymmetricPrivateNodes => current_inventory
659 .symmetric_private_node_vms
660 .iter()
661 .map(|node_vm| &node_vm.vm)
662 .cloned()
663 .collect(),
664 it => return Err(Error::UpscaleInventoryTypeNotSupported(it.to_string())),
665 };
666 let new_vms: Vec<_> = inventory
667 .into_iter()
668 .filter(|item| !old_set.contains(item))
669 .collect();
670 for vm in new_vms.iter() {
671 self.ssh_client.wait_for_ssh_availability(
672 &vm.public_ip_addr,
673 &self.cloud_provider.get_ssh_user(),
674 )?;
675 }
676 Ok(())
677 }
678}