1use crate::{
8 ansible::{
9 inventory::AnsibleInventoryType,
10 provisioning::{PrivateNodeProvisionInventory, ProvisionOptions},
11 },
12 error::{Error, Result},
13 get_anvil_node_data, get_bootstrap_cache_url, get_genesis_multiaddr, get_multiaddr,
14 DeploymentInventory, DeploymentType, EvmNetwork, InfraRunOptions, NodeType, TestnetDeployer,
15};
16use colored::Colorize;
17use evmlib::common::U256;
18use log::debug;
19use std::{collections::HashSet, time::Duration};
20
21#[derive(Clone)]
22pub struct UpscaleOptions {
23 pub ansible_verbose: bool,
24 pub ant_version: Option<String>,
25 pub current_inventory: DeploymentInventory,
26 pub desired_client_vm_count: Option<u16>,
27 pub desired_full_cone_private_node_count: Option<u16>,
28 pub desired_full_cone_private_node_vm_count: Option<u16>,
29 pub desired_node_count: Option<u16>,
30 pub desired_node_vm_count: Option<u16>,
31 pub desired_peer_cache_node_count: Option<u16>,
32 pub desired_peer_cache_node_vm_count: Option<u16>,
33 pub desired_symmetric_private_node_count: Option<u16>,
34 pub desired_symmetric_private_node_vm_count: Option<u16>,
35 pub desired_uploaders_count: Option<u16>,
36 pub enable_delayed_verifier: bool,
37 pub enable_random_verifier: bool,
38 pub enable_performance_verifier: bool,
39 pub funding_wallet_secret_key: Option<String>,
40 pub gas_amount: Option<U256>,
41 pub interval: Duration,
42 pub infra_only: bool,
43 pub max_archived_log_files: u16,
44 pub max_log_files: u16,
45 pub network_dashboard_branch: Option<String>,
46 pub node_env_variables: Option<Vec<(String, String)>>,
47 pub plan: bool,
48 pub public_rpc: bool,
49 pub provision_only: bool,
50 pub token_amount: Option<U256>,
51}
52
53impl TestnetDeployer {
54 pub async fn upscale(&self, options: &UpscaleOptions) -> Result<()> {
55 let is_bootstrap_deploy = matches!(
56 options
57 .current_inventory
58 .environment_details
59 .deployment_type,
60 DeploymentType::Bootstrap
61 );
62
63 if is_bootstrap_deploy
64 && (options.desired_peer_cache_node_count.is_some()
65 || options.desired_peer_cache_node_vm_count.is_some()
66 || options.desired_client_vm_count.is_some())
67 {
68 return Err(Error::InvalidUpscaleOptionsForBootstrapDeployment);
69 }
70
71 let desired_peer_cache_node_vm_count = options
72 .desired_peer_cache_node_vm_count
73 .unwrap_or(options.current_inventory.peer_cache_node_vms.len() as u16);
74 if desired_peer_cache_node_vm_count
75 < options.current_inventory.peer_cache_node_vms.len() as u16
76 {
77 return Err(Error::InvalidUpscaleDesiredPeerCacheVmCount);
78 }
79 debug!("Using {desired_peer_cache_node_vm_count} for desired Peer Cache node VM count");
80
81 let desired_node_vm_count = options
82 .desired_node_vm_count
83 .unwrap_or(options.current_inventory.node_vms.len() as u16);
84 if desired_node_vm_count < options.current_inventory.node_vms.len() as u16 {
85 return Err(Error::InvalidUpscaleDesiredNodeVmCount);
86 }
87 debug!("Using {desired_node_vm_count} for desired node VM count");
88
89 let desired_full_cone_private_node_vm_count = options
90 .desired_full_cone_private_node_vm_count
91 .unwrap_or(options.current_inventory.full_cone_private_node_vms.len() as u16);
92 if desired_full_cone_private_node_vm_count
93 < options.current_inventory.full_cone_private_node_vms.len() as u16
94 {
95 return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeVmCount);
96 }
97 debug!("Using {desired_full_cone_private_node_vm_count} for desired full cone private node VM count");
98
99 let desired_symmetric_private_node_vm_count = options
100 .desired_symmetric_private_node_vm_count
101 .unwrap_or(options.current_inventory.symmetric_private_node_vms.len() as u16);
102 if desired_symmetric_private_node_vm_count
103 < options.current_inventory.symmetric_private_node_vms.len() as u16
104 {
105 return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeVmCount);
106 }
107 debug!("Using {desired_symmetric_private_node_vm_count} for desired full cone private node VM count");
108
109 let desired_client_vm_count = options
110 .desired_client_vm_count
111 .unwrap_or(options.current_inventory.client_vms.len() as u16);
112 if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
113 return Err(Error::InvalidUpscaleDesiredClientVmCount);
114 }
115 debug!("Using {desired_client_vm_count} for desired Client VM count");
116
117 let desired_peer_cache_node_count = options
118 .desired_peer_cache_node_count
119 .unwrap_or(options.current_inventory.peer_cache_node_count() as u16);
120 if desired_peer_cache_node_count < options.current_inventory.peer_cache_node_count() as u16
121 {
122 return Err(Error::InvalidUpscaleDesiredPeerCacheNodeCount);
123 }
124 debug!("Using {desired_peer_cache_node_count} for desired peer cache node count");
125
126 let desired_node_count = options
127 .desired_node_count
128 .unwrap_or(options.current_inventory.node_count() as u16);
129 if desired_node_count < options.current_inventory.node_count() as u16 {
130 return Err(Error::InvalidUpscaleDesiredNodeCount);
131 }
132 debug!("Using {desired_node_count} for desired node count");
133
134 let desired_full_cone_private_node_count = options
135 .desired_full_cone_private_node_count
136 .unwrap_or(options.current_inventory.full_cone_private_node_count() as u16);
137 if desired_full_cone_private_node_count
138 < options.current_inventory.full_cone_private_node_count() as u16
139 {
140 return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeCount);
141 }
142 debug!(
143 "Using {desired_full_cone_private_node_count} for desired full cone private node count"
144 );
145
146 let desired_symmetric_private_node_count = options
147 .desired_symmetric_private_node_count
148 .unwrap_or(options.current_inventory.symmetric_private_node_count() as u16);
149 if desired_symmetric_private_node_count
150 < options.current_inventory.symmetric_private_node_count() as u16
151 {
152 return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeCount);
153 }
154 debug!(
155 "Using {desired_symmetric_private_node_count} for desired symmetric private node count"
156 );
157
158 let mut infra_run_options = InfraRunOptions::generate_existing(
159 &options.current_inventory.name,
160 &options.current_inventory.environment_details.region,
161 &self.terraform_runner,
162 Some(&options.current_inventory.environment_details),
163 )
164 .await?;
165 infra_run_options.peer_cache_node_vm_count = Some(desired_peer_cache_node_vm_count);
166 infra_run_options.node_vm_count = Some(desired_node_vm_count);
167 infra_run_options.full_cone_private_node_vm_count =
168 Some(desired_full_cone_private_node_vm_count);
169 infra_run_options.symmetric_private_node_vm_count =
170 Some(desired_symmetric_private_node_vm_count);
171 infra_run_options.client_vm_count = Some(desired_client_vm_count);
172
173 if options.plan {
174 self.plan(&infra_run_options)?;
175 return Ok(());
176 }
177
178 self.create_or_update_infra(&infra_run_options)
179 .map_err(|err| {
180 println!("Failed to create infra {err:?}");
181 err
182 })?;
183
184 if options.infra_only {
185 return Ok(());
186 }
187
188 let mut provision_options = ProvisionOptions {
189 ant_version: options.ant_version.clone(),
190 binary_option: options.current_inventory.binary_option.clone(),
191 chunk_size: None,
192 client_env_variables: None,
193 delayed_verifier_batch_size: None,
194 delayed_verifier_quorum_value: None,
195 enable_delayed_verifier: options.enable_delayed_verifier,
196 enable_performance_verifier: options.enable_performance_verifier,
197 enable_random_verifier: options.enable_random_verifier,
198 enable_telegraf: true,
199 enable_uploaders: true,
200 evm_data_payments_address: options
201 .current_inventory
202 .environment_details
203 .evm_details
204 .data_payments_address
205 .clone(),
206 evm_network: options
207 .current_inventory
208 .environment_details
209 .evm_details
210 .network
211 .clone(),
212 evm_payment_token_address: options
213 .current_inventory
214 .environment_details
215 .evm_details
216 .payment_token_address
217 .clone(),
218 evm_rpc_url: options
219 .current_inventory
220 .environment_details
221 .evm_details
222 .rpc_url
223 .clone(),
224 expected_hash: None,
225 expected_size: None,
226 file_address: None,
227 full_cone_private_node_count: desired_full_cone_private_node_count,
228 funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
229 gas_amount: options.gas_amount,
230 interval: Some(options.interval),
231 log_format: None,
232 max_archived_log_files: options.max_archived_log_files,
233 max_log_files: options.max_log_files,
234 max_uploads: None,
235 name: options.current_inventory.name.clone(),
236 network_id: options.current_inventory.environment_details.network_id,
237 network_dashboard_branch: None,
238 node_count: desired_node_count,
239 node_env_variables: options.node_env_variables.clone(),
240 output_inventory_dir_path: self
241 .working_directory_path
242 .join("ansible")
243 .join("inventory"),
244 peer_cache_node_count: desired_peer_cache_node_count,
245 performance_verifier_batch_size: None,
246 public_rpc: options.public_rpc,
247 random_verifier_batch_size: None,
248 rewards_address: options
249 .current_inventory
250 .environment_details
251 .rewards_address
252 .clone(),
253 sleep_duration: None,
254 symmetric_private_node_count: desired_symmetric_private_node_count,
255 token_amount: None,
256 upload_batch_size: None,
257 upload_size: None,
258 uploaders_count: options.desired_uploaders_count,
259 upload_interval: None,
260 upnp_private_node_count: 0,
261 wallet_secret_keys: None,
262 };
263 let mut node_provision_failed = false;
264
265 let (initial_multiaddr, initial_ip_addr) = if is_bootstrap_deploy {
266 get_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client).map_err(
267 |err| {
268 println!("Failed to get node multiaddr {err:?}");
269 err
270 },
271 )?
272 } else {
273 get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
274 .map_err(|err| {
275 println!("Failed to get genesis multiaddr {err:?}");
276 err
277 })?
278 .ok_or_else(|| Error::GenesisListenAddress)?
279 };
280 let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
281 debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
282
283 if !is_bootstrap_deploy {
284 self.wait_for_ssh_availability_on_new_machines(
285 AnsibleInventoryType::PeerCacheNodes,
286 &options.current_inventory,
287 )?;
288 self.ansible_provisioner
289 .print_ansible_run_banner("Provision Peer Cache Nodes");
290 match self.ansible_provisioner.provision_nodes(
291 &provision_options,
292 Some(initial_multiaddr.clone()),
293 Some(initial_network_contacts_url.clone()),
294 NodeType::PeerCache,
295 ) {
296 Ok(()) => {
297 println!("Provisioned Peer Cache nodes");
298 }
299 Err(err) => {
300 log::error!("Failed to provision Peer Cache nodes: {err}");
301 node_provision_failed = true;
302 }
303 }
304 }
305
306 self.wait_for_ssh_availability_on_new_machines(
307 AnsibleInventoryType::Nodes,
308 &options.current_inventory,
309 )?;
310 self.ansible_provisioner
311 .print_ansible_run_banner("Provision Normal Nodes");
312 match self.ansible_provisioner.provision_nodes(
313 &provision_options,
314 Some(initial_multiaddr.clone()),
315 Some(initial_network_contacts_url.clone()),
316 NodeType::Generic,
317 ) {
318 Ok(()) => {
319 println!("Provisioned normal nodes");
320 }
321 Err(err) => {
322 log::error!("Failed to provision normal nodes: {err}");
323 node_provision_failed = true;
324 }
325 }
326
327 let private_node_inventory = PrivateNodeProvisionInventory::new(
328 &self.ansible_provisioner,
329 Some(desired_full_cone_private_node_vm_count),
330 Some(desired_symmetric_private_node_vm_count),
331 )?;
332
333 if private_node_inventory.should_provision_full_cone_private_nodes() {
334 let full_cone_nat_gateway_inventory = self
335 .ansible_provisioner
336 .ansible_runner
337 .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)?;
338
339 let full_cone_nat_gateway_new_vms: Vec<_> = full_cone_nat_gateway_inventory
340 .into_iter()
341 .filter(|item| {
342 !options
343 .current_inventory
344 .full_cone_nat_gateway_vms
345 .contains(item)
346 })
347 .collect();
348
349 for vm in full_cone_nat_gateway_new_vms.iter() {
350 self.ssh_client.wait_for_ssh_availability(
351 &vm.public_ip_addr,
352 &self.cloud_provider.get_ssh_user(),
353 )?;
354 }
355
356 let full_cone_nat_gateway_new_vms = if full_cone_nat_gateway_new_vms.is_empty() {
357 None
358 } else {
359 debug!("Full Cone NAT Gateway new VMs: {full_cone_nat_gateway_new_vms:?}");
360 Some(full_cone_nat_gateway_new_vms)
361 };
362
363 match self.ansible_provisioner.provision_full_cone(
364 &provision_options,
365 Some(initial_multiaddr.clone()),
366 Some(initial_network_contacts_url.clone()),
367 private_node_inventory.clone(),
368 full_cone_nat_gateway_new_vms,
369 ) {
370 Ok(()) => {
371 println!("Provisioned Full Cone nodes and Gateway");
372 }
373 Err(err) => {
374 log::error!("Failed to provision Full Cone nodes and Gateway: {err}");
375 node_provision_failed = true;
376 }
377 }
378 }
379
380 if private_node_inventory.should_provision_symmetric_private_nodes() {
381 self.wait_for_ssh_availability_on_new_machines(
382 AnsibleInventoryType::SymmetricNatGateway,
383 &options.current_inventory,
384 )?;
385 self.ansible_provisioner
386 .print_ansible_run_banner("Provision Symmetric NAT Gateway");
387 self.ansible_provisioner
388 .provision_symmetric_nat_gateway(&provision_options, &private_node_inventory)
389 .map_err(|err| {
390 println!("Failed to provision symmetric NAT gateway {err:?}");
391 err
392 })?;
393
394 self.wait_for_ssh_availability_on_new_machines(
395 AnsibleInventoryType::SymmetricPrivateNodes,
396 &options.current_inventory,
397 )?;
398 self.ansible_provisioner
399 .print_ansible_run_banner("Provision Symmetric Private Nodes");
400 match self.ansible_provisioner.provision_symmetric_private_nodes(
401 &mut provision_options,
402 Some(initial_multiaddr.clone()),
403 Some(initial_network_contacts_url.clone()),
404 &private_node_inventory,
405 ) {
406 Ok(()) => {
407 println!("Provisioned symmetric private nodes");
408 }
409 Err(err) => {
410 log::error!("Failed to provision symmetric private nodes: {err}");
411 node_provision_failed = true;
412 }
413 }
414 }
415
416 let should_provision_uploaders =
417 options.desired_uploaders_count.is_some() || options.desired_client_vm_count.is_some();
418 if should_provision_uploaders {
419 if provision_options.evm_network == EvmNetwork::Anvil {
421 let anvil_node_data =
422 get_anvil_node_data(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
423 .map_err(|err| {
424 println!("Failed to get evm testnet data {err:?}");
425 err
426 })?;
427
428 provision_options.funding_wallet_secret_key =
429 Some(anvil_node_data.deployer_wallet_private_key);
430 }
431
432 self.wait_for_ssh_availability_on_new_machines(
433 AnsibleInventoryType::Clients,
434 &options.current_inventory,
435 )?;
436 let genesis_network_contacts = get_bootstrap_cache_url(&initial_ip_addr);
437 self.ansible_provisioner
438 .print_ansible_run_banner("Provision Clients");
439 self.ansible_provisioner
440 .provision_uploaders(
441 &provision_options,
442 Some(initial_multiaddr.clone()),
443 Some(genesis_network_contacts.clone()),
444 )
445 .await
446 .map_err(|err| {
447 println!("Failed to provision Clients {err:?}");
448 err
449 })?;
450 }
451
452 if node_provision_failed {
453 println!();
454 println!("{}", "WARNING!".yellow());
455 println!("Some nodes failed to provision without error.");
456 println!("This usually means a small number of nodes failed to start on a few VMs.");
457 println!("However, most of the time the deployment will still be usable.");
458 println!("See the output from Ansible to determine which VMs had failures.");
459 }
460
461 Ok(())
462 }
463
464 pub async fn upscale_clients(&self, options: &UpscaleOptions) -> Result<()> {
465 let is_bootstrap_deploy = matches!(
466 options
467 .current_inventory
468 .environment_details
469 .deployment_type,
470 DeploymentType::Bootstrap
471 );
472
473 if is_bootstrap_deploy {
474 return Err(Error::InvalidClientUpscaleDeploymentType(
475 "bootstrap".to_string(),
476 ));
477 }
478
479 let desired_client_vm_count = options
480 .desired_client_vm_count
481 .unwrap_or(options.current_inventory.client_vms.len() as u16);
482 if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
483 return Err(Error::InvalidUpscaleDesiredClientVmCount);
484 }
485 debug!("Using {desired_client_vm_count} for desired Client VM count");
486
487 let mut infra_run_options = InfraRunOptions::generate_existing(
488 &options.current_inventory.name,
489 &options.current_inventory.environment_details.region,
490 &self.terraform_runner,
491 Some(&options.current_inventory.environment_details),
492 )
493 .await?;
494 infra_run_options.client_vm_count = Some(desired_client_vm_count);
495
496 if options.plan {
497 self.plan(&infra_run_options)?;
498 return Ok(());
499 }
500
501 if !options.provision_only {
502 self.create_or_update_infra(&infra_run_options)
503 .map_err(|err| {
504 println!("Failed to create infra {err:?}");
505 err
506 })?;
507 }
508
509 if options.infra_only {
510 return Ok(());
511 }
512
513 let (initial_multiaddr, initial_ip_addr) =
514 get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)?
515 .ok_or_else(|| Error::GenesisListenAddress)?;
516 let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
517 debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
518
519 let provision_options = ProvisionOptions {
520 ant_version: options.ant_version.clone(),
521 binary_option: options.current_inventory.binary_option.clone(),
522 chunk_size: None,
523 client_env_variables: None,
524 delayed_verifier_batch_size: None,
525 delayed_verifier_quorum_value: None,
526 enable_delayed_verifier: options.enable_delayed_verifier,
527 enable_random_verifier: options.enable_random_verifier,
528 enable_performance_verifier: options.enable_performance_verifier,
529 enable_telegraf: true,
530 enable_uploaders: true,
531 evm_data_payments_address: options
532 .current_inventory
533 .environment_details
534 .evm_details
535 .data_payments_address
536 .clone(),
537 evm_network: options
538 .current_inventory
539 .environment_details
540 .evm_details
541 .network
542 .clone(),
543 evm_payment_token_address: options
544 .current_inventory
545 .environment_details
546 .evm_details
547 .payment_token_address
548 .clone(),
549 evm_rpc_url: options
550 .current_inventory
551 .environment_details
552 .evm_details
553 .rpc_url
554 .clone(),
555 expected_hash: None,
556 expected_size: None,
557 file_address: None,
558 full_cone_private_node_count: 0,
559 funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
560 gas_amount: options.gas_amount,
561 interval: Some(options.interval),
562 log_format: None,
563 max_archived_log_files: options.max_archived_log_files,
564 max_log_files: options.max_log_files,
565 max_uploads: None,
566 name: options.current_inventory.name.clone(),
567 network_id: options.current_inventory.environment_details.network_id,
568 network_dashboard_branch: None,
569 node_count: 0,
570 node_env_variables: None,
571 output_inventory_dir_path: self
572 .working_directory_path
573 .join("ansible")
574 .join("inventory"),
575 peer_cache_node_count: 0,
576 performance_verifier_batch_size: None,
577 public_rpc: options.public_rpc,
578 random_verifier_batch_size: None,
579 rewards_address: options
580 .current_inventory
581 .environment_details
582 .rewards_address
583 .clone(),
584 sleep_duration: None,
585 symmetric_private_node_count: 0,
586 token_amount: options.token_amount,
587 uploaders_count: options.desired_uploaders_count,
588 upload_batch_size: None,
589 upload_size: None,
590 upload_interval: None,
591 upnp_private_node_count: 0,
592 wallet_secret_keys: None,
593 };
594
595 self.wait_for_ssh_availability_on_new_machines(
596 AnsibleInventoryType::Clients,
597 &options.current_inventory,
598 )?;
599 self.ansible_provisioner
600 .print_ansible_run_banner("Provision Clients");
601 self.ansible_provisioner
602 .provision_uploaders(
603 &provision_options,
604 Some(initial_multiaddr),
605 Some(initial_network_contacts_url),
606 )
607 .await
608 .map_err(|err| {
609 println!("Failed to provision clients {err:?}");
610 err
611 })?;
612
613 Ok(())
614 }
615
616 fn wait_for_ssh_availability_on_new_machines(
617 &self,
618 inventory_type: AnsibleInventoryType,
619 current_inventory: &DeploymentInventory,
620 ) -> Result<()> {
621 let inventory = self
622 .ansible_provisioner
623 .ansible_runner
624 .get_inventory(inventory_type, true)?;
625 let old_set: HashSet<_> = match inventory_type {
626 AnsibleInventoryType::Clients => current_inventory
627 .client_vms
628 .iter()
629 .map(|client_vm| &client_vm.vm)
630 .cloned()
631 .collect(),
632 AnsibleInventoryType::PeerCacheNodes => current_inventory
633 .peer_cache_node_vms
634 .iter()
635 .map(|node_vm| &node_vm.vm)
636 .cloned()
637 .collect(),
638 AnsibleInventoryType::Nodes => current_inventory
639 .node_vms
640 .iter()
641 .map(|node_vm| &node_vm.vm)
642 .cloned()
643 .collect(),
644 AnsibleInventoryType::FullConeNatGateway => current_inventory
645 .full_cone_nat_gateway_vms
646 .iter()
647 .cloned()
648 .collect(),
649 AnsibleInventoryType::SymmetricNatGateway => current_inventory
650 .symmetric_nat_gateway_vms
651 .iter()
652 .cloned()
653 .collect(),
654 AnsibleInventoryType::FullConePrivateNodes => current_inventory
655 .full_cone_private_node_vms
656 .iter()
657 .map(|node_vm| &node_vm.vm)
658 .cloned()
659 .collect(),
660 AnsibleInventoryType::SymmetricPrivateNodes => current_inventory
661 .symmetric_private_node_vms
662 .iter()
663 .map(|node_vm| &node_vm.vm)
664 .cloned()
665 .collect(),
666 it => return Err(Error::UpscaleInventoryTypeNotSupported(it.to_string())),
667 };
668 let new_vms: Vec<_> = inventory
669 .into_iter()
670 .filter(|item| !old_set.contains(item))
671 .collect();
672 for vm in new_vms.iter() {
673 self.ssh_client.wait_for_ssh_availability(
674 &vm.public_ip_addr,
675 &self.cloud_provider.get_ssh_user(),
676 )?;
677 }
678 Ok(())
679 }
680}