1use crate::{
8 ansible::{
9 inventory::AnsibleInventoryType,
10 provisioning::{PrivateNodeProvisionInventory, ProvisionOptions},
11 },
12 error::{Error, Result},
13 get_anvil_node_data, get_bootstrap_cache_url, get_genesis_multiaddr, get_multiaddr,
14 DeploymentInventory, DeploymentType, EvmNetwork, InfraRunOptions, NodeType, TestnetDeployer,
15};
16use colored::Colorize;
17use evmlib::common::U256;
18use log::debug;
19use std::{collections::HashSet, time::Duration};
20
21#[derive(Clone)]
22pub struct UpscaleOptions {
23 pub ansible_verbose: bool,
24 pub ant_version: Option<String>,
25 pub current_inventory: DeploymentInventory,
26 pub desired_client_vm_count: Option<u16>,
27 pub desired_full_cone_private_node_count: Option<u16>,
28 pub desired_full_cone_private_node_vm_count: Option<u16>,
29 pub desired_node_count: Option<u16>,
30 pub desired_node_vm_count: Option<u16>,
31 pub desired_peer_cache_node_count: Option<u16>,
32 pub desired_peer_cache_node_vm_count: Option<u16>,
33 pub desired_symmetric_private_node_count: Option<u16>,
34 pub desired_symmetric_private_node_vm_count: Option<u16>,
35 pub desired_uploaders_count: Option<u16>,
36 pub enable_delayed_verifier: bool,
37 pub enable_random_verifier: bool,
38 pub enable_performance_verifier: bool,
39 pub funding_wallet_secret_key: Option<String>,
40 pub gas_amount: Option<U256>,
41 pub interval: Duration,
42 pub infra_only: bool,
43 pub max_archived_log_files: u16,
44 pub max_log_files: u16,
45 pub network_dashboard_branch: Option<String>,
46 pub node_env_variables: Option<Vec<(String, String)>>,
47 pub plan: bool,
48 pub public_rpc: bool,
49 pub provision_only: bool,
50 pub token_amount: Option<U256>,
51}
52
53impl TestnetDeployer {
54 pub async fn upscale(&self, options: &UpscaleOptions) -> Result<()> {
55 let is_bootstrap_deploy = matches!(
56 options
57 .current_inventory
58 .environment_details
59 .deployment_type,
60 DeploymentType::Bootstrap
61 );
62
63 if is_bootstrap_deploy
64 && (options.desired_peer_cache_node_count.is_some()
65 || options.desired_peer_cache_node_vm_count.is_some()
66 || options.desired_client_vm_count.is_some())
67 {
68 return Err(Error::InvalidUpscaleOptionsForBootstrapDeployment);
69 }
70
71 let desired_peer_cache_node_vm_count = options
72 .desired_peer_cache_node_vm_count
73 .unwrap_or(options.current_inventory.peer_cache_node_vms.len() as u16);
74 if desired_peer_cache_node_vm_count
75 < options.current_inventory.peer_cache_node_vms.len() as u16
76 {
77 return Err(Error::InvalidUpscaleDesiredPeerCacheVmCount);
78 }
79 debug!("Using {desired_peer_cache_node_vm_count} for desired Peer Cache node VM count");
80
81 let desired_node_vm_count = options
82 .desired_node_vm_count
83 .unwrap_or(options.current_inventory.node_vms.len() as u16);
84 if desired_node_vm_count < options.current_inventory.node_vms.len() as u16 {
85 return Err(Error::InvalidUpscaleDesiredNodeVmCount);
86 }
87 debug!("Using {desired_node_vm_count} for desired node VM count");
88
89 let desired_full_cone_private_node_vm_count = options
90 .desired_full_cone_private_node_vm_count
91 .unwrap_or(options.current_inventory.full_cone_private_node_vms.len() as u16);
92 if desired_full_cone_private_node_vm_count
93 < options.current_inventory.full_cone_private_node_vms.len() as u16
94 {
95 return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeVmCount);
96 }
97 debug!("Using {desired_full_cone_private_node_vm_count} for desired full cone private node VM count");
98
99 let desired_symmetric_private_node_vm_count = options
100 .desired_symmetric_private_node_vm_count
101 .unwrap_or(options.current_inventory.symmetric_private_node_vms.len() as u16);
102 if desired_symmetric_private_node_vm_count
103 < options.current_inventory.symmetric_private_node_vms.len() as u16
104 {
105 return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeVmCount);
106 }
107 debug!("Using {desired_symmetric_private_node_vm_count} for desired full cone private node VM count");
108
109 let desired_client_vm_count = options
110 .desired_client_vm_count
111 .unwrap_or(options.current_inventory.client_vms.len() as u16);
112 if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
113 return Err(Error::InvalidUpscaleDesiredClientVmCount);
114 }
115 debug!("Using {desired_client_vm_count} for desired Client VM count");
116
117 let desired_peer_cache_node_count = options
118 .desired_peer_cache_node_count
119 .unwrap_or(options.current_inventory.peer_cache_node_count() as u16);
120 if desired_peer_cache_node_count < options.current_inventory.peer_cache_node_count() as u16
121 {
122 return Err(Error::InvalidUpscaleDesiredPeerCacheNodeCount);
123 }
124 debug!("Using {desired_peer_cache_node_count} for desired peer cache node count");
125
126 let desired_node_count = options
127 .desired_node_count
128 .unwrap_or(options.current_inventory.node_count() as u16);
129 if desired_node_count < options.current_inventory.node_count() as u16 {
130 return Err(Error::InvalidUpscaleDesiredNodeCount);
131 }
132 debug!("Using {desired_node_count} for desired node count");
133
134 let desired_full_cone_private_node_count = options
135 .desired_full_cone_private_node_count
136 .unwrap_or(options.current_inventory.full_cone_private_node_count() as u16);
137 if desired_full_cone_private_node_count
138 < options.current_inventory.full_cone_private_node_count() as u16
139 {
140 return Err(Error::InvalidUpscaleDesiredFullConePrivateNodeCount);
141 }
142 debug!(
143 "Using {desired_full_cone_private_node_count} for desired full cone private node count"
144 );
145
146 let desired_symmetric_private_node_count = options
147 .desired_symmetric_private_node_count
148 .unwrap_or(options.current_inventory.symmetric_private_node_count() as u16);
149 if desired_symmetric_private_node_count
150 < options.current_inventory.symmetric_private_node_count() as u16
151 {
152 return Err(Error::InvalidUpscaleDesiredSymmetricPrivateNodeCount);
153 }
154 debug!(
155 "Using {desired_symmetric_private_node_count} for desired symmetric private node count"
156 );
157
158 let mut infra_run_options = InfraRunOptions::generate_existing(
159 &options.current_inventory.name,
160 &options.current_inventory.environment_details.region,
161 &self.terraform_runner,
162 Some(&options.current_inventory.environment_details),
163 )
164 .await?;
165 infra_run_options.peer_cache_node_vm_count = Some(desired_peer_cache_node_vm_count);
166 infra_run_options.node_vm_count = Some(desired_node_vm_count);
167 infra_run_options.full_cone_private_node_vm_count =
168 Some(desired_full_cone_private_node_vm_count);
169 infra_run_options.symmetric_private_node_vm_count =
170 Some(desired_symmetric_private_node_vm_count);
171 infra_run_options.client_vm_count = Some(desired_client_vm_count);
172
173 if options.plan {
174 self.plan(&infra_run_options)?;
175 return Ok(());
176 }
177
178 self.create_or_update_infra(&infra_run_options)
179 .map_err(|err| {
180 println!("Failed to create infra {err:?}");
181 err
182 })?;
183
184 if options.infra_only {
185 return Ok(());
186 }
187
188 let mut provision_options = ProvisionOptions {
189 ant_version: options.ant_version.clone(),
190 binary_option: options.current_inventory.binary_option.clone(),
191 chunk_size: None,
192 client_env_variables: None,
193 delayed_verifier_batch_size: None,
194 delayed_verifier_quorum_value: None,
195 enable_delayed_verifier: options.enable_delayed_verifier,
196 enable_performance_verifier: options.enable_performance_verifier,
197 enable_random_verifier: options.enable_random_verifier,
198 enable_telegraf: true,
199 enable_uploaders: true,
200 evm_data_payments_address: options
201 .current_inventory
202 .environment_details
203 .evm_details
204 .data_payments_address
205 .clone(),
206 evm_network: options
207 .current_inventory
208 .environment_details
209 .evm_details
210 .network
211 .clone(),
212 evm_payment_token_address: options
213 .current_inventory
214 .environment_details
215 .evm_details
216 .payment_token_address
217 .clone(),
218 evm_rpc_url: options
219 .current_inventory
220 .environment_details
221 .evm_details
222 .rpc_url
223 .clone(),
224 expected_hash: None,
225 expected_size: None,
226 file_address: None,
227 full_cone_private_node_count: desired_full_cone_private_node_count,
228 funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
229 gas_amount: options.gas_amount,
230 interval: Some(options.interval),
231 log_format: None,
232 max_archived_log_files: options.max_archived_log_files,
233 max_log_files: options.max_log_files,
234 max_uploads: None,
235 name: options.current_inventory.name.clone(),
236 network_id: options.current_inventory.environment_details.network_id,
237 network_dashboard_branch: None,
238 node_count: desired_node_count,
239 node_env_variables: options.node_env_variables.clone(),
240 output_inventory_dir_path: self
241 .working_directory_path
242 .join("ansible")
243 .join("inventory"),
244 peer_cache_node_count: desired_peer_cache_node_count,
245 performance_verifier_batch_size: None,
246 public_rpc: options.public_rpc,
247 random_verifier_batch_size: None,
248 rewards_address: options
249 .current_inventory
250 .environment_details
251 .rewards_address
252 .clone(),
253 sleep_duration: None,
254 symmetric_private_node_count: desired_symmetric_private_node_count,
255 token_amount: None,
256 upload_size: None,
257 uploaders_count: options.desired_uploaders_count,
258 upload_interval: None,
259 upnp_private_node_count: 0,
260 wallet_secret_keys: None,
261 };
262 let mut node_provision_failed = false;
263
264 let (initial_multiaddr, initial_ip_addr) = if is_bootstrap_deploy {
265 get_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client).map_err(
266 |err| {
267 println!("Failed to get node multiaddr {err:?}");
268 err
269 },
270 )?
271 } else {
272 get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
273 .map_err(|err| {
274 println!("Failed to get genesis multiaddr {err:?}");
275 err
276 })?
277 };
278 let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
279 debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
280
281 if !is_bootstrap_deploy {
282 self.wait_for_ssh_availability_on_new_machines(
283 AnsibleInventoryType::PeerCacheNodes,
284 &options.current_inventory,
285 )?;
286 self.ansible_provisioner
287 .print_ansible_run_banner("Provision Peer Cache Nodes");
288 match self.ansible_provisioner.provision_nodes(
289 &provision_options,
290 Some(initial_multiaddr.clone()),
291 Some(initial_network_contacts_url.clone()),
292 NodeType::PeerCache,
293 ) {
294 Ok(()) => {
295 println!("Provisioned Peer Cache nodes");
296 }
297 Err(err) => {
298 log::error!("Failed to provision Peer Cache nodes: {err}");
299 node_provision_failed = true;
300 }
301 }
302 }
303
304 self.wait_for_ssh_availability_on_new_machines(
305 AnsibleInventoryType::Nodes,
306 &options.current_inventory,
307 )?;
308 self.ansible_provisioner
309 .print_ansible_run_banner("Provision Normal Nodes");
310 match self.ansible_provisioner.provision_nodes(
311 &provision_options,
312 Some(initial_multiaddr.clone()),
313 Some(initial_network_contacts_url.clone()),
314 NodeType::Generic,
315 ) {
316 Ok(()) => {
317 println!("Provisioned normal nodes");
318 }
319 Err(err) => {
320 log::error!("Failed to provision normal nodes: {err}");
321 node_provision_failed = true;
322 }
323 }
324
325 let private_node_inventory = PrivateNodeProvisionInventory::new(
326 &self.ansible_provisioner,
327 Some(desired_full_cone_private_node_vm_count),
328 Some(desired_symmetric_private_node_vm_count),
329 )?;
330
331 if private_node_inventory.should_provision_full_cone_private_nodes() {
332 let full_cone_nat_gateway_inventory = self
333 .ansible_provisioner
334 .ansible_runner
335 .get_inventory(AnsibleInventoryType::FullConeNatGateway, true)?;
336
337 let full_cone_nat_gateway_new_vms: Vec<_> = full_cone_nat_gateway_inventory
338 .into_iter()
339 .filter(|item| {
340 !options
341 .current_inventory
342 .full_cone_nat_gateway_vms
343 .contains(item)
344 })
345 .collect();
346
347 for vm in full_cone_nat_gateway_new_vms.iter() {
348 self.ssh_client.wait_for_ssh_availability(
349 &vm.public_ip_addr,
350 &self.cloud_provider.get_ssh_user(),
351 )?;
352 }
353
354 let full_cone_nat_gateway_new_vms = if full_cone_nat_gateway_new_vms.is_empty() {
355 None
356 } else {
357 debug!("Full Cone NAT Gateway new VMs: {full_cone_nat_gateway_new_vms:?}");
358 Some(full_cone_nat_gateway_new_vms)
359 };
360
361 match self.ansible_provisioner.provision_full_cone(
362 &provision_options,
363 Some(initial_multiaddr.clone()),
364 Some(initial_network_contacts_url.clone()),
365 private_node_inventory.clone(),
366 full_cone_nat_gateway_new_vms,
367 ) {
368 Ok(()) => {
369 println!("Provisioned Full Cone nodes and Gateway");
370 }
371 Err(err) => {
372 log::error!("Failed to provision Full Cone nodes and Gateway: {err}");
373 node_provision_failed = true;
374 }
375 }
376 }
377
378 if private_node_inventory.should_provision_symmetric_private_nodes() {
379 self.wait_for_ssh_availability_on_new_machines(
380 AnsibleInventoryType::SymmetricNatGateway,
381 &options.current_inventory,
382 )?;
383 self.ansible_provisioner
384 .print_ansible_run_banner("Provision Symmetric NAT Gateway");
385 self.ansible_provisioner
386 .provision_symmetric_nat_gateway(&provision_options, &private_node_inventory)
387 .map_err(|err| {
388 println!("Failed to provision symmetric NAT gateway {err:?}");
389 err
390 })?;
391
392 self.wait_for_ssh_availability_on_new_machines(
393 AnsibleInventoryType::SymmetricPrivateNodes,
394 &options.current_inventory,
395 )?;
396 self.ansible_provisioner
397 .print_ansible_run_banner("Provision Symmetric Private Nodes");
398 match self.ansible_provisioner.provision_symmetric_private_nodes(
399 &mut provision_options,
400 Some(initial_multiaddr.clone()),
401 Some(initial_network_contacts_url.clone()),
402 &private_node_inventory,
403 ) {
404 Ok(()) => {
405 println!("Provisioned symmetric private nodes");
406 }
407 Err(err) => {
408 log::error!("Failed to provision symmetric private nodes: {err}");
409 node_provision_failed = true;
410 }
411 }
412 }
413
414 let should_provision_uploaders =
415 options.desired_uploaders_count.is_some() || options.desired_client_vm_count.is_some();
416 if should_provision_uploaders {
417 if provision_options.evm_network == EvmNetwork::Anvil {
419 let anvil_node_data =
420 get_anvil_node_data(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
421 .map_err(|err| {
422 println!("Failed to get evm testnet data {err:?}");
423 err
424 })?;
425
426 provision_options.funding_wallet_secret_key =
427 Some(anvil_node_data.deployer_wallet_private_key);
428 }
429
430 self.wait_for_ssh_availability_on_new_machines(
431 AnsibleInventoryType::Clients,
432 &options.current_inventory,
433 )?;
434 let genesis_network_contacts = get_bootstrap_cache_url(&initial_ip_addr);
435 self.ansible_provisioner
436 .print_ansible_run_banner("Provision Clients");
437 self.ansible_provisioner
438 .provision_uploaders(
439 &provision_options,
440 Some(initial_multiaddr.clone()),
441 Some(genesis_network_contacts.clone()),
442 )
443 .await
444 .map_err(|err| {
445 println!("Failed to provision Clients {err:?}");
446 err
447 })?;
448 }
449
450 if node_provision_failed {
451 println!();
452 println!("{}", "WARNING!".yellow());
453 println!("Some nodes failed to provision without error.");
454 println!("This usually means a small number of nodes failed to start on a few VMs.");
455 println!("However, most of the time the deployment will still be usable.");
456 println!("See the output from Ansible to determine which VMs had failures.");
457 }
458
459 Ok(())
460 }
461
462 pub async fn upscale_clients(&self, options: &UpscaleOptions) -> Result<()> {
463 let is_bootstrap_deploy = matches!(
464 options
465 .current_inventory
466 .environment_details
467 .deployment_type,
468 DeploymentType::Bootstrap
469 );
470
471 if is_bootstrap_deploy {
472 return Err(Error::InvalidClientUpscaleDeploymentType(
473 "bootstrap".to_string(),
474 ));
475 }
476
477 let desired_client_vm_count = options
478 .desired_client_vm_count
479 .unwrap_or(options.current_inventory.client_vms.len() as u16);
480 if desired_client_vm_count < options.current_inventory.client_vms.len() as u16 {
481 return Err(Error::InvalidUpscaleDesiredClientVmCount);
482 }
483 debug!("Using {desired_client_vm_count} for desired Client VM count");
484
485 let mut infra_run_options = InfraRunOptions::generate_existing(
486 &options.current_inventory.name,
487 &options.current_inventory.environment_details.region,
488 &self.terraform_runner,
489 Some(&options.current_inventory.environment_details),
490 )
491 .await?;
492 infra_run_options.client_vm_count = Some(desired_client_vm_count);
493
494 if options.plan {
495 self.plan(&infra_run_options)?;
496 return Ok(());
497 }
498
499 if !options.provision_only {
500 self.create_or_update_infra(&infra_run_options)
501 .map_err(|err| {
502 println!("Failed to create infra {err:?}");
503 err
504 })?;
505 }
506
507 if options.infra_only {
508 return Ok(());
509 }
510
511 let (initial_multiaddr, initial_ip_addr) =
512 get_genesis_multiaddr(&self.ansible_provisioner.ansible_runner, &self.ssh_client)
513 .map_err(|err| {
514 println!("Failed to get genesis multiaddr {err:?}");
515 err
516 })?;
517 let initial_network_contacts_url = get_bootstrap_cache_url(&initial_ip_addr);
518 debug!("Retrieved initial peer {initial_multiaddr} and initial network contacts {initial_network_contacts_url}");
519
520 let provision_options = ProvisionOptions {
521 ant_version: options.ant_version.clone(),
522 binary_option: options.current_inventory.binary_option.clone(),
523 chunk_size: None,
524 client_env_variables: None,
525 delayed_verifier_batch_size: None,
526 delayed_verifier_quorum_value: None,
527 enable_delayed_verifier: options.enable_delayed_verifier,
528 enable_random_verifier: options.enable_random_verifier,
529 enable_performance_verifier: options.enable_performance_verifier,
530 enable_telegraf: true,
531 enable_uploaders: true,
532 evm_data_payments_address: options
533 .current_inventory
534 .environment_details
535 .evm_details
536 .data_payments_address
537 .clone(),
538 evm_network: options
539 .current_inventory
540 .environment_details
541 .evm_details
542 .network
543 .clone(),
544 evm_payment_token_address: options
545 .current_inventory
546 .environment_details
547 .evm_details
548 .payment_token_address
549 .clone(),
550 evm_rpc_url: options
551 .current_inventory
552 .environment_details
553 .evm_details
554 .rpc_url
555 .clone(),
556 expected_hash: None,
557 expected_size: None,
558 file_address: None,
559 full_cone_private_node_count: 0,
560 funding_wallet_secret_key: options.funding_wallet_secret_key.clone(),
561 gas_amount: options.gas_amount,
562 interval: Some(options.interval),
563 log_format: None,
564 max_archived_log_files: options.max_archived_log_files,
565 max_log_files: options.max_log_files,
566 max_uploads: None,
567 name: options.current_inventory.name.clone(),
568 network_id: options.current_inventory.environment_details.network_id,
569 network_dashboard_branch: None,
570 node_count: 0,
571 node_env_variables: None,
572 output_inventory_dir_path: self
573 .working_directory_path
574 .join("ansible")
575 .join("inventory"),
576 peer_cache_node_count: 0,
577 performance_verifier_batch_size: None,
578 public_rpc: options.public_rpc,
579 random_verifier_batch_size: None,
580 rewards_address: options
581 .current_inventory
582 .environment_details
583 .rewards_address
584 .clone(),
585 sleep_duration: None,
586 symmetric_private_node_count: 0,
587 token_amount: options.token_amount,
588 uploaders_count: options.desired_uploaders_count,
589 upload_size: None,
590 upload_interval: None,
591 upnp_private_node_count: 0,
592 wallet_secret_keys: None,
593 };
594
595 self.wait_for_ssh_availability_on_new_machines(
596 AnsibleInventoryType::Clients,
597 &options.current_inventory,
598 )?;
599 self.ansible_provisioner
600 .print_ansible_run_banner("Provision Clients");
601 self.ansible_provisioner
602 .provision_uploaders(
603 &provision_options,
604 Some(initial_multiaddr),
605 Some(initial_network_contacts_url),
606 )
607 .await
608 .map_err(|err| {
609 println!("Failed to provision clients {err:?}");
610 err
611 })?;
612
613 Ok(())
614 }
615
616 fn wait_for_ssh_availability_on_new_machines(
617 &self,
618 inventory_type: AnsibleInventoryType,
619 current_inventory: &DeploymentInventory,
620 ) -> Result<()> {
621 let inventory = self
622 .ansible_provisioner
623 .ansible_runner
624 .get_inventory(inventory_type, true)?;
625 let old_set: HashSet<_> = match inventory_type {
626 AnsibleInventoryType::Clients => current_inventory
627 .client_vms
628 .iter()
629 .map(|client_vm| &client_vm.vm)
630 .cloned()
631 .collect(),
632 AnsibleInventoryType::PeerCacheNodes => current_inventory
633 .peer_cache_node_vms
634 .iter()
635 .map(|node_vm| &node_vm.vm)
636 .cloned()
637 .collect(),
638 AnsibleInventoryType::Nodes => current_inventory
639 .node_vms
640 .iter()
641 .map(|node_vm| &node_vm.vm)
642 .cloned()
643 .collect(),
644 AnsibleInventoryType::FullConeNatGateway => current_inventory
645 .full_cone_nat_gateway_vms
646 .iter()
647 .cloned()
648 .collect(),
649 AnsibleInventoryType::SymmetricNatGateway => current_inventory
650 .symmetric_nat_gateway_vms
651 .iter()
652 .cloned()
653 .collect(),
654 AnsibleInventoryType::FullConePrivateNodes => current_inventory
655 .full_cone_private_node_vms
656 .iter()
657 .map(|node_vm| &node_vm.vm)
658 .cloned()
659 .collect(),
660 AnsibleInventoryType::SymmetricPrivateNodes => current_inventory
661 .symmetric_private_node_vms
662 .iter()
663 .map(|node_vm| &node_vm.vm)
664 .cloned()
665 .collect(),
666 it => return Err(Error::UpscaleInventoryTypeNotSupported(it.to_string())),
667 };
668 let new_vms: Vec<_> = inventory
669 .into_iter()
670 .filter(|item| !old_set.contains(item))
671 .collect();
672 for vm in new_vms.iter() {
673 self.ssh_client.wait_for_ssh_availability(
674 &vm.public_ip_addr,
675 &self.cloud_provider.get_ssh_user(),
676 )?;
677 }
678 Ok(())
679 }
680}