1#![forbid(unsafe_op_in_unsafe_fn)]
18
19mod child_manager;
20use child_manager::*;
21
22#[cfg(windows)]
23mod windows_kill_tree;
24
25mod shutdown;
26use shutdown::*;
27
28mod utilities;
29use utilities::*;
30
31use anyhow::{Context as AnyhowContext, Result as AnyhowResult, anyhow, bail, ensure};
32use chrono::Local;
33use clap::Parser;
34use dunce::canonicalize;
35use itertools::Itertools;
36use parking_lot::Mutex;
37use std::{
38 env,
39 path::{Path, PathBuf},
40 process::{Child, Command as StdCommand, Stdio},
41 sync::Arc,
42 time::Duration,
43};
44use tracing::{self, Span};
45
46#[cfg(unix)]
47use {libc::setsid, std::os::unix::process::CommandExt};
48
49use super::*;
50use leo_ast::NetworkName;
51
52const REST_RPS: &str = "999999999";
54
55#[derive(Parser, Debug)]
57pub struct LeoDevnet {
58 #[clap(long, help = "Number of validators", default_value = "4")]
59 pub(crate) num_validators: usize,
60 #[clap(long, help = "Number of clients", default_value = "2")]
61 pub(crate) num_clients: usize,
62 #[clap(short = 'n', long, help = "Network (mainnet=0, testnet=1, canary=2)", default_value = "testnet")]
63 pub(crate) network: NetworkName,
64 #[clap(short = 's', long, help = "Ledger / log root directory", default_value = "./")]
65 pub(crate) storage: PathBuf,
66 #[clap(long, help = "Path to snarkOS binary. If it does not exist, set `--install` to build it at this path.")]
67 pub(crate) snarkos: Option<PathBuf>,
68 #[clap(long, help = "Required features for snarkOS (e.g. `test_network`)", value_delimiter = ',')]
69 pub(crate) snarkos_features: Vec<String>,
70 #[clap(long, help = "Required version for snarkOS (e.g. `4.1.0`). Defaults to latest version on `crates.io`.")]
71 pub(crate) snarkos_version: Option<String>,
72 #[clap(long, help = "(Re)install snarkOS at the provided `--snarkos` path with the given `--snarkos-features`")]
73 pub(crate) install: bool,
74 #[clap(
75 long,
76 help = "Optional consensus heights to use. The `test_network` feature must be enabled for this to work.",
77 value_delimiter = ',',
78 env = "CONSENSUS_VERSION_HEIGHTS"
79 )]
80 pub(crate) consensus_heights: Option<Vec<u32>>,
81 #[clap(long, help = "Run nodes in tmux (only available on Unix)")]
82 pub(crate) tmux: bool,
83 #[clap(short = 'v', long, help = "snarkOS verbosity (0-4)", default_value = "1")]
84 pub(crate) verbosity: u8,
85 #[clap(long, short = 'y', help = "Skip confirmation prompts and proceed with the devnet startup")]
86 pub(crate) yes: bool,
87 #[clap(long, help = "Base REST port (each node uses base + dev_index)")]
88 pub(crate) rest_port: Option<u16>,
89 #[clap(long, help = "Base node port (each node uses base + dev_index)")]
90 pub(crate) node_port: Option<u16>,
91 #[clap(long, help = "Base BFT port (each node uses base + dev_index)")]
92 pub(crate) bft_port: Option<u16>,
93 #[clap(long, help = "Base metrics port (each validator uses base + dev_index)")]
94 pub(crate) metrics_port: Option<u16>,
95 #[clap(short = 'c', long, help = "Remove existing devnet storage before starting")]
96 pub(crate) clear_storage: bool,
97 #[clap(long, help = "Only clean devnet storage (ledgers, node data, logs) without starting")]
98 pub(crate) clean_only: bool,
99}
100
101impl Command for LeoDevnet {
102 type Input = ();
103 type Output = ();
104
105 fn log_span(&self) -> Span {
106 tracing::span!(tracing::Level::INFO, "LeoDevnet")
107 }
108
109 fn prelude(&self, _: Context) -> Result<Self::Input> {
110 Ok(())
111 }
112
113 fn apply(self, _cx: Context, _: Self::Input) -> Result<Self::Output> {
114 self.handle_apply().map_err(|e| crate::errors::custom(format!("Failed to run devnet command: {e}")).into())
115 }
116}
117
118impl LeoDevnet {
119 fn handle_apply(&self) -> AnyhowResult<()> {
121 if self.clean_only {
125 return self.handle_clean();
126 }
127
128 let snarkos_path = self.snarkos.as_ref().ok_or_else(|| {
132 anyhow!(
133 "The `--snarkos` flag is required when starting a devnet. Use `--clean-only` to only clean storage."
134 )
135 })?;
136
137 if cfg!(windows) && self.tmux {
138 bail!("tmux mode is not available on Windows – remove `--tmux`.");
139 }
140 if self.tmux && std::env::var("TMUX").is_ok() {
141 bail!("Nested tmux session detected. Unset $TMUX and retry.");
142 }
143
144 if let Some(ref heights) = self.consensus_heights {
146 if !self.snarkos_features.contains(&"test_network".to_string()) {
147 bail!("The `test_network` feature must be enabled on snarkOS to use `--consensus-heights`.");
148 }
149 validate_consensus_heights(heights.as_slice())?;
150 }
151
152 if self.num_validators < 4 {
154 bail!("The number of validators must be at least 4.");
155 }
156
157 let total_nodes = self.num_validators + self.num_clients;
159 Self::validate_port_range("--rest-port", self.rest_port, total_nodes)?;
160 Self::validate_port_range("--node-port", self.node_port, total_nodes)?;
161 Self::validate_port_range("--bft-port", self.bft_port, total_nodes)?;
162 Self::validate_port_range("--metrics-port", self.metrics_port, self.num_validators)?;
163
164 if self.install {
166 if let Some(parent) = snarkos_path.parent()
168 && !parent.exists()
169 {
170 std::fs::create_dir_all(parent)
171 .with_context(|| format!("Failed to create directory for binary: {}", parent.display()))?;
172 }
173 std::fs::write(snarkos_path, [0u8]).with_context(|| {
174 format!("Failed to write to path {} for snarkos installation", snarkos_path.display())
175 })?;
176 } else {
177 if !snarkos_path.exists() {
179 bail!(
180 "The snarkOS binary at `{}` does not exist. Please provide a valid path or use `--install`.",
181 snarkos_path.display()
182 );
183 }
184 };
185 let snarkos = canonicalize(snarkos_path)
186 .with_context(|| format!("Failed to resolve snarkOS path: {}", snarkos_path.display()))?;
187
188 println!("🔧 Starting devnet with the following options:");
190 println!(" • Network: {}", self.network);
191 println!(" • Validators: {}", self.num_validators);
192 println!(" • Clients: {}", self.num_clients);
193 println!(" • Storage: {}", self.storage.display());
194 if self.install {
195 println!(" • Installing snarkOS at: {}", snarkos.display());
196 if let Some(ref version) = self.snarkos_version {
197 println!(" • version: {version}");
198 }
199 if !self.snarkos_features.is_empty() {
200 println!(" • features: {}", self.snarkos_features.iter().format(","));
201 }
202 } else {
203 println!(" • Using snarkOS binary at: {}", snarkos.display());
204 }
205 if let Some(heights) = &self.consensus_heights {
206 println!(" • Consensus heights: {}", heights.iter().format(","));
207 } else {
208 println!(" • Consensus heights: default (based on your snarkOS binary)");
209 }
210 println!(" • Clear storage: {}", if self.clear_storage { "yes" } else { "no" });
211 println!(" • Verbosity: {}", self.verbosity);
212 println!(" • tmux: {}", if self.tmux { "yes" } else { "no" });
213
214 let manager = Arc::new(Mutex::new(ChildManager::new()));
218
219 let (tx_shutdown, rx_shutdown) = crossbeam_channel::bounded::<()>(1);
221 let _signal_thread =
222 install_shutdown_listener(tx_shutdown.clone()).context("Failed to install shutdown listener")?;
223
224 let snarkos = if self.install {
228 if !confirm("\nProceed with snarkOS installation?", self.yes)? {
229 println!("❌ Installation aborted.");
230 return Ok(());
231 }
232 install_snarkos(&snarkos, self.snarkos_version.as_deref(), &self.snarkos_features)?
233 } else {
234 snarkos
235 };
236
237 let version_output = StdCommand::new(&snarkos)
239 .arg("--version")
240 .output()
241 .context(format!("Failed to run `{}`", snarkos.display()))?;
242 if !version_output.status.success() {
243 bail!("Failed to run `{}`: {}", snarkos.display(), String::from_utf8_lossy(&version_output.stderr));
244 }
245
246 let version_str = String::from_utf8_lossy(&version_output.stdout);
248 println!("🔍 Detected: {version_str}");
249
250 let features_str = version_str
254 .trim()
255 .split("features=[")
256 .nth(1)
257 .and_then(|s| s.split(']').next())
258 .ok_or_else(|| anyhow!("Failed to parse snarkOS features from version string: {version_str}"))?;
259 let found_features: Vec<String> = features_str.split(',').map(|s| s.trim().to_string()).collect();
260 for feature in &self.snarkos_features {
261 if !found_features.contains(feature) {
262 println!("⚠️ Warning: snarkOS does not have the required feature `{feature}` enabled.");
263 }
264 }
265
266 if !confirm("\nProceed with devnet startup?", self.yes)? {
267 println!("❌ Devnet aborted.");
268 return Ok(());
269 }
270
271 if !self.storage.exists() {
276 std::fs::create_dir_all(&self.storage)
277 .context(format!("Failed to create storage directory: {}", self.storage.display()))?;
278 } else if !self.storage.is_dir() {
279 bail!("The storage path `{}` is not a directory.", self.storage.display());
280 }
281 let storage = canonicalize(&self.storage)
283 .with_context(|| format!("Failed to resolve storage path: {}", self.storage.display()))?;
284
285 if self.clear_storage {
287 println!("🧹 Cleaning ledgers …");
288 let mut cleaners = Vec::new();
289 for idx in 0..self.num_validators {
290 cleaners.push(clean_snarkos(
291 &snarkos,
292 self.network as usize,
293 idx,
294 &storage.join(format!("node-{idx}")),
295 &storage.join(format!("node-data-{idx}")),
296 )?);
297 }
298 for idx in 0..self.num_clients {
299 let dev_idx = idx + self.num_validators;
300 cleaners.push(clean_snarkos(
301 &snarkos,
302 self.network as usize,
303 dev_idx,
304 &storage.join(format!("node-{dev_idx}")),
305 &storage.join(format!("node-data-{dev_idx}")),
306 )?);
307 }
308 for mut c in cleaners {
309 c.wait()?;
310 }
311 }
312
313 let log_dir = {
315 let ts = Local::now().format(".logs-%Y-%m-%d-%H-%M-%S").to_string();
316 let p = storage.join(ts);
317 std::fs::create_dir_all(&p)?;
318 p
319 };
320
321 #[allow(clippy::too_many_arguments)]
326 fn build_args(
327 role: &str,
328 verbosity: u8,
329 network: usize,
330 num_validators: usize,
331 num_nodes: usize,
332 idx: usize,
333 log_file: &Path,
334 storage: &Path,
335 rest_port: Option<u16>,
336 node_port: Option<u16>,
337 bft_port: Option<u16>,
338 metrics_port: Option<u16>,
339 ) -> Vec<String> {
340 let idx_u16 = idx as u16;
341 let mut base = vec![
342 "start".to_string(),
343 "--nodisplay".to_string(),
344 "--network".to_string(),
345 network.to_string(),
346 "--dev".to_string(),
347 idx.to_string(),
348 "--dev-num-validators".to_string(),
349 num_validators.to_string(),
350 "--rest-rps".to_string(),
351 REST_RPS.to_string(),
352 "--logfile".to_string(),
353 log_file.to_str().expect("log path is valid UTF-8").to_string(),
354 "--verbosity".to_string(),
355 verbosity.to_string(),
356 "--ledger-storage".to_string(),
357 storage.join(format!("node-{idx}")).to_str().expect("storage path is valid UTF-8").to_string(),
358 "--node-data-storage".to_string(),
359 storage.join(format!("node-data-{idx}")).to_str().expect("node-data path is valid UTF-8").to_string(),
360 ];
361 if let Some(port) = rest_port {
362 base.extend(["--rest".into(), format!("0.0.0.0:{}", port + idx_u16)]);
364 }
365 if let Some(port) = node_port {
366 base.extend(["--node".into(), format!("0.0.0.0:{}", port + idx_u16)]);
367 let peers: String =
371 (0..num_nodes).map(|i| format!("127.0.0.1:{}", port + i as u16)).collect::<Vec<_>>().join(",");
372 base.extend(["--peers".into(), peers]);
373 }
374 if let Some(port) = bft_port {
375 base.extend(["--bft".into(), format!("0.0.0.0:{}", port + idx_u16)]);
376 let validators: String =
380 (0..num_validators).map(|i| format!("127.0.0.1:{}", port + i as u16)).collect::<Vec<_>>().join(",");
381 base.extend(["--validators".into(), validators]);
382 }
383 match role {
384 "validator" => {
385 base.extend(
386 ["--allow-external-peers", "--validator", "--no-dev-txs"].into_iter().map(String::from),
387 );
388 if let Some(port) = metrics_port {
389 base.extend(["--metrics".into(), "--metrics-ip".into(), format!("0.0.0.0:{}", port + idx_u16)]);
390 }
391 }
392 "client" => base.push("--client".into()),
393 _ => unreachable!(),
394 }
395 base
396 }
397
398 if let Some(ref heights) = self.consensus_heights {
401 let heights = heights.iter().join(",");
402 println!("🔧 Setting consensus heights: {heights}");
403 #[allow(unsafe_code)]
404 unsafe {
405 env::set_var("CONSENSUS_VERSION_HEIGHTS", heights);
412 }
413 }
414
415 if self.tmux {
417 let mut args: Vec<String> =
419 vec!["new-session", "-d", "-s", "devnet", "-n", "validator-0"].into_iter().map(Into::into).collect();
420
421 if let Some(ref heights) = self.consensus_heights {
425 let heights = heights.iter().join(",");
426 args.push("-e".to_string());
427 args.push(format!("CONSENSUS_VERSION_HEIGHTS={heights}"));
428 }
429
430 ensure!(StdCommand::new("tmux").args(args).status()?.success(), "tmux failed to create session");
431
432 let num_nodes = self.num_validators + self.num_clients;
433
434 let base_index = {
436 let out = StdCommand::new("tmux").args(["show-option", "-gv", "base-index"]).output()?;
437 String::from_utf8_lossy(&out.stdout).trim().parse::<usize>().unwrap_or(0)
438 };
439
440 for idx in 0..self.num_validators {
442 let win_idx = idx + base_index;
443 let window_name = format!("validator-{idx}");
444 if idx != 0 {
445 StdCommand::new("tmux")
446 .args(["new-window", "-t", &format!("devnet:{win_idx}"), "-n", &window_name])
447 .status()?;
448 }
449 let log_file = log_dir.join(format!("{window_name}.log"));
450 let cmd = std::iter::once(snarkos.to_string_lossy().into_owned())
451 .chain(build_args(
452 "validator",
453 self.verbosity,
454 self.network as usize,
455 self.num_validators,
456 num_nodes,
457 idx,
458 log_file.as_path(),
459 &storage,
460 self.rest_port,
461 self.node_port,
462 self.bft_port,
463 self.metrics_port,
464 ))
465 .collect::<Vec<_>>()
466 .join(" ");
467 StdCommand::new("tmux")
468 .args(["send-keys", "-t", &format!("devnet:{win_idx}"), &cmd, "C-m"])
469 .status()?;
470 }
471
472 for idx in 0..self.num_clients {
474 let dev_idx = idx + self.num_validators;
475 let win_idx = dev_idx + base_index;
476 let window_name = format!("client-{idx}");
477 StdCommand::new("tmux")
478 .args(["new-window", "-t", &format!("devnet:{win_idx}"), "-n", &window_name])
479 .status()?;
480 let log_file = log_dir.join(format!("{window_name}.log"));
481 let cmd = std::iter::once(snarkos.to_string_lossy().into_owned())
482 .chain(build_args(
483 "client",
484 self.verbosity,
485 self.network as usize,
486 self.num_validators,
487 num_nodes,
488 dev_idx,
489 log_file.as_path(),
490 &storage,
491 self.rest_port,
492 self.node_port,
493 self.bft_port,
494 None,
495 ))
496 .collect::<Vec<_>>()
497 .join(" ");
498 StdCommand::new("tmux")
499 .args(["send-keys", "-t", &format!("devnet:{win_idx}"), &cmd, "C-m"])
500 .status()?;
501 }
502
503 println!("✅ tmux session \"devnet\" is ready – attaching …");
504 StdCommand::new("tmux").args(["attach-session", "-t", "devnet"]).status()?;
505 return Ok(()); }
507
508 println!("⚙️ Spawning nodes as background tasks …");
510
511 let spawn_with_group = |mut cmd: StdCommand, log_file: &Path| -> AnyhowResult<Child> {
513 let log_handle = std::fs::OpenOptions::new().create(true).append(true).open(log_file)?;
514 cmd.stdout(Stdio::from(log_handle.try_clone()?));
515 cmd.stderr(Stdio::from(log_handle));
516
517 #[cfg(unix)]
518 #[allow(unsafe_code)]
519 unsafe {
520 cmd.pre_exec(|| {
523 setsid();
524 Ok(())
525 });
526 }
527
528 let child = cmd.spawn().map_err(|e| anyhow!("spawn {e}"))?;
529
530 #[cfg(windows)]
531 windows_kill_tree::attach_to_global_job(child.id())?;
532
533 Ok(child)
534 };
535
536 let num_nodes = self.num_validators + self.num_clients;
537
538 {
539 let mut guard = manager.lock();
541
542 for idx in 0..self.num_validators {
544 let log_file = log_dir.join(format!("validator-{idx}.log"));
545 let child = spawn_with_group(
546 {
547 let mut c = StdCommand::new(&snarkos);
548 c.args(build_args(
549 "validator",
550 self.verbosity,
551 self.network as usize,
552 self.num_validators,
553 num_nodes,
554 idx,
555 &log_file,
556 &storage,
557 self.rest_port,
558 self.node_port,
559 self.bft_port,
560 self.metrics_port,
561 ));
562 c
563 },
564 &log_file,
565 )?;
566 println!(" • validator {idx} (pid = {})", child.id());
567 guard.push(child);
568 }
569
570 for idx in 0..self.num_clients {
572 let dev_idx = idx + self.num_validators;
573 let log_file = log_dir.join(format!("client-{idx}.log"));
574 let child = spawn_with_group(
575 {
576 let mut c = StdCommand::new(&snarkos);
577 c.args(build_args(
578 "client",
579 self.verbosity,
580 self.network as usize,
581 self.num_validators,
582 num_nodes,
583 dev_idx,
584 &log_file,
585 &storage,
586 self.rest_port,
587 self.node_port,
588 self.bft_port,
589 None,
590 ));
591 c
592 },
593 &log_file,
594 )?;
595 println!(" • client {idx} (pid = {})", child.id());
596 guard.push(child);
597 }
598 }
599
600 println!("📌 Main process ID: {}", std::process::id());
602 println!("\nDevnet running – Ctrl+C, SIGTERM, or terminal close to stop.");
603
604 let _ = rx_shutdown.recv();
606 manager.lock().shutdown_all(Duration::from_secs(30));
607
608 Ok(())
609 }
610
611 fn handle_clean(&self) -> AnyhowResult<()> {
614 let snarkos_path = self.snarkos.as_ref().ok_or_else(|| {
615 anyhow!("The `--snarkos` flag is required for `--clean-only`. Provide the path to the snarkOS binary.")
616 })?;
617 if !snarkos_path.exists() {
618 bail!("The snarkOS binary at `{}` does not exist.", snarkos_path.display());
619 }
620 let snarkos = canonicalize(snarkos_path)?;
621
622 if !self.storage.exists() {
623 println!("Storage path `{}` does not exist. Nothing to clean.", self.storage.display());
624 return Ok(());
625 }
626 if !self.storage.is_dir() {
627 bail!("Storage path `{}` is not a directory.", self.storage.display());
628 }
629 let storage = canonicalize(&self.storage)?;
630
631 let total_nodes = self.num_validators + self.num_clients;
632 if !confirm(
633 &format!("\nClean devnet storage for {total_nodes} nodes in `{}`?", self.storage.display()),
634 self.yes,
635 )? {
636 println!("Aborted.");
637 return Ok(());
638 }
639
640 println!("🧹 Cleaning ledgers …");
641 let mut cleaners = Vec::new();
642 for idx in 0..self.num_validators {
643 cleaners.push(clean_snarkos(
644 &snarkos,
645 self.network as usize,
646 idx,
647 &storage.join(format!("node-{idx}")),
648 &storage.join(format!("node-data-{idx}")),
649 )?);
650 }
651 for idx in 0..self.num_clients {
652 let dev_idx = idx + self.num_validators;
653 cleaners.push(clean_snarkos(
654 &snarkos,
655 self.network as usize,
656 dev_idx,
657 &storage.join(format!("node-{dev_idx}")),
658 &storage.join(format!("node-data-{dev_idx}")),
659 )?);
660 }
661 for mut c in cleaners {
662 c.wait()?;
663 }
664
665 for entry in std::fs::read_dir(&storage)? {
667 let entry = entry?;
668 let name = entry.file_name();
669 let name = name.to_string_lossy();
670 if entry.file_type()?.is_dir() && name.starts_with(".logs-") {
671 std::fs::remove_dir_all(entry.path())?;
672 println!(" Removed {}", entry.path().display());
673 }
674 }
675
676 println!("Cleaned devnet storage.");
677 Ok(())
678 }
679
680 fn validate_port_range(flag: &str, base: Option<u16>, count: usize) -> AnyhowResult<()> {
682 if let Some(port) = base {
683 if count == 0 {
684 return Ok(());
685 }
686 let max_idx = (count - 1) as u16;
687 if port.checked_add(max_idx).is_none() {
688 bail!("{flag} {port} + {max_idx} nodes exceeds the maximum port number (65535).");
689 }
690 }
691 Ok(())
692 }
693}