Skip to main content

astrs_cli/command/
cluster.rs

1//! `astrs up`, `astrs down` and the cluster half of `astrs status`
2//! (blueprint §17's Lifecycle row, §4.2's process model, §16's token,
3//! §24.2's runtime directory).
4//!
5//! ```text
6//!   astrs up
7//!     ├─ <working dir>/.astrs-token        generate (0600) or reuse   §16
8//!     ├─ spawn self: `astrs coordinator --announce --pidfile …`
9//!     │      └─ wait for <runtime>/coordinator.pid to name an address
10//!     ├─ greet it as a CLI (Hello{role=Cli,token}) — proof, not hope
11//!     └─ spawn self: `astrs daemon --coordinator <addr> --pidfile …`
12//!            └─ wait for <runtime>/daemon.pid
13//!
14//!   astrs status ─► pidfiles + one `Check` over the wire
15//!   astrs down   ─► Destroy ─► SIGTERM ─► grace ─► SIGKILL ─► rm pidfiles
16//! ```
17//!
18//! # One binary, two more processes
19//!
20//! `up` spawns **this same executable** with the hidden verbs
21//! ([`crate::command::serve`]) rather than looking for `astrs-coordinator`
22//! and `astrs-daemon` on `PATH`. That is what makes a deployment one file
23//! to copy (§4.2) and what makes `up` work from a `cargo run` target, a
24//! container layer, or a test's `CARGO_BIN_EXE_astrs` without any of them
25//! having to install anything.
26//!
27//! # These children must *outlive* their parent
28//!
29//! Exactly opposite to `astrs run`'s orphan guard. `astrs run` wants every
30//! child dead the moment the CLI dies; `astrs up` returns to the shell
31//! immediately and the cluster must survive that — and must survive the
32//! Ctrl-C that ends the *next* foreground command in that terminal. Each
33//! child therefore gets its own process group (so a terminal's `SIGINT`
34//! never reaches it) and is explicitly *not* given
35//! [`astrs_wire::ENV_RUN_PARENT_PID`]. `astrs down` is the way to stop
36//! them, and the pidfile is how it finds them.
37//!
38//! # Nothing here trusts a pid alone
39//!
40//! A pidfile can outlive its process and a pid can be recycled, so
41//! [`status`] confirms a *coordinator* by greeting it over the wire and
42//! only falls back to the `kill(pid, 0)` probe for the daemon, which has no
43//! CLI-facing listener of its own.
44
45use std::io::Write;
46use std::net::SocketAddr;
47use std::path::{Path, PathBuf};
48use std::process::{Command, Stdio};
49use std::time::{Duration, Instant};
50
51use astrs_wire::{ControlReply, ControlRequest};
52
53use crate::command::client::{Client, Endpoint, resolve_addr, runtime};
54use crate::error::CliError;
55use crate::runtime_dir::{
56    COORDINATOR_PIDFILE, DAEMON_PIDFILE, PidFile, read_pidfile, remove_pidfile,
57};
58
59/// How long a spawned child has to bind its listener and write its pidfile
60/// before `astrs up` gives up on it.
61///
62/// Generous: a cold coordinator opens (and possibly creates) a redb store
63/// on a filesystem that may be slow, and a machine under a heavy build is
64/// exactly when someone runs `astrs up`.
65pub const READY_TIMEOUT: Duration = Duration::from_secs(20);
66
67/// How often the parent looks for a child's pidfile while waiting.
68pub const READY_POLL: Duration = Duration::from_millis(25);
69
70/// How long a stopping cluster process has after `SIGTERM` before `SIGKILL`.
71///
72/// The coordinator has connections to close and a store to flush; the
73/// daemon has nodes to stop through its own finish ladder (§12). Five
74/// seconds is long enough for both to end cleanly and short enough that
75/// `astrs down` stays an interactive command.
76pub const STOP_GRACE: Duration = Duration::from_secs(5);
77
78/// How often a stopping process is re-checked while the grace runs.
79pub const STOP_POLL: Duration = Duration::from_millis(20);
80
81/// The file each spawned child's stdout and stderr are redirected into.
82///
83/// A child of `astrs up` has no terminal — the parent returns to the shell
84/// while it keeps running — so its output goes where an operator can read
85/// it afterwards, next to its pidfile (§24.2).
86#[must_use]
87pub fn log_path(runtime_dir: &Path, role: &str) -> PathBuf {
88    runtime_dir.join(format!("{role}.log"))
89}
90
91/// What one cluster process is doing.
92#[derive(Debug, Clone, PartialEq, Eq)]
93pub struct ProcessStatus {
94    /// `"coordinator"` or `"daemon"`.
95    pub role: &'static str,
96    /// Its process id, when a pidfile named one.
97    pub pid: Option<u32>,
98    /// Where it listens, when it announced an address.
99    pub address: Option<String>,
100    /// Whether the process exists right now.
101    pub running: bool,
102    /// What was observed, in words.
103    pub note: String,
104}
105
106impl ProcessStatus {
107    /// A status for a process no pidfile describes.
108    #[must_use]
109    pub fn absent(role: &'static str) -> Self {
110        Self {
111            role,
112            pid: None,
113            address: None,
114            running: false,
115            note: "not running here (no pidfile)".to_owned(),
116        }
117    }
118
119    /// The status a pidfile plus a liveness probe implies.
120    #[must_use]
121    pub fn from_pidfile(role: &'static str, record: &PidFile) -> Self {
122        let running = record.is_alive();
123        Self {
124            role,
125            pid: Some(record.pid),
126            address: record.addr.clone(),
127            running,
128            note: if running {
129                "running".to_owned()
130            } else {
131                "a pidfile is here but the process is gone (stale)".to_owned()
132            },
133        }
134    }
135
136    /// Reads `path` and classifies whatever is there.
137    #[must_use]
138    pub fn probe(role: &'static str, path: &Path) -> Self {
139        read_pidfile(path).map_or_else(
140            || Self::absent(role),
141            |record| Self::from_pidfile(role, &record),
142        )
143    }
144
145    /// The `--json` form.
146    #[must_use]
147    pub fn to_json(&self) -> serde_json::Value {
148        serde_json::json!({
149            "role": self.role,
150            "pid": self.pid,
151            "address": self.address,
152            "running": self.running,
153            "note": self.note,
154        })
155    }
156
157    /// The one-line human form.
158    #[must_use]
159    pub fn to_line(&self) -> String {
160        let pid = self
161            .pid
162            .map_or_else(|| "-".to_owned(), |pid| pid.to_string());
163        let addr = self.address.clone().unwrap_or_else(|| "-".to_owned());
164        format!("{:<12} {:<8} {:<24} {}", self.role, pid, addr, self.note)
165    }
166}
167
168/// `astrs up`'s arguments, already parsed and resolved.
169#[derive(Debug, Clone, Default)]
170pub struct UpArgs {
171    /// The port the coordinator should bind; `0` takes a free one.
172    pub port: Option<u16>,
173    /// Where pidfiles, sockets and child logs live (§24.2).
174    pub runtime_dir: Option<PathBuf>,
175    /// Where `.astrs-token` is written or found (§16).
176    pub working_dir: Option<PathBuf>,
177    /// An explicit token, instead of generating one.
178    pub token: Option<String>,
179    /// A file holding that token.
180    pub token_file: Option<PathBuf>,
181    /// Delete the coordinator's store before it opens it.
182    pub recreate_store: bool,
183    /// Start the coordinator only.
184    pub no_daemon: bool,
185    /// Emit JSON rather than a human summary.
186    pub json: bool,
187}
188
189/// What one `astrs up` did.
190#[derive(Debug, Clone)]
191pub struct UpReport {
192    /// The coordinator, started or adopted.
193    pub coordinator: ProcessStatus,
194    /// This machine's daemon, unless `--no-daemon`.
195    pub daemon: Option<ProcessStatus>,
196    /// Where the cluster token lives (§16).
197    pub token_path: Option<PathBuf>,
198    /// Where the pidfiles and logs are (§24.2).
199    pub runtime_dir: PathBuf,
200    /// Whether anything was already running and was adopted rather than
201    /// started.
202    pub adopted: bool,
203}
204
205impl UpReport {
206    /// The human summary.
207    #[must_use]
208    pub fn summary(&self) -> String {
209        let mut lines = vec![if self.adopted {
210            "cluster already running (adopted)".to_owned()
211        } else {
212            "cluster up".to_owned()
213        }];
214        lines.push(self.coordinator.to_line());
215        if let Some(daemon) = &self.daemon {
216            lines.push(daemon.to_line());
217        } else {
218            lines.push(
219                "daemon       -        -                        not started (--no-daemon)"
220                    .to_owned(),
221            );
222        }
223        if let Some(token) = &self.token_path {
224            lines.push(format!("token: {}", token.display()));
225        }
226        lines.push(format!("runtime dir: {}", self.runtime_dir.display()));
227        lines.join("\n")
228    }
229
230    /// The `--json` form.
231    #[must_use]
232    pub fn to_json(&self) -> serde_json::Value {
233        serde_json::json!({
234            "adopted": self.adopted,
235            "coordinator": self.coordinator.to_json(),
236            "daemon": self.daemon.as_ref().map(ProcessStatus::to_json),
237            "token": self.token_path.as_ref().map(|p| p.display().to_string()),
238            "runtime_dir": self.runtime_dir.display().to_string(),
239        })
240    }
241}
242
243/// Brings a cluster up: a coordinator, and (unless `--no-daemon`) this
244/// machine's daemon.
245///
246/// # Errors
247///
248/// - [`CliError::AlreadyRunning`] if `--recreate-store` is asked for while a
249///   coordinator is live — wiping the store under a running cluster would
250///   lose exactly the state the flag exists to reset.
251/// - [`CliError::Io`] if the runtime directory, the token file or a child
252///   log cannot be written, or a child cannot be spawned.
253/// - [`CliError::Cluster`] if a child exits before it is ready, or never
254///   becomes ready.
255/// - [`CliError::BadToken`] if an existing token file holds something
256///   unusable.
257/// - Whatever greeting the coordinator refuses with.
258pub fn up(out: &mut dyn Write, args: &UpArgs) -> Result<UpReport, CliError> {
259    let runtime_dir = crate::runtime_dir::runtime_dir(args.runtime_dir.as_deref());
260    crate::runtime_dir::ensure_dir(&runtime_dir)?;
261    let working_dir = crate::runtime_dir::working_dir(args.working_dir.as_deref());
262
263    let coordinator_pidfile = runtime_dir.join(COORDINATOR_PIDFILE);
264    let daemon_pidfile = runtime_dir.join(DAEMON_PIDFILE);
265    let live_coordinator = ProcessStatus::probe("coordinator", &coordinator_pidfile);
266    if live_coordinator.running && args.recreate_store {
267        return Err(CliError::AlreadyRunning {
268            process: "coordinator",
269            pid: live_coordinator.pid.unwrap_or(0),
270            address: live_coordinator
271                .address
272                .as_ref()
273                .map_or_else(String::new, |addr| format!(", {addr}")),
274        });
275    }
276
277    // §16: the token is per-cluster and lives beside the manifest, not in
278    // the per-machine runtime directory.
279    let (token_argument, token_path) = match (&args.token, &args.token_file) {
280        (Some(token), _) => (TokenArgument::Inline(token.clone()), None),
281        (None, Some(path)) => (TokenArgument::File(path.clone()), Some(path.clone())),
282        (None, None) => {
283            let (_, path) = crate::runtime_dir::ensure_token_file(&working_dir)?;
284            (TokenArgument::File(path.clone()), Some(path))
285        }
286    };
287
288    let mut adopted = live_coordinator.running;
289    let coordinator = if live_coordinator.running {
290        // Gated on `--json`: the report already carries `adopted`, and a
291        // stray human line ahead of the object would make the whole output
292        // unparsable for the script that asked for JSON.
293        if !args.json {
294            let _ = writeln!(
295                out,
296                "coordinator already running (pid {}); adopting it",
297                live_coordinator.pid.unwrap_or(0)
298            );
299        }
300        live_coordinator
301    } else {
302        remove_pidfile(&coordinator_pidfile)?;
303        let mut argv = vec![
304            "coordinator".to_owned(),
305            "--port".to_owned(),
306            args.port
307                .unwrap_or_else(crate::command::client::default_port)
308                .to_string(),
309            "--store".to_owned(),
310            runtime_dir
311                .join(crate::command::serve::STORE_FILE)
312                .display()
313                .to_string(),
314            "--working-dir".to_owned(),
315            working_dir.display().to_string(),
316            "--pidfile".to_owned(),
317            coordinator_pidfile.display().to_string(),
318            "--announce".to_owned(),
319        ];
320        argv.extend(token_argument.to_argv());
321        if args.recreate_store {
322            argv.push("--recreate-store".to_owned());
323        }
324        start_child("coordinator", &argv, &runtime_dir, &working_dir)?;
325        wait_for_ready(
326            "coordinator",
327            &coordinator_pidfile,
328            &runtime_dir,
329            true,
330            READY_TIMEOUT,
331        )?
332    };
333
334    // Proof rather than hope: a pidfile says a process exists, a completed
335    // greeting says the *cluster* is usable with the token this `up` chose.
336    let endpoint = greet(&coordinator, args, &working_dir)?;
337
338    let daemon = if args.no_daemon {
339        None
340    } else {
341        let live_daemon = ProcessStatus::probe("daemon", &daemon_pidfile);
342        if live_daemon.running {
343            adopted = true;
344            if !args.json {
345                let _ = writeln!(
346                    out,
347                    "daemon already running (pid {}); adopting it",
348                    live_daemon.pid.unwrap_or(0)
349                );
350            }
351            Some(live_daemon)
352        } else {
353            remove_pidfile(&daemon_pidfile)?;
354            let mut argv = vec![
355                "daemon".to_owned(),
356                "--coordinator".to_owned(),
357                endpoint.clone(),
358                "--runtime-dir".to_owned(),
359                runtime_dir.display().to_string(),
360                "--working-dir".to_owned(),
361                working_dir.display().to_string(),
362                "--pidfile".to_owned(),
363                daemon_pidfile.display().to_string(),
364                // An ephemeral peer port (§6.4), not §24.2's 7409: the
365                // address other daemons dial reaches them through the
366                // daemon's *registration*, never through a well-known
367                // number, so a fixed port buys nothing and costs an
368                // `EADDRINUSE` the moment two clusters share a machine —
369                // which is exactly what a test suite is.
370                "--peer-port".to_owned(),
371                "0".to_owned(),
372                "--announce".to_owned(),
373            ];
374            argv.extend(token_argument.to_argv());
375            start_child("daemon", &argv, &runtime_dir, &working_dir)?;
376            Some(wait_for_ready(
377                "daemon",
378                &daemon_pidfile,
379                &runtime_dir,
380                false,
381                READY_TIMEOUT,
382            )?)
383        }
384    };
385
386    let report = UpReport {
387        coordinator,
388        daemon,
389        token_path,
390        runtime_dir,
391        adopted,
392    };
393    emit(out, args.json, &report.summary(), || report.to_json());
394    Ok(report)
395}
396
397/// How a child is told which token to use.
398#[derive(Debug, Clone)]
399enum TokenArgument {
400    /// A 64-hex value on the command line.
401    Inline(String),
402    /// A path to read it from — always preferred, since an argument is
403    /// visible in `ps` to every user on the machine (§16).
404    File(PathBuf),
405}
406
407impl TokenArgument {
408    /// The two argv entries this spelling contributes.
409    fn to_argv(&self) -> Vec<String> {
410        match self {
411            Self::Inline(token) => vec!["--token".to_owned(), token.clone()],
412            Self::File(path) => vec!["--token-file".to_owned(), path.display().to_string()],
413        }
414    }
415}
416
417/// Greets a running coordinator, returning the address that worked.
418///
419/// # Errors
420///
421/// As [`Client::connect`] — a refusal here is the useful one: it means the
422/// process is up but the token (or the protocol) does not match.
423fn greet(
424    coordinator: &ProcessStatus,
425    args: &UpArgs,
426    working_dir: &Path,
427) -> Result<String, CliError> {
428    let address = coordinator.address.clone().unwrap_or_else(|| {
429        format!(
430            "127.0.0.1:{}",
431            args.port
432                .unwrap_or_else(crate::command::client::default_port)
433        )
434    });
435    let endpoint = crate::command::client::endpoint(
436        Some(&address),
437        args.token.as_deref(),
438        args.token_file.as_deref(),
439        Some(working_dir),
440        false,
441    )?;
442    let runtime = runtime()?;
443    runtime.block_on(async {
444        let mut client = Client::connect(&endpoint).await?;
445        client
446            .request_ok("status", &ControlRequest::Check { dataflow: None })
447            .await
448            .or_else(|error| match error {
449                // A cluster with dataflows but no daemon answers `Check`
450                // with an error; the greeting still proved the listener and
451                // the token, which is all `up` is asserting here.
452                CliError::Refused { .. } | CliError::UnexpectedReply { .. } => Ok(()),
453                other => Err(other),
454            })?;
455        Ok::<(), CliError>(())
456    })?;
457    Ok(address)
458}
459
460/// Spawns one hidden verb as a detached child of this process.
461///
462/// # Errors
463///
464/// [`CliError::Io`] if this executable cannot be located, the log file
465/// cannot be opened, or the process cannot be spawned.
466fn start_child(
467    role: &'static str,
468    argv: &[String],
469    runtime_dir: &Path,
470    working_dir: &Path,
471) -> Result<u32, CliError> {
472    let exe = std::env::current_exe().map_err(|source| CliError::io("this executable", source))?;
473    let log = log_path(runtime_dir, role);
474    let file = std::fs::OpenOptions::new()
475        .create(true)
476        .append(true)
477        .open(&log)
478        .map_err(|source| CliError::io(&log, source))?;
479    let errors = file
480        .try_clone()
481        .map_err(|source| CliError::io(&log, source))?;
482
483    let mut command = Command::new(exe);
484    command
485        .args(argv)
486        .current_dir(working_dir)
487        .stdin(Stdio::null())
488        .stdout(Stdio::from(file))
489        .stderr(Stdio::from(errors));
490    // A cluster process is not this shell's child in any way that matters:
491    // its own process group means the terminal's Ctrl-C never reaches it,
492    // and no orphan guard means it does not die with the CLI that started
493    // it (see this module's own docs).
494    std::os::unix::process::CommandExt::process_group(&mut command, 0);
495    command.env_remove(astrs_wire::ENV_RUN_PARENT_PID);
496    command.env_remove(astrs_wire::ENV_NODE_CONFIG);
497
498    let child = command
499        .spawn()
500        .map_err(|source| CliError::io(format!("spawning `astrs {role}`"), source))?;
501    Ok(child.id())
502}
503
504/// Waits for a spawned child to write a pidfile, and (for a listener) an
505/// address in it.
506///
507/// # Errors
508///
509/// [`CliError::Cluster`] when the child never becomes ready, with the tail
510/// of its own log attached — the only place its failure was written.
511fn wait_for_ready(
512    role: &'static str,
513    pidfile: &Path,
514    runtime_dir: &Path,
515    needs_address: bool,
516    timeout: Duration,
517) -> Result<ProcessStatus, CliError> {
518    let deadline = Instant::now() + timeout;
519    loop {
520        if let Some(record) = read_pidfile(pidfile)
521            && (!needs_address || record.addr.is_some())
522        {
523            return Ok(ProcessStatus::from_pidfile(role, &record));
524        }
525        if Instant::now() >= deadline {
526            return Err(CliError::Cluster {
527                action: "start",
528                process: role,
529                reason: format!(
530                    "it did not become ready within {:.0}s; its log says:\n{}",
531                    timeout.as_secs_f64(),
532                    log_tail(&log_path(runtime_dir, role))
533                ),
534            });
535        }
536        std::thread::sleep(READY_POLL);
537    }
538}
539
540/// The last few lines of a child's log, for an error message.
541fn log_tail(path: &Path) -> String {
542    let Ok(text) = std::fs::read_to_string(path) else {
543        return format!("(nothing was written to {})", path.display());
544    };
545    let lines: Vec<&str> = text.lines().collect();
546    let start = lines.len().saturating_sub(12);
547    let tail = lines[start..].join("\n");
548    if tail.trim().is_empty() {
549        format!("(nothing was written to {})", path.display())
550    } else {
551        tail
552    }
553}
554
555/// `astrs down`'s arguments, already parsed and resolved.
556#[derive(Debug, Clone, Default)]
557pub struct DownArgs {
558    /// Where the pidfiles live (§24.2).
559    pub runtime_dir: Option<PathBuf>,
560    /// Where `.astrs-token` is found (§16).
561    pub working_dir: Option<PathBuf>,
562    /// The coordinator address, when it is not the one in the pidfile.
563    pub coordinator: Option<String>,
564    /// An explicit token.
565    pub token: Option<String>,
566    /// A file holding that token.
567    pub token_file: Option<PathBuf>,
568    /// Stop the cluster even while dataflows are running.
569    pub force: bool,
570    /// Emit JSON rather than a human summary.
571    pub json: bool,
572}
573
574/// How one process was stopped.
575#[derive(Debug, Clone, Copy, PartialEq, Eq)]
576pub enum StopOutcome {
577    /// Nothing was running under that pidfile.
578    NotRunning,
579    /// It exited after `SIGTERM`.
580    Terminated,
581    /// It ignored `SIGTERM` and was killed.
582    Killed,
583    /// It could not be signalled at all (another user's process, or a pid
584    /// that vanished between the probe and the signal).
585    Unreachable,
586}
587
588impl StopOutcome {
589    /// A stable lowercase word, for both output forms.
590    #[must_use]
591    pub const fn as_str(self) -> &'static str {
592        match self {
593            Self::NotRunning => "not-running",
594            Self::Terminated => "terminated",
595            Self::Killed => "killed",
596            Self::Unreachable => "unreachable",
597        }
598    }
599}
600
601/// What one `astrs down` did.
602#[derive(Debug, Clone)]
603pub struct DownReport {
604    /// The daemon's outcome, and the pid it had.
605    pub daemon: (StopOutcome, Option<u32>),
606    /// The coordinator's outcome, and the pid it had.
607    pub coordinator: (StopOutcome, Option<u32>),
608    /// Whether the cluster was asked to destroy its dataflows first, and
609    /// what came of it.
610    pub destroy: Option<String>,
611}
612
613impl DownReport {
614    /// Whether anything at all was running.
615    #[must_use]
616    pub fn stopped_anything(&self) -> bool {
617        !matches!(
618            (self.daemon.0, self.coordinator.0),
619            (StopOutcome::NotRunning, StopOutcome::NotRunning)
620        )
621    }
622
623    /// The human summary.
624    #[must_use]
625    pub fn summary(&self) -> String {
626        let mut lines = Vec::new();
627        if let Some(note) = &self.destroy {
628            lines.push(format!("dataflows: {note}"));
629        }
630        for (role, (outcome, pid)) in [("daemon", self.daemon), ("coordinator", self.coordinator)] {
631            lines.push(match pid {
632                Some(pid) => format!("{role}: {} (pid {pid})", outcome.as_str()),
633                None => format!("{role}: {}", outcome.as_str()),
634            });
635        }
636        lines.push(if self.stopped_anything() {
637            "cluster down".to_owned()
638        } else {
639            "nothing was running here".to_owned()
640        });
641        lines.join("\n")
642    }
643
644    /// The `--json` form.
645    #[must_use]
646    pub fn to_json(&self) -> serde_json::Value {
647        serde_json::json!({
648            "stopped_anything": self.stopped_anything(),
649            "destroy": self.destroy,
650            "daemon": { "outcome": self.daemon.0.as_str(), "pid": self.daemon.1 },
651            "coordinator": {
652                "outcome": self.coordinator.0.as_str(),
653                "pid": self.coordinator.1,
654            },
655        })
656    }
657}
658
659/// Takes a cluster down: dataflows first, then the daemon, then the
660/// coordinator.
661///
662/// That order is the one that avoids lying to anybody: nodes are stopped
663/// through their own daemon while the coordinator still exists to be told
664/// about it, the daemon goes next so no node outlives its supervisor, and
665/// the coordinator last so it can record what happened.
666///
667/// # Errors
668///
669/// - [`CliError::Io`] if a pidfile cannot be removed.
670/// - Never for "nothing was running": that is a report, not a failure —
671///   `astrs down` twice in a script is a normal thing to do.
672pub fn down(out: &mut dyn Write, args: &DownArgs) -> Result<DownReport, CliError> {
673    let runtime_dir = crate::runtime_dir::runtime_dir(args.runtime_dir.as_deref());
674    let coordinator_pidfile = runtime_dir.join(COORDINATOR_PIDFILE);
675    let daemon_pidfile = runtime_dir.join(DAEMON_PIDFILE);
676
677    let coordinator = ProcessStatus::probe("coordinator", &coordinator_pidfile);
678    let daemon = ProcessStatus::probe("daemon", &daemon_pidfile);
679
680    let destroy = if coordinator.running {
681        Some(destroy_dataflows(args, &coordinator, &runtime_dir))
682    } else {
683        None
684    };
685
686    let daemon_outcome = stop_process(&daemon);
687    let coordinator_outcome = stop_process(&coordinator);
688
689    remove_pidfile(&daemon_pidfile)?;
690    remove_pidfile(&coordinator_pidfile)?;
691
692    let report = DownReport {
693        daemon: (daemon_outcome, daemon.pid),
694        coordinator: (coordinator_outcome, coordinator.pid),
695        destroy,
696    };
697    emit(out, args.json, &report.summary(), || report.to_json());
698    Ok(report)
699}
700
701/// Asks a live coordinator to stop its dataflows, reporting what it said.
702///
703/// Best-effort by design: `down`'s job is to stop *processes*, and a
704/// coordinator that refuses (because dataflows are running and `--force`
705/// was not given) or cannot be reached must not stop that from happening.
706/// The refusal is reported rather than swallowed.
707fn destroy_dataflows(args: &DownArgs, coordinator: &ProcessStatus, working: &Path) -> String {
708    let address = args
709        .coordinator
710        .clone()
711        .or_else(|| coordinator.address.clone());
712    let working_dir = args
713        .working_dir
714        .clone()
715        .unwrap_or_else(|| working.to_path_buf());
716    let endpoint = match crate::command::client::endpoint(
717        address.as_deref(),
718        args.token.as_deref(),
719        args.token_file.as_deref(),
720        Some(&working_dir),
721        false,
722    ) {
723        Ok(endpoint) => endpoint,
724        Err(error) => return format!("not asked to stop ({error})"),
725    };
726    let Ok(runtime) = runtime() else {
727        return "not asked to stop (no async runtime)".to_owned();
728    };
729    let outcome = runtime.block_on(async {
730        let mut client = Client::connect(&endpoint).await?;
731        client
732            .request("destroy", &ControlRequest::Destroy { force: args.force })
733            .await
734    });
735    match outcome {
736        Ok(ControlReply::Ok) => "destroyed".to_owned(),
737        Ok(other) => format!(
738            "the coordinator answered `{}`",
739            crate::command::client::reply_name(&other)
740        ),
741        Err(error) => format!("not stopped cleanly ({error})"),
742    }
743}
744
745/// `SIGTERM`, then the grace, then `SIGKILL` (§12's ladder, applied to a
746/// cluster process rather than a node).
747fn stop_process(status: &ProcessStatus) -> StopOutcome {
748    let Some(pid) = status.pid.filter(|_| status.running) else {
749        return StopOutcome::NotRunning;
750    };
751    let Ok(pid) = rustix::process::Pid::from_raw(pid.cast_signed()).ok_or(()) else {
752        return StopOutcome::Unreachable;
753    };
754    if rustix::process::kill_process(pid, rustix::process::Signal::TERM).is_err() {
755        return StopOutcome::Unreachable;
756    }
757    if wait_for_exit(pid, STOP_GRACE) {
758        return StopOutcome::Terminated;
759    }
760    if rustix::process::kill_process(pid, rustix::process::Signal::KILL).is_err() {
761        return StopOutcome::Unreachable;
762    }
763    if wait_for_exit(pid, STOP_GRACE) {
764        StopOutcome::Killed
765    } else {
766        StopOutcome::Unreachable
767    }
768}
769
770/// Polls until `pid` is gone or `grace` elapses.
771///
772/// Reaps first, every round: a process that has exited but is still this
773/// process's un-waited child is a zombie, and a zombie answers `kill(pid,
774/// 0)` exactly like a live process would. That case is rare in production —
775/// `astrs down` stops processes an earlier `astrs up` spawned, which were
776/// reparented when that CLI exited — and constant in a test that spawns its
777/// own. `waitpid` with `NOHANG` costs one syscall, never blocks, and simply
778/// fails with `ECHILD` when the pid is not ours, so it is the cheap way to
779/// be right in both worlds.
780fn wait_for_exit(pid: rustix::process::Pid, grace: Duration) -> bool {
781    let deadline = Instant::now() + grace;
782    loop {
783        let _ = rustix::process::waitpid(Some(pid), rustix::process::WaitOptions::NOHANG);
784        if rustix::process::test_kill_process(pid).is_err() {
785            return true;
786        }
787        if Instant::now() >= deadline {
788            return false;
789        }
790        std::thread::sleep(STOP_POLL);
791    }
792}
793
794/// `astrs status`'s arguments when no dataflow is named.
795#[derive(Debug, Clone, Default)]
796pub struct StatusArgs {
797    /// Where the pidfiles live (§24.2).
798    pub runtime_dir: Option<PathBuf>,
799    /// Where `.astrs-token` is found (§16).
800    pub working_dir: Option<PathBuf>,
801    /// The coordinator address, when it is not the one in the pidfile.
802    pub coordinator: Option<String>,
803    /// An explicit token.
804    pub token: Option<String>,
805    /// A file holding that token.
806    pub token_file: Option<PathBuf>,
807    /// Emit JSON rather than a human summary.
808    pub json: bool,
809}
810
811/// What `astrs status` found.
812#[derive(Debug, Clone)]
813pub struct ClusterStatus {
814    /// The coordinator's process, per its pidfile.
815    pub coordinator: ProcessStatus,
816    /// The daemon's process, per its pidfile.
817    pub daemon: ProcessStatus,
818    /// The address that was greeted.
819    pub address: Option<SocketAddr>,
820    /// Whether the coordinator answered a `Check` over the wire.
821    pub reachable: bool,
822    /// What the greeting said, when it did not succeed.
823    pub reachability_note: String,
824}
825
826impl ClusterStatus {
827    /// `0` when the coordinator answered, `1` otherwise — so
828    /// `astrs status >/dev/null && …` is a usable cluster-liveness test.
829    #[must_use]
830    pub const fn exit_code(&self) -> i32 {
831        if self.reachable { 0 } else { 1 }
832    }
833
834    /// The human summary.
835    #[must_use]
836    pub fn summary(&self) -> String {
837        let mut lines = vec![format!(
838            "{:<12} {:<8} {:<24} {}",
839            "PROCESS", "PID", "ADDRESS", "STATE"
840        )];
841        lines.push(self.coordinator.to_line());
842        lines.push(self.daemon.to_line());
843        lines.push(match (&self.address, self.reachable) {
844            (Some(addr), true) => format!("the coordinator at {addr} answered a health check"),
845            (Some(addr), false) => {
846                format!(
847                    "the coordinator at {addr} did not answer: {}",
848                    self.reachability_note
849                )
850            }
851            (None, _) => format!("no coordinator was reached: {}", self.reachability_note),
852        });
853        lines.join("\n")
854    }
855
856    /// The `--json` form.
857    #[must_use]
858    pub fn to_json(&self) -> serde_json::Value {
859        serde_json::json!({
860            "reachable": self.reachable,
861            "address": self.address.map(|addr| addr.to_string()),
862            "note": self.reachability_note,
863            "coordinator": self.coordinator.to_json(),
864            "daemon": self.daemon.to_json(),
865            "exit_code": self.exit_code(),
866        })
867    }
868}
869
870/// Probes this machine's cluster: both pidfiles, and one greeting.
871///
872/// # Errors
873///
874/// [`CliError::BadAddress`] if an explicit `--coordinator` cannot be
875/// resolved. Everything else is *content* of the report: a coordinator that
876/// is down is the answer to `astrs status`, not a failure of it.
877pub fn status(out: &mut dyn Write, args: &StatusArgs) -> Result<ClusterStatus, CliError> {
878    let runtime_dir = crate::runtime_dir::runtime_dir(args.runtime_dir.as_deref());
879    let coordinator = ProcessStatus::probe("coordinator", &runtime_dir.join(COORDINATOR_PIDFILE));
880    let daemon = ProcessStatus::probe("daemon", &runtime_dir.join(DAEMON_PIDFILE));
881
882    let address_text = args
883        .coordinator
884        .clone()
885        .or_else(|| coordinator.address.clone());
886    let address = resolve_addr(address_text.as_deref())?;
887    let working_dir = crate::runtime_dir::working_dir(args.working_dir.as_deref());
888    let endpoint = crate::command::client::endpoint(
889        address_text.as_deref(),
890        args.token.as_deref(),
891        args.token_file.as_deref(),
892        Some(&working_dir),
893        false,
894    )?;
895
896    let (reachable, note) = check(&endpoint);
897    let report = ClusterStatus {
898        coordinator,
899        daemon,
900        address: Some(address),
901        reachable,
902        reachability_note: note,
903    };
904    emit(out, args.json, &report.summary(), || report.to_json());
905    Ok(report)
906}
907
908/// One `Check`, reduced to "did it answer" plus what it said.
909fn check(endpoint: &Endpoint) -> (bool, String) {
910    let Ok(runtime) = runtime() else {
911        return (false, "no async runtime could be built".to_owned());
912    };
913    let outcome = runtime.block_on(async {
914        let mut client = Client::connect(endpoint).await?;
915        client
916            .request("status", &ControlRequest::Check { dataflow: None })
917            .await
918    });
919    match outcome {
920        Ok(_) => (true, "healthy".to_owned()),
921        // A refusal is an *answer*: the listener is there and speaking the
922        // protocol, and saying so is more useful than calling it dead.
923        Err(CliError::Refused { message, .. }) => (
924            true,
925            format!("reachable, but the health check reports: {message}"),
926        ),
927        Err(error) => (false, error.to_string()),
928    }
929}
930
931/// Writes either the human text or the JSON object.
932fn emit(out: &mut dyn Write, json: bool, text: &str, value: impl FnOnce() -> serde_json::Value) {
933    if json {
934        let _ = writeln!(
935            out,
936            "{}",
937            serde_json::to_string_pretty(&value()).unwrap_or_else(|_| "{}".to_owned())
938        );
939    } else {
940        let _ = writeln!(out, "{text}");
941    }
942    let _ = out.flush();
943}
944
945#[cfg(test)]
946mod tests {
947    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
948
949    use super::*;
950    use crate::runtime_dir::write_pidfile;
951
952    fn scratch(name: &str) -> PathBuf {
953        let dir =
954            std::env::temp_dir().join(format!("astrs-cli-cluster-{}-{name}", std::process::id()));
955        let _ = std::fs::remove_dir_all(&dir);
956        std::fs::create_dir_all(&dir).unwrap();
957        dir
958    }
959
960    /// A pid that cannot be alive: `kill(0, …)` addresses a process group,
961    /// so `Pid::from_raw(0)` is rejected outright.
962    const DEAD_PID: u32 = 0;
963
964    #[test]
965    fn a_missing_pidfile_reads_as_absent() {
966        let dir = scratch("absent");
967        let status = ProcessStatus::probe("coordinator", &dir.join(COORDINATOR_PIDFILE));
968        assert!(!status.running);
969        assert_eq!(status.pid, None);
970        assert!(status.note.contains("no pidfile"), "{status:?}");
971    }
972
973    #[test]
974    fn a_live_pidfile_reads_as_running_with_its_address() {
975        let dir = scratch("live");
976        let path = dir.join(COORDINATOR_PIDFILE);
977        write_pidfile(
978            &path,
979            &PidFile::listening(std::process::id(), "127.0.0.1:7407"),
980        )
981        .unwrap();
982        let status = ProcessStatus::probe("coordinator", &path);
983        assert!(status.running);
984        assert_eq!(status.address.as_deref(), Some("127.0.0.1:7407"));
985        assert!(status.to_line().contains("127.0.0.1:7407"));
986    }
987
988    #[test]
989    fn a_stale_pidfile_is_named_as_stale_rather_than_running() {
990        let dir = scratch("stale");
991        let path = dir.join(DAEMON_PIDFILE);
992        write_pidfile(&path, &PidFile::new(DEAD_PID)).unwrap();
993        let status = ProcessStatus::probe("daemon", &path);
994        assert!(!status.running);
995        assert!(status.note.contains("stale"), "{status:?}");
996    }
997
998    #[test]
999    fn stopping_something_that_is_not_running_is_not_an_error() {
1000        let status = ProcessStatus::absent("daemon");
1001        assert_eq!(stop_process(&status), StopOutcome::NotRunning);
1002    }
1003
1004    #[test]
1005    fn a_real_child_is_terminated_by_the_stop_ladder() {
1006        // `sleep` ignores nothing, so `SIGTERM` alone must be enough; the
1007        // point is that the ladder observes the exit rather than returning
1008        // before the process is actually gone.
1009        let child = Command::new("/bin/sh")
1010            .args(["-c", "sleep 30"])
1011            .stdout(Stdio::null())
1012            .stderr(Stdio::null())
1013            .spawn()
1014            .expect("a child");
1015        let status = ProcessStatus::from_pidfile("daemon", &PidFile::new(child.id()));
1016        assert!(status.running);
1017        assert_eq!(stop_process(&status), StopOutcome::Terminated);
1018        // `stop_process` reaped it (see `wait_for_exit`), so this only
1019        // releases the handle's own bookkeeping.
1020        let mut child = child;
1021        let _ = child.try_wait();
1022    }
1023
1024    #[test]
1025    fn down_reports_an_empty_runtime_directory_without_failing() {
1026        let dir = scratch("down-empty");
1027        let args = DownArgs {
1028            runtime_dir: Some(dir.clone()),
1029            working_dir: Some(dir),
1030            ..DownArgs::default()
1031        };
1032        let mut out = Vec::new();
1033        let report = down(&mut out, &args).expect("a report");
1034        assert!(!report.stopped_anything());
1035        assert_eq!(report.daemon.0, StopOutcome::NotRunning);
1036        let text = String::from_utf8(out).unwrap();
1037        assert!(text.contains("nothing was running here"), "{text}");
1038    }
1039
1040    #[test]
1041    fn down_removes_the_pidfiles_it_finds() {
1042        let dir = scratch("down-pidfiles");
1043        let coordinator = dir.join(COORDINATOR_PIDFILE);
1044        let daemon = dir.join(DAEMON_PIDFILE);
1045        write_pidfile(&coordinator, &PidFile::new(DEAD_PID)).unwrap();
1046        write_pidfile(&daemon, &PidFile::new(DEAD_PID)).unwrap();
1047
1048        let args = DownArgs {
1049            runtime_dir: Some(dir.clone()),
1050            working_dir: Some(dir),
1051            ..DownArgs::default()
1052        };
1053        let report = down(&mut Vec::new(), &args).expect("a report");
1054        assert!(!report.stopped_anything());
1055        assert!(!coordinator.exists(), "a stale pidfile must be cleared");
1056        assert!(!daemon.exists());
1057    }
1058
1059    #[test]
1060    fn status_of_an_empty_runtime_directory_is_unreachable_but_reported() {
1061        let dir = scratch("status-empty");
1062        let args = StatusArgs {
1063            runtime_dir: Some(dir.clone()),
1064            working_dir: Some(dir),
1065            // Port 1 is refused instantly rather than timing out.
1066            coordinator: Some("127.0.0.1:1".to_owned()),
1067            ..StatusArgs::default()
1068        };
1069        let mut out = Vec::new();
1070        let report = status(&mut out, &args).expect("a report");
1071        assert!(!report.reachable);
1072        assert_eq!(report.exit_code(), 1);
1073        assert!(!report.coordinator.running);
1074        let text = String::from_utf8(out).unwrap();
1075        assert!(text.contains("PROCESS"), "{text}");
1076    }
1077
1078    #[test]
1079    fn status_json_carries_both_processes_and_the_exit_code() {
1080        let dir = scratch("status-json");
1081        let args = StatusArgs {
1082            runtime_dir: Some(dir.clone()),
1083            working_dir: Some(dir),
1084            coordinator: Some("127.0.0.1:1".to_owned()),
1085            json: true,
1086            ..StatusArgs::default()
1087        };
1088        let mut out = Vec::new();
1089        let _ = status(&mut out, &args).expect("a report");
1090        let text = String::from_utf8(out).unwrap();
1091        let value: serde_json::Value = serde_json::from_str(&text).unwrap();
1092        assert_eq!(value["reachable"], false);
1093        assert_eq!(value["exit_code"], 1);
1094        assert_eq!(value["coordinator"]["running"], false);
1095        assert_eq!(value["daemon"]["running"], false);
1096    }
1097
1098    #[test]
1099    fn recreating_the_store_under_a_live_coordinator_is_refused() {
1100        let dir = scratch("recreate-live");
1101        write_pidfile(
1102            &dir.join(COORDINATOR_PIDFILE),
1103            &PidFile::listening(std::process::id(), "127.0.0.1:7407"),
1104        )
1105        .unwrap();
1106        let args = UpArgs {
1107            runtime_dir: Some(dir.clone()),
1108            working_dir: Some(dir),
1109            recreate_store: true,
1110            ..UpArgs::default()
1111        };
1112        let error = up(&mut Vec::new(), &args).unwrap_err();
1113        match error {
1114            CliError::AlreadyRunning { process, .. } => assert_eq!(process, "coordinator"),
1115            other => panic!("expected AlreadyRunning, got {other}"),
1116        }
1117    }
1118
1119    #[test]
1120    fn a_token_argument_is_spelled_as_a_file_whenever_there_is_one() {
1121        assert_eq!(
1122            TokenArgument::File(PathBuf::from("/x/.astrs-token")).to_argv(),
1123            vec!["--token-file".to_owned(), "/x/.astrs-token".to_owned()]
1124        );
1125        assert_eq!(
1126            TokenArgument::Inline("ab".to_owned()).to_argv(),
1127            vec!["--token".to_owned(), "ab".to_owned()]
1128        );
1129    }
1130
1131    #[test]
1132    fn a_child_that_never_becomes_ready_reports_its_own_log() {
1133        let dir = scratch("never-ready");
1134        std::fs::write(
1135            log_path(&dir, "coordinator"),
1136            "astrs coordinator: Address already in use (os error 48)\n",
1137        )
1138        .unwrap();
1139        let error = wait_for_ready(
1140            "coordinator",
1141            &dir.join(COORDINATOR_PIDFILE),
1142            &dir,
1143            true,
1144            Duration::from_millis(80),
1145        )
1146        .unwrap_err();
1147        match error {
1148            CliError::Cluster {
1149                action,
1150                process,
1151                reason,
1152            } => {
1153                assert_eq!(action, "start");
1154                assert_eq!(process, "coordinator");
1155                assert!(reason.contains("Address already in use"), "{reason}");
1156            }
1157            other => panic!("expected Cluster, got {other}"),
1158        }
1159    }
1160
1161    #[test]
1162    fn a_pidfile_without_an_address_is_not_ready_for_a_listener_but_is_for_a_daemon() {
1163        let dir = scratch("ready-address");
1164        let path = dir.join(DAEMON_PIDFILE);
1165        write_pidfile(&path, &PidFile::new(std::process::id())).unwrap();
1166
1167        let ready = wait_for_ready("daemon", &path, &dir, false, Duration::from_millis(80))
1168            .expect("a daemon needs no address to be ready");
1169        assert_eq!(ready.pid, Some(std::process::id()));
1170
1171        let error = wait_for_ready("coordinator", &path, &dir, true, Duration::from_millis(80))
1172            .unwrap_err();
1173        assert!(matches!(error, CliError::Cluster { .. }), "{error}");
1174    }
1175
1176    #[test]
1177    fn a_log_tail_of_a_missing_file_says_so() {
1178        let dir = scratch("log-tail");
1179        let text = log_tail(&dir.join("nothing.log"));
1180        assert!(text.contains("nothing was written"), "{text}");
1181    }
1182
1183    #[test]
1184    fn a_log_tail_keeps_only_the_last_lines() {
1185        let dir = scratch("log-tail-long");
1186        let path = dir.join("long.log");
1187        let body: String = (0..40).map(|index| format!("line {index}\n")).collect();
1188        std::fs::write(&path, body).unwrap();
1189        let tail = log_tail(&path);
1190        assert!(tail.contains("line 39"), "{tail}");
1191        assert!(!tail.contains("line 0\n"), "{tail}");
1192        assert_eq!(tail.lines().count(), 12);
1193    }
1194
1195    #[test]
1196    fn every_stop_outcome_has_a_stable_word() {
1197        assert_eq!(StopOutcome::NotRunning.as_str(), "not-running");
1198        assert_eq!(StopOutcome::Terminated.as_str(), "terminated");
1199        assert_eq!(StopOutcome::Killed.as_str(), "killed");
1200        assert_eq!(StopOutcome::Unreachable.as_str(), "unreachable");
1201    }
1202}