Skip to main content

astrs_cli/command/
lifecycle.rs

1//! The cluster-side lifecycle verbs: `start`, `stop`, `restart`,
2//! `destroy`, `clean` (blueprint §17).
3//!
4//! Each is one [`ControlRequest`] over the shared
5//! [`crate::command::client::Client`], plus the presentation a terminal and
6//! a script each need. Nothing here decides *policy* — placement, grace
7//! ladders, artefact reclamation all belong to the coordinator; this module
8//! decides which request a user's words mean, and how to say what came back.
9//!
10//! ```text
11//!   astrs start d.yml --attach ─► Start{Manifest}   ─► Started{dataflow}
12//!                                 LogSubscribe      ─► Log frames ──► terminal
13//!                                 Check (polled)    ─► DataflowResult ─► exit code
14//!
15//!   astrs stop <ref>           ─► Stop / StopByName ─► Ok
16//!   astrs restart <ref>        ─► Restart / RestartByName
17//!   astrs destroy [<ref>]      ─► Destroy{force}
18//!   astrs clean [<ref>]        ─► Clean{artifacts, logs}
19//! ```
20//!
21//! # An id or a name, never both spellings at one call site
22//!
23//! Every one of these verbs takes "the dataflow" as one positional word.
24//! The protocol has *two* requests for most of them — `Stop{dataflow}` and
25//! `StopByName{name}` — so [`crate::command::client::DataflowRef`] does the
26//! classification once and each verb picks the matching request. A user
27//! never types `--by-name`.
28
29use std::io::Write;
30use std::path::{Path, PathBuf};
31use std::time::{Duration, Instant};
32
33use astrs_wire::{
34    ControlReply, ControlRequest, DataflowId, DataflowResult, DataflowSource, DataflowStatus,
35    DurationMs, FrameKind, LogFrame, LogLevel, LogQuery, LogRecord, WireDecode,
36};
37
38use crate::command::client::{
39    Client, DataflowRef, Endpoint, new_subscription_id, reply_name, runtime,
40};
41use crate::command::log_stream::{LogFilter, LogStyle, StreamItem, prefix_width, render};
42use crate::error::CliError;
43
44/// How often an attached `start` asks whether the dataflow has finished.
45///
46/// The coordinator has no "tell me when it ends" push for a CLI (§7.3's
47/// control family answers requests; only subscriptions push), so an attached
48/// run polls. Fast enough that a short graph's exit is not perceptibly
49/// delayed, slow enough that a long one costs nothing.
50pub const ATTACH_POLL_INTERVAL: Duration = Duration::from_millis(200);
51
52/// `astrs start`'s arguments.
53#[derive(Debug, Clone)]
54pub struct StartArgs {
55    /// The manifest to start.
56    pub manifest_path: PathBuf,
57    /// A name for the run.
58    pub name: Option<String>,
59    /// `--machine` placement overrides, as `MACHINE` or `NODE=MACHINE`
60    /// (§8.3 `deploy.machine`).
61    pub machines: Vec<String>,
62    /// Stream logs until it finishes, then exit with its severity.
63    pub attach: bool,
64    /// Hide attached output below this level.
65    pub level: Option<LogLevel>,
66    /// Emit JSON rather than a human summary.
67    pub json: bool,
68    /// Colorize attached output.
69    pub color: bool,
70}
71
72impl StartArgs {
73    /// The arguments for starting `manifest_path` with every default.
74    #[must_use]
75    pub fn new(manifest_path: impl Into<PathBuf>) -> Self {
76        Self {
77            manifest_path: manifest_path.into(),
78            name: None,
79            machines: Vec::new(),
80            attach: false,
81            level: None,
82            json: false,
83            color: false,
84        }
85    }
86}
87
88/// What one `astrs start` did.
89#[derive(Debug, Clone)]
90pub struct StartReport {
91    /// The id the coordinator assigned.
92    pub dataflow: DataflowId,
93    /// The name it was started under.
94    pub name: Option<String>,
95    /// The final verdict, when `--attach` waited for one.
96    pub result: Option<DataflowResult>,
97}
98
99impl StartReport {
100    /// The process exit code: the dataflow's severity when attached, `0`
101    /// when the start merely succeeded.
102    #[must_use]
103    pub fn exit_code(&self) -> i32 {
104        match &self.result {
105            None => 0,
106            Some(result) => match result.status {
107                DataflowStatus::Finished if !result.has_failures() => 0,
108                DataflowStatus::Finished | DataflowStatus::Failed => 1,
109                _ => 2,
110            },
111        }
112    }
113
114    /// The `--json` form.
115    #[must_use]
116    pub fn to_json(&self) -> serde_json::Value {
117        serde_json::json!({
118            "dataflow": self.dataflow.to_string(),
119            "name": self.name,
120            "attached": self.result.is_some(),
121            "status": self.result.as_ref().map(|result| result.status.as_str()),
122            "failed": self.result.as_ref().is_some_and(DataflowResult::has_failures),
123            "exit_code": self.exit_code(),
124        })
125    }
126}
127
128/// Starts a dataflow on a running cluster.
129///
130/// # Errors
131///
132/// - [`CliError::Manifest`] / [`CliError::Validation`] if the manifest is
133///   not usable — checked *locally* before it is sent, so a typo costs a
134///   round trip's worth of nothing rather than a coordinator error about a
135///   plan.
136/// - [`CliError::NoCluster`] if nothing is listening.
137/// - [`CliError::Refused`] if the coordinator refused the start.
138pub fn start(
139    out: &mut dyn Write,
140    endpoint: &Endpoint,
141    args: &StartArgs,
142) -> Result<StartReport, CliError> {
143    let mut manifest = astrs_manifest::Manifest::from_yaml_file(&args.manifest_path)?;
144    manifest.validate()?;
145    // The file's own bytes are sent unchanged when nothing overrides them,
146    // so a manifest with comments, anchors or key ordering the coordinator
147    // never sees round-trips exactly as written. Re-serializing is the
148    // *override* path's cost, not everybody's.
149    let yaml = if args.machines.is_empty() {
150        std::fs::read_to_string(&args.manifest_path)
151            .map_err(|source| CliError::io(&args.manifest_path, source))?
152    } else {
153        apply_machine_overrides(&mut manifest, &args.machines)?;
154        manifest.to_yaml()?
155    };
156    let working_dir = manifest_dir(&args.manifest_path);
157
158    let style = LogStyle {
159        color: args.color,
160        prefix_width: prefix_width(manifest.nodes.iter().map(|node| node.id.as_str())),
161        elapsed: true,
162    };
163    let filter = LogFilter::new().with_min_level(args.level.unwrap_or(LogLevel::Trace));
164
165    let runtime = runtime()?;
166    let report = runtime.block_on(async {
167        let mut client = Client::connect(endpoint).await?;
168        let request = ControlRequest::Start {
169            source: DataflowSource::Manifest {
170                yaml,
171                working_dir: Some(working_dir.display().to_string()),
172            },
173            name: args.name.clone(),
174            detach: !args.attach,
175        };
176        let (dataflow, name) = match client.request("start", &request).await? {
177            ControlReply::Started { dataflow, name } => (dataflow, name),
178            other => {
179                return Err(CliError::UnexpectedReply {
180                    request: "start",
181                    reply: reply_name(&other),
182                });
183            }
184        };
185        if !args.attach {
186            return Ok(StartReport {
187                dataflow,
188                name,
189                result: None,
190            });
191        }
192        let result = attach(out, endpoint, &mut client, dataflow, &filter, style).await?;
193        Ok(StartReport {
194            dataflow,
195            name,
196            result: Some(result),
197        })
198    })?;
199
200    if args.json {
201        let _ = writeln!(
202            out,
203            "{}",
204            serde_json::to_string_pretty(&report.to_json()).unwrap_or_else(|_| "{}".to_owned())
205        );
206    } else {
207        let _ = writeln!(out, "{}", start_summary(&report));
208    }
209    Ok(report)
210}
211
212/// The human line a finished `start` prints.
213fn start_summary(report: &StartReport) -> String {
214    let named = report
215        .name
216        .as_ref()
217        .map_or_else(String::new, |name| format!(" ({name})"));
218    match &report.result {
219        None => format!("started {}{named}", report.dataflow),
220        Some(result) => format!(
221            "{}{named} {}: {}",
222            report.dataflow,
223            result.status.as_str(),
224            if result.message.is_empty() {
225                "no message".to_owned()
226            } else {
227                result.message.clone()
228            }
229        ),
230    }
231}
232
233/// Streams a started dataflow's logs on a second connection until it reaches
234/// a terminal status, polling for that status on `control`, then backfills
235/// whatever the live stream was too late to see.
236///
237/// Two connections rather than one: pushed `Log` frames and answered
238/// `ControlReply` frames share a socket, so a single connection would have
239/// to demultiplex them by kind while a request is in flight — and a poll
240/// that consumed a pushed log frame while waiting for its own reply would
241/// lose it. One socket subscribes and only reads; the other only asks.
242///
243/// # Why a subscription alone is not enough
244///
245/// `LogSubscribe` is the *push* path (§13): the coordinator fans out records
246/// as daemons produce them and delivers nothing retroactively. A short
247/// dataflow — the whole `examples/` estate, and every conformance graph —
248/// can start, run and finish inside the round trip that opens the
249/// subscription, so an attached start that only subscribed printed nothing
250/// at all and then reported the verdict for a run whose output the user
251/// never saw. The *pull* path (`ControlRequest::Logs`, answered from each
252/// daemon's `LogHistory` ring) is what remembers, so the two are used
253/// together: subscribe first so a long run streams as it happens, pull once
254/// at the end so a short one is not lost. [`LogTail`] de-duplicates the
255/// overlap the two paths necessarily share.
256async fn attach(
257    out: &mut dyn Write,
258    endpoint: &Endpoint,
259    control: &mut Client,
260    dataflow: DataflowId,
261    filter: &LogFilter,
262    style: LogStyle,
263) -> Result<DataflowResult, CliError> {
264    let mut logs = Client::connect(endpoint).await?;
265    let subscription = new_subscription_id();
266    logs.request_ok(
267        "logs -f",
268        &ControlRequest::LogSubscribe {
269            dataflow: Some(dataflow),
270            node: None,
271            query: LogQuery::default(),
272            subscription,
273        },
274    )
275    .await?;
276
277    let mut tail = LogTail::new(filter.clone(), style);
278    let mut poll = tokio::time::interval(ATTACH_POLL_INTERVAL);
279    poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
280    let result = loop {
281        tokio::select! {
282            frame = logs.next_frame() => {
283                match frame? {
284                    Some(frame) => tail.push_frame(out, &frame),
285                    // The coordinator closed the log connection; the poll
286                    // below still decides the outcome.
287                    None => tokio::time::sleep(ATTACH_POLL_INTERVAL).await,
288                }
289            }
290            _ = poll.tick() => {
291                if let Some(result) = check_terminal(control, dataflow).await? {
292                    break result;
293                }
294            }
295        }
296    };
297    backfill(out, control, dataflow, &mut tail).await;
298    Ok(result)
299}
300
301/// Prints every record the live subscription missed, on the connection that
302/// was already asking questions.
303///
304/// Deliberately infallible: the dataflow's verdict is already decided and is
305/// what the caller's exit code is made of, so a coordinator that cannot
306/// answer a *history* query — the dataflow was cleaned up between the verdict
307/// and this call, a daemon dropped out — must not turn a successful run into
308/// a failed command. The records are a courtesy; the verdict is the contract.
309async fn backfill(
310    out: &mut dyn Write,
311    control: &mut Client,
312    dataflow: DataflowId,
313    tail: &mut LogTail,
314) {
315    let reply = control
316        .request(
317            "logs",
318            &ControlRequest::Logs {
319                dataflow,
320                node: None,
321                query: LogQuery::default(),
322            },
323        )
324        .await;
325    if let Ok(ControlReply::Logs { records, .. }) = reply {
326        tail.backfill(out, records);
327    }
328}
329
330/// One attached run's log output: renders records, and remembers which ones
331/// it has already rendered.
332///
333/// The push and pull paths overlap by construction — every record the
334/// subscription delivered is also in the daemon's history ring — so the
335/// closing pull would otherwise reprint the whole run. The key is
336/// `(timestamp, node, message)` rather than the record itself because
337/// [`astrs_wire::LogRecord`] is not ordered or hashed, and because those
338/// three fields are what a reader would call "the same line": an HLC stamp
339/// is unique per producer per event (§14), so two records sharing all three
340/// are the same record having travelled two paths.
341struct LogTail {
342    /// Which records to render at all.
343    filter: LogFilter,
344    /// How to render them.
345    style: LogStyle,
346    /// When the attach began, for the elapsed-time prefix.
347    started: Instant,
348    /// The keys of every record already rendered.
349    seen: std::collections::BTreeSet<(astrs_time::HlcTimestamp, String, String)>,
350}
351
352impl LogTail {
353    /// An empty tail, starting its elapsed clock now.
354    fn new(filter: LogFilter, style: LogStyle) -> Self {
355        Self {
356            filter,
357            style,
358            started: Instant::now(),
359            seen: std::collections::BTreeSet::new(),
360        }
361    }
362
363    /// The de-duplication key for one record.
364    fn key(record: &LogRecord) -> (astrs_time::HlcTimestamp, String, String) {
365        (
366            record.timestamp,
367            record
368                .node
369                .as_ref()
370                .map(|node| node.as_str().to_owned())
371                .unwrap_or_default(),
372            record.message.clone(),
373        )
374    }
375
376    /// Renders one pushed frame, ignoring kinds this verb did not subscribe
377    /// to.
378    ///
379    /// A frame that fails to decode is dropped rather than ending the stream:
380    /// a log tail must survive one malformed record from a node whose output
381    /// was not valid UTF-8, not exit because of it.
382    fn push_frame(&mut self, out: &mut dyn Write, frame: &astrs_wire::Frame) {
383        if frame.kind() != FrameKind::Log {
384            return;
385        }
386        let Ok(log) = LogFrame::decode_exact(frame.payload()) else {
387            return;
388        };
389        self.render(out, log.record);
390    }
391
392    /// Renders every record of a closing history pull that the live stream
393    /// did not already show, oldest first.
394    fn backfill(&mut self, out: &mut dyn Write, records: Vec<LogRecord>) {
395        let mut records = records;
396        records.sort_by_key(|record| record.timestamp);
397        for record in records {
398            self.render(out, record);
399        }
400    }
401
402    /// Renders one record, once.
403    fn render(&mut self, out: &mut dyn Write, record: LogRecord) {
404        if !self.seen.insert(Self::key(&record)) {
405            return;
406        }
407        let item = StreamItem::Record(Box::new(record));
408        if !self.filter.accepts(&item) {
409            return;
410        }
411        let _ = writeln!(out, "{}", render(&item, self.style, self.started.elapsed()));
412        let _ = out.flush();
413    }
414}
415
416/// One `Check`, answering `Some` only for a terminal verdict.
417async fn check_terminal(
418    client: &mut Client,
419    dataflow: DataflowId,
420) -> Result<Option<DataflowResult>, CliError> {
421    match client
422        .request(
423            "status",
424            &ControlRequest::Check {
425                dataflow: Some(dataflow),
426            },
427        )
428        .await?
429    {
430        ControlReply::DataflowResult { result } => Ok(Some(*result)),
431        // `Ok` means "known, still going" — the coordinator's own answer for
432        // a dataflow that has not reached a verdict yet.
433        ControlReply::Ok => Ok(None),
434        other => Err(CliError::UnexpectedReply {
435            request: "status",
436            reply: reply_name(&other),
437        }),
438    }
439}
440
441/// `astrs stop`'s arguments.
442#[derive(Debug, Clone)]
443pub struct StopArgs {
444    /// The dataflow to stop.
445    pub dataflow: DataflowRef,
446    /// How long nodes get to finish.
447    pub grace: Option<Duration>,
448    /// Emit JSON rather than a human line.
449    pub json: bool,
450}
451
452/// Stops a dataflow.
453///
454/// # Errors
455///
456/// As [`Client::request`], plus [`CliError::UnexpectedReply`].
457pub fn stop(out: &mut dyn Write, endpoint: &Endpoint, args: &StopArgs) -> Result<(), CliError> {
458    let grace = args.grace.map(DurationMs::from_duration);
459    let request = match &args.dataflow {
460        DataflowRef::Id(dataflow) => ControlRequest::Stop {
461            dataflow: *dataflow,
462            grace,
463        },
464        DataflowRef::Name(name) => ControlRequest::StopByName {
465            name: name.clone(),
466            grace,
467        },
468    };
469    one_shot(out, endpoint, "stop", &request, args.json, |()| {
470        format!("stopped {}", args.dataflow)
471    })
472}
473
474/// `astrs restart`'s arguments.
475#[derive(Debug, Clone)]
476pub struct RestartArgs {
477    /// The dataflow to restart.
478    pub dataflow: DataflowRef,
479    /// Re-run the `build:` lines first.
480    pub rebuild: bool,
481    /// Emit JSON rather than a human line.
482    pub json: bool,
483}
484
485/// Restarts a dataflow.
486///
487/// # Errors
488///
489/// As [`Client::request`], plus [`CliError::UnexpectedReply`].
490pub fn restart(
491    out: &mut dyn Write,
492    endpoint: &Endpoint,
493    args: &RestartArgs,
494) -> Result<(), CliError> {
495    let request = match &args.dataflow {
496        DataflowRef::Id(dataflow) => ControlRequest::Restart {
497            dataflow: *dataflow,
498            rebuild: args.rebuild,
499        },
500        DataflowRef::Name(name) => ControlRequest::RestartByName {
501            name: name.clone(),
502            rebuild: args.rebuild,
503        },
504    };
505    one_shot(out, endpoint, "restart", &request, args.json, |()| {
506        format!("restarted {}", args.dataflow)
507    })
508}
509
510/// `astrs destroy`'s arguments.
511#[derive(Debug, Clone)]
512pub struct DestroyArgs {
513    /// The dataflow to destroy; every one when absent.
514    pub dataflow: Option<DataflowRef>,
515    /// Destroy even while dataflows are running.
516    pub force: bool,
517    /// Emit JSON rather than a human line.
518    pub json: bool,
519}
520
521/// Destroys a dataflow, or the whole cluster's dataflows.
522///
523/// A named dataflow is stopped first and then cleaned: the protocol's
524/// [`ControlRequest::Destroy`] is cluster-wide by design (§24.1), so
525/// "destroy this one" is expressed as the two requests that mean it rather
526/// than by inventing a variant the wire does not have.
527///
528/// # Errors
529///
530/// As [`Client::request`], plus [`CliError::UnexpectedReply`].
531pub fn destroy(
532    out: &mut dyn Write,
533    endpoint: &Endpoint,
534    args: &DestroyArgs,
535) -> Result<(), CliError> {
536    let Some(reference) = args.dataflow.clone() else {
537        return one_shot(
538            out,
539            endpoint,
540            "destroy",
541            &ControlRequest::Destroy { force: args.force },
542            args.json,
543            |()| "destroyed every dataflow".to_owned(),
544        );
545    };
546
547    let runtime = runtime()?;
548    let dataflow = reference.clone();
549    runtime.block_on(async {
550        let mut client = Client::connect(endpoint).await?;
551        let stop_request = match &dataflow {
552            DataflowRef::Id(id) => ControlRequest::Stop {
553                dataflow: *id,
554                grace: None,
555            },
556            DataflowRef::Name(name) => ControlRequest::StopByName {
557                name: name.clone(),
558                grace: None,
559            },
560        };
561        client.request_ok("destroy", &stop_request).await?;
562        if let DataflowRef::Id(id) = &dataflow {
563            client
564                .request_ok(
565                    "destroy",
566                    &ControlRequest::Clean {
567                        dataflow: Some(*id),
568                        artifacts: true,
569                        logs: false,
570                    },
571                )
572                .await?;
573        }
574        Ok::<(), CliError>(())
575    })?;
576
577    emit(
578        out,
579        args.json,
580        &format!("destroyed {reference}"),
581        || serde_json::json!({ "destroyed": reference.to_string() }),
582    );
583    Ok(())
584}
585
586/// `astrs clean`'s arguments.
587#[derive(Debug, Clone)]
588pub struct CleanArgs {
589    /// The finished dataflow to clean; every finished one when absent.
590    pub dataflow: Option<DataflowRef>,
591    /// Also delete build artefacts.
592    pub artifacts: bool,
593    /// Also delete captured logs.
594    pub logs: bool,
595    /// Emit JSON rather than a human line.
596    pub json: bool,
597}
598
599/// Reclaims finished dataflows' resources.
600///
601/// A named (rather than id'd) dataflow costs one `List` first: the wire's
602/// [`ControlRequest::Clean`] takes an id or nothing at all, so the name a
603/// user types has to become an id somewhere, and doing it here keeps
604/// `astrs clean my-flow` working exactly like `astrs stop my-flow`.
605///
606/// # Errors
607///
608/// As [`Client::request`], plus [`CliError::UnknownDataflow`] when a name
609/// resolves to nothing.
610pub fn clean(out: &mut dyn Write, endpoint: &Endpoint, args: &CleanArgs) -> Result<(), CliError> {
611    let runtime = runtime()?;
612    let reference = args.dataflow.clone();
613    let cleaned = runtime.block_on(async {
614        let mut client = Client::connect(endpoint).await?;
615        let dataflow = match &reference {
616            Some(reference) => Some(client.resolve(reference).await?),
617            None => None,
618        };
619        client
620            .request_ok(
621                "clean",
622                &ControlRequest::Clean {
623                    dataflow,
624                    artifacts: args.artifacts,
625                    logs: args.logs,
626                },
627            )
628            .await?;
629        Ok::<Option<DataflowId>, CliError>(dataflow)
630    })?;
631
632    let text = cleaned.map_or_else(
633        || "cleaned every finished dataflow".to_owned(),
634        |id| format!("cleaned {id}"),
635    );
636    emit(
637        out,
638        args.json,
639        &text,
640        || serde_json::json!({ "ok": true, "message": text }),
641    );
642    Ok(())
643}
644
645/// Connects, sends one request that must answer `Ok`, and reports it.
646fn one_shot(
647    out: &mut dyn Write,
648    endpoint: &Endpoint,
649    label: &'static str,
650    request: &ControlRequest,
651    json: bool,
652    summary: impl FnOnce(()) -> String,
653) -> Result<(), CliError> {
654    let runtime = runtime()?;
655    runtime.block_on(async {
656        let mut client = Client::connect(endpoint).await?;
657        client.request_ok(label, request).await
658    })?;
659    let text = summary(());
660    emit(
661        out,
662        json,
663        &text,
664        || serde_json::json!({ "ok": true, "message": text }),
665    );
666    Ok(())
667}
668
669/// Writes either the human line or the JSON object.
670fn emit(out: &mut dyn Write, json: bool, text: &str, value: impl FnOnce() -> serde_json::Value) {
671    if json {
672        let _ = writeln!(
673            out,
674            "{}",
675            serde_json::to_string_pretty(&value()).unwrap_or_else(|_| "{}".to_owned())
676        );
677    } else {
678        let _ = writeln!(out, "{text}");
679    }
680}
681
682/// Applies `astrs start --machine` placement overrides to `manifest`
683/// (blueprint §8.3 `deploy.machine`, §17).
684///
685/// Two forms, and the difference between them is deliberate:
686///
687/// | Form | Meaning |
688/// |---|---|
689/// | `MACHINE` | the default for every node the manifest left unplaced |
690/// | `NODE=MACHINE` | pins one node, *overriding* whatever the manifest said |
691///
692/// A bare machine is a **default**, not an override, because a manifest that
693/// pins one node to the machine with the camera on it meant that, and a
694/// deployment-time flag that silently moved it would break the graph. A
695/// `NODE=MACHINE` is an override, because it names the node it is moving and
696/// therefore cannot be an accident.
697///
698/// # Errors
699///
700/// [`CliError::BadArgument`] for an empty machine name, a `NODE=MACHINE`
701/// naming a node the manifest does not declare, or more than one bare
702/// machine (which of the two would be the default is not a question this
703/// answers by picking).
704pub fn apply_machine_overrides(
705    manifest: &mut astrs_manifest::Manifest,
706    specs: &[String],
707) -> Result<(), CliError> {
708    let bad = |value: &str, reason: &str| CliError::BadArgument {
709        flag: "machine",
710        value: value.to_owned(),
711        reason: reason.to_owned(),
712    };
713
714    let mut default_machine: Option<&str> = None;
715    let mut pinned: std::collections::BTreeMap<&str, &str> = std::collections::BTreeMap::new();
716    for spec in specs {
717        match spec.split_once('=') {
718            Some((node, machine)) => {
719                if node.is_empty() || machine.is_empty() {
720                    return Err(bad(spec, "expected NODE=MACHINE, with both halves present"));
721                }
722                pinned.insert(node, machine);
723            }
724            None => {
725                if spec.is_empty() {
726                    return Err(bad(spec, "a machine name may not be empty"));
727                }
728                if default_machine.is_some_and(|existing| existing != spec.as_str()) {
729                    return Err(bad(
730                        spec,
731                        "only one bare --machine may be given; use NODE=MACHINE to place                          individual nodes",
732                    ));
733                }
734                default_machine = Some(spec);
735            }
736        }
737    }
738
739    let declared: std::collections::BTreeSet<&str> =
740        manifest.nodes.iter().map(|node| node.id.as_str()).collect();
741    for node in pinned.keys() {
742        if !declared.contains(node) {
743            return Err(bad(node, "the manifest declares no node with that id"));
744        }
745    }
746
747    for node in &mut manifest.nodes {
748        let wanted = pinned.get(node.id.as_str()).copied().or_else(|| {
749            let already_placed = node
750                .deploy
751                .as_ref()
752                .is_some_and(|deploy| deploy.machine.is_some());
753            if already_placed {
754                None
755            } else {
756                default_machine
757            }
758        });
759        let Some(machine) = wanted else {
760            continue;
761        };
762        node.deploy
763            .get_or_insert_with(astrs_manifest::Deploy::default)
764            .machine = Some(machine.to_owned());
765    }
766    Ok(())
767}
768
769/// The directory a manifest's relative paths resolve against.
770fn manifest_dir(path: &Path) -> PathBuf {
771    path.parent()
772        .filter(|parent| !parent.as_os_str().is_empty())
773        .map_or_else(|| PathBuf::from("."), Path::to_path_buf)
774}
775
776#[cfg(test)]
777mod tests {
778    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
779
780    use super::*;
781    use astrs_time::HlcTimestamp;
782    use astrs_wire::{AuthToken, NodeExitCause, NodeId, SubscriptionId, WireEncode};
783
784    fn endpoint() -> Endpoint {
785        Endpoint::new(
786            std::net::SocketAddr::from(([127, 0, 0, 1], 1)),
787            AuthToken::ZERO,
788        )
789    }
790
791    /// A two-node manifest with one node already placed, so every
792    /// override rule below has something to be right or wrong about.
793    fn placed_manifest() -> astrs_manifest::Manifest {
794        astrs_manifest::Manifest::from_yaml_str(
795            "\
796nodes:
797  - id: camera
798    path: ./camera
799    outputs: [frames]
800  - id: planner
801    path: ./planner
802    deploy:
803      machine: robot-9
804    inputs:
805      frames: camera/frames
806",
807        )
808        .unwrap()
809    }
810
811    fn machine_of(manifest: &astrs_manifest::Manifest, node: &str) -> Option<String> {
812        manifest
813            .nodes
814            .iter()
815            .find(|candidate| candidate.id == node)
816            .and_then(|candidate| candidate.deploy.as_ref())
817            .and_then(|deploy| deploy.machine.clone())
818    }
819
820    #[test]
821    fn no_machine_override_changes_nothing() {
822        let mut manifest = placed_manifest();
823        apply_machine_overrides(&mut manifest, &[]).unwrap();
824        assert_eq!(machine_of(&manifest, "camera"), None);
825        assert_eq!(machine_of(&manifest, "planner").as_deref(), Some("robot-9"));
826    }
827
828    #[test]
829    fn a_bare_machine_places_only_the_unplaced_nodes() {
830        let mut manifest = placed_manifest();
831        apply_machine_overrides(&mut manifest, &["robot-1".to_owned()]).unwrap();
832        assert_eq!(machine_of(&manifest, "camera").as_deref(), Some("robot-1"));
833        assert_eq!(
834            machine_of(&manifest, "planner").as_deref(),
835            Some("robot-9"),
836            "a bare --machine is a default, never an override"
837        );
838    }
839
840    #[test]
841    fn a_node_scoped_machine_overrides_the_manifest() {
842        let mut manifest = placed_manifest();
843        apply_machine_overrides(&mut manifest, &["planner=robot-2".to_owned()]).unwrap();
844        assert_eq!(machine_of(&manifest, "planner").as_deref(), Some("robot-2"));
845        assert_eq!(machine_of(&manifest, "camera"), None);
846    }
847
848    #[test]
849    fn the_two_forms_combine() {
850        let mut manifest = placed_manifest();
851        apply_machine_overrides(
852            &mut manifest,
853            &["robot-1".to_owned(), "planner=robot-2".to_owned()],
854        )
855        .unwrap();
856        assert_eq!(machine_of(&manifest, "camera").as_deref(), Some("robot-1"));
857        assert_eq!(machine_of(&manifest, "planner").as_deref(), Some("robot-2"));
858    }
859
860    #[test]
861    fn an_overridden_manifest_still_serializes_back_to_yaml() {
862        let mut manifest = placed_manifest();
863        apply_machine_overrides(&mut manifest, &["robot-1".to_owned()]).unwrap();
864        let yaml = manifest.to_yaml().unwrap();
865        let round_tripped = astrs_manifest::Manifest::from_yaml_str(&yaml).unwrap();
866        assert_eq!(
867            machine_of(&round_tripped, "camera").as_deref(),
868            Some("robot-1"),
869            "the coordinator is sent yaml, so the override has to survive it"
870        );
871    }
872
873    #[test]
874    fn an_unknown_node_is_a_typed_argument_error() {
875        let mut manifest = placed_manifest();
876        let err = apply_machine_overrides(&mut manifest, &["ghost=robot-1".to_owned()])
877            .expect_err("no such node");
878        assert!(matches!(
879            err,
880            CliError::BadArgument {
881                flag: "machine",
882                ..
883            }
884        ));
885    }
886
887    #[test]
888    fn two_different_bare_machines_are_refused_rather_than_guessed_at() {
889        let mut manifest = placed_manifest();
890        let err =
891            apply_machine_overrides(&mut manifest, &["robot-1".to_owned(), "robot-2".to_owned()])
892                .expect_err("ambiguous");
893        assert!(matches!(
894            err,
895            CliError::BadArgument {
896                flag: "machine",
897                ..
898            }
899        ));
900        // The same one twice is not ambiguous at all.
901        apply_machine_overrides(&mut manifest, &["robot-1".to_owned(), "robot-1".to_owned()])
902            .unwrap();
903    }
904
905    #[test]
906    fn a_malformed_pair_is_refused() {
907        let mut manifest = placed_manifest();
908        for spec in ["=robot-1", "planner=", ""] {
909            assert!(
910                apply_machine_overrides(&mut manifest, &[spec.to_owned()]).is_err(),
911                "{spec:?} must be refused"
912            );
913        }
914    }
915
916    fn scratch(name: &str) -> PathBuf {
917        let dir =
918            std::env::temp_dir().join(format!("astrs-cli-lifecycle-{}-{name}", std::process::id()));
919        let _ = std::fs::remove_dir_all(&dir);
920        std::fs::create_dir_all(&dir).unwrap();
921        dir
922    }
923
924    fn result(status: DataflowStatus, failed: bool) -> DataflowResult {
925        let mut result = DataflowResult::new(DataflowId::from_u128(1), HlcTimestamp::new(1, 0));
926        result.status = status;
927        if failed {
928            result.record(
929                NodeId::new("a").unwrap(),
930                NodeExitCause::ExitCode { code: 1 },
931            );
932        }
933        result
934    }
935
936    #[test]
937    fn a_detached_start_always_exits_zero() {
938        let report = StartReport {
939            dataflow: DataflowId::from_u128(1),
940            name: Some("flow".to_owned()),
941            result: None,
942        };
943        assert_eq!(report.exit_code(), 0);
944        assert!(report.to_json()["attached"] == false);
945        assert!(start_summary(&report).contains("started"));
946    }
947
948    #[test]
949    fn an_attached_start_reports_the_dataflow_s_own_severity() {
950        for (status, failed, code) in [
951            (DataflowStatus::Finished, false, 0),
952            (DataflowStatus::Finished, true, 1),
953            (DataflowStatus::Failed, true, 1),
954            (DataflowStatus::Running, false, 2),
955        ] {
956            let report = StartReport {
957                dataflow: DataflowId::from_u128(1),
958                name: None,
959                result: Some(result(status, failed)),
960            };
961            assert_eq!(report.exit_code(), code, "{status:?} failed={failed}");
962        }
963    }
964
965    #[test]
966    fn the_start_json_carries_the_exit_code_and_the_name() {
967        let report = StartReport {
968            dataflow: DataflowId::from_u128(7),
969            name: Some("perception".to_owned()),
970            result: Some(result(DataflowStatus::Failed, true)),
971        };
972        let json = report.to_json();
973        assert_eq!(json["name"], "perception");
974        assert_eq!(json["exit_code"], 1);
975        assert_eq!(json["failed"], true);
976    }
977
978    #[test]
979    fn start_validates_the_manifest_before_dialling_anything() {
980        let dir = scratch("bad-manifest");
981        let path = dir.join("d.yml");
982        std::fs::write(
983            &path,
984            "nodes:\n  - id: a\n    path: /usr/bin/true\n    inputs:\n      in: ghost/out\n",
985        )
986        .unwrap();
987        // Port 1 would refuse instantly, so a `NoCluster` here would prove
988        // the dial happened first; `Validation` proves it did not.
989        let error = start(&mut Vec::new(), &endpoint(), &StartArgs::new(path)).unwrap_err();
990        assert!(matches!(error, CliError::Validation(_)), "{error}");
991    }
992
993    #[test]
994    fn start_reports_a_missing_manifest_as_a_manifest_error() {
995        let dir = scratch("absent");
996        let error = start(
997            &mut Vec::new(),
998            &endpoint(),
999            &StartArgs::new(dir.join("nope.yml")),
1000        )
1001        .unwrap_err();
1002        assert!(matches!(error, CliError::Manifest(_)), "{error}");
1003    }
1004
1005    #[test]
1006    fn stop_by_id_and_by_name_are_different_requests() {
1007        // The classification the whole module hangs off, asserted directly:
1008        // an id becomes `Stop`, anything else becomes `StopByName`.
1009        let id = DataflowId::generate();
1010        assert!(matches!(
1011            DataflowRef::parse(&id.to_string()),
1012            DataflowRef::Id(_)
1013        ));
1014        assert!(matches!(
1015            DataflowRef::parse("perception"),
1016            DataflowRef::Name(_)
1017        ));
1018    }
1019
1020    #[test]
1021    fn every_lifecycle_verb_reports_a_dead_cluster_rather_than_hanging() {
1022        let endpoint = endpoint();
1023        let checks: Vec<Box<dyn Fn() -> Result<(), CliError>>> = vec![
1024            Box::new(|| {
1025                stop(
1026                    &mut Vec::new(),
1027                    &endpoint,
1028                    &StopArgs {
1029                        dataflow: DataflowRef::Name("f".to_owned()),
1030                        grace: None,
1031                        json: false,
1032                    },
1033                )
1034            }),
1035            Box::new(|| {
1036                restart(
1037                    &mut Vec::new(),
1038                    &endpoint,
1039                    &RestartArgs {
1040                        dataflow: DataflowRef::Name("f".to_owned()),
1041                        rebuild: false,
1042                        json: false,
1043                    },
1044                )
1045            }),
1046            Box::new(|| {
1047                destroy(
1048                    &mut Vec::new(),
1049                    &endpoint,
1050                    &DestroyArgs {
1051                        dataflow: None,
1052                        force: true,
1053                        json: false,
1054                    },
1055                )
1056            }),
1057            Box::new(|| {
1058                clean(
1059                    &mut Vec::new(),
1060                    &endpoint,
1061                    &CleanArgs {
1062                        dataflow: None,
1063                        artifacts: false,
1064                        logs: false,
1065                        json: false,
1066                    },
1067                )
1068            }),
1069        ];
1070        for check in checks {
1071            let error = check().unwrap_err();
1072            assert!(matches!(error, CliError::NoCluster { .. }), "{error}");
1073        }
1074    }
1075
1076    /// The plain rendering style every log test below asserts against.
1077    fn plain() -> LogStyle {
1078        LogStyle {
1079            color: false,
1080            prefix_width: 0,
1081            elapsed: false,
1082        }
1083    }
1084
1085    /// One node record, at `stamp`, saying `message`.
1086    fn record(stamp: u64, message: &str) -> LogRecord {
1087        LogRecord::new(HlcTimestamp::new(stamp, 0), LogLevel::Info, message)
1088            .with_node(NodeId::new("cam").unwrap())
1089    }
1090
1091    /// `record` wrapped in the pushed frame a subscription delivers.
1092    fn log_frame(record: LogRecord) -> astrs_wire::Frame {
1093        let log = LogFrame {
1094            subscription: SubscriptionId::new(7),
1095            record,
1096        };
1097        astrs_wire::Frame::new(
1098            FrameKind::Log,
1099            astrs_wire::FrameFlags::EMPTY,
1100            log.encode_to_vec().unwrap(),
1101        )
1102        .unwrap()
1103    }
1104
1105    #[test]
1106    fn a_non_log_frame_is_ignored_by_the_log_tail() {
1107        let frame = astrs_wire::Frame::new(
1108            FrameKind::ControlReply,
1109            astrs_wire::FrameFlags::EMPTY,
1110            b"not a log".to_vec(),
1111        )
1112        .unwrap();
1113        let mut out = Vec::new();
1114        LogTail::new(LogFilter::new(), LogStyle::default()).push_frame(&mut out, &frame);
1115        assert!(out.is_empty());
1116    }
1117
1118    #[test]
1119    fn a_log_frame_is_rendered_through_the_shared_filter() {
1120        let frame = log_frame(record(1, "hello"));
1121
1122        let mut out = Vec::new();
1123        LogTail::new(LogFilter::new(), plain()).push_frame(&mut out, &frame);
1124        assert_eq!(String::from_utf8(out).unwrap().trim_end(), "[cam] hello");
1125
1126        // …and a filter that excludes it prints nothing.
1127        let mut out = Vec::new();
1128        LogTail::new(
1129            LogFilter::new().with_min_level(LogLevel::Error),
1130            LogStyle::default(),
1131        )
1132        .push_frame(&mut out, &frame);
1133        assert!(out.is_empty());
1134    }
1135
1136    #[test]
1137    fn a_malformed_log_frame_is_dropped_rather_than_ending_the_stream() {
1138        let frame =
1139            astrs_wire::Frame::new(FrameKind::Log, astrs_wire::FrameFlags::EMPTY, vec![0xFF; 4])
1140                .unwrap();
1141        let mut out = Vec::new();
1142        LogTail::new(LogFilter::new(), LogStyle::default()).push_frame(&mut out, &frame);
1143        assert!(out.is_empty());
1144    }
1145
1146    #[test]
1147    fn the_closing_pull_prints_only_what_the_live_stream_missed() {
1148        // The push path delivered the second line; the history ring holds
1149        // both. An attached start must end up having printed each exactly
1150        // once, in timestamp order for the part it backfilled.
1151        let mut tail = LogTail::new(LogFilter::new(), plain());
1152        let mut out = Vec::new();
1153        tail.push_frame(&mut out, &log_frame(record(2, "second")));
1154        tail.backfill(
1155            &mut out,
1156            vec![record(2, "second"), record(1, "first"), record(3, "third")],
1157        );
1158
1159        let text = String::from_utf8(out).unwrap();
1160        let lines: Vec<&str> = text.lines().collect();
1161        assert_eq!(lines, ["[cam] second", "[cam] first", "[cam] third"]);
1162    }
1163
1164    #[test]
1165    fn a_backfill_with_nothing_new_prints_nothing() {
1166        let mut tail = LogTail::new(LogFilter::new(), plain());
1167        let mut out = Vec::new();
1168        tail.push_frame(&mut out, &log_frame(record(1, "only")));
1169        out.clear();
1170        tail.backfill(&mut out, vec![record(1, "only")]);
1171        assert!(
1172            out.is_empty(),
1173            "the whole point of the seen-set: a short run is backfilled, a long one is not reprinted"
1174        );
1175    }
1176
1177    #[test]
1178    fn two_nodes_logging_the_same_words_at_the_same_stamp_are_two_lines() {
1179        // The de-duplication key includes the node, so a fan-out of identical
1180        // messages is not collapsed into one.
1181        let mut tail = LogTail::new(LogFilter::new(), plain());
1182        let mut out = Vec::new();
1183        let stamp = HlcTimestamp::new(4, 0);
1184        tail.backfill(
1185            &mut out,
1186            vec![
1187                LogRecord::new(stamp, LogLevel::Info, "ready").with_node(NodeId::new("a").unwrap()),
1188                LogRecord::new(stamp, LogLevel::Info, "ready").with_node(NodeId::new("b").unwrap()),
1189            ],
1190        );
1191        assert_eq!(String::from_utf8(out).unwrap().lines().count(), 2);
1192    }
1193}