1use std::io::Write;
43use std::path::{Path, PathBuf};
44use std::sync::Arc;
45use std::time::{Duration, Instant};
46
47use astrs_daemon::{RunOptions, run_dataflow_with};
48use astrs_manifest::Manifest;
49use astrs_wire::{DataflowResult, DataflowStatus, LogLevel, NodeExitCause, NodeId};
50use tokio::sync::watch;
51
52use crate::command::log_stream::{
53 LogFilter, LogStyle, StreamItem, TerminalLogSink, prefix_width, render,
54};
55use crate::command::signals::Signals;
56use crate::error::CliError;
57
58pub const EXIT_OK: i32 = 0;
60
61pub const EXIT_FAILED: i32 = 1;
63
64pub const EXIT_INCOMPLETE: i32 = 2;
68
69pub const ABANDON_AFTER: Duration = Duration::from_secs(2);
76
77#[derive(Debug, Clone)]
79pub struct RunArgs {
80 pub manifest_path: PathBuf,
82 pub deterministic: bool,
87 pub from_recording: Option<PathBuf>,
92 pub speed: Option<f64>,
97 pub exit_when_nodes_finish: bool,
99 pub skip_build: bool,
101 pub level: Option<LogLevel>,
103 pub working_dir: Option<PathBuf>,
106 pub runtime_dir: Option<PathBuf>,
108 pub timeout: Option<Duration>,
110 pub grace: Option<Duration>,
112 pub json: bool,
114 pub color: bool,
116}
117
118impl RunArgs {
119 #[must_use]
121 pub fn new(manifest_path: impl Into<PathBuf>) -> Self {
122 Self {
123 manifest_path: manifest_path.into(),
124 deterministic: false,
125 from_recording: None,
126 speed: None,
127 exit_when_nodes_finish: false,
128 skip_build: false,
129 level: None,
130 working_dir: None,
131 runtime_dir: None,
132 timeout: None,
133 grace: None,
134 json: false,
135 color: false,
136 }
137 }
138}
139
140#[derive(Debug, Clone)]
142pub struct RunReport {
143 pub result: DataflowResult,
145 pub dropped: u64,
147 pub printed: usize,
149 pub abandoned: bool,
151}
152
153impl RunReport {
154 #[must_use]
164 pub fn exit_code(&self) -> i32 {
165 if self.abandoned {
166 return EXIT_INCOMPLETE;
167 }
168 match self.result.status {
169 DataflowStatus::Finished if !self.result.has_failures() => EXIT_OK,
170 DataflowStatus::Finished | DataflowStatus::Failed => EXIT_FAILED,
171 _ => EXIT_INCOMPLETE,
172 }
173 }
174
175 #[must_use]
177 pub fn summary(&self) -> String {
178 let nodes = self.result.node_results.len();
179 let failed = self
180 .result
181 .node_results
182 .values()
183 .filter(|cause| cause.is_failure())
184 .count();
185 let mut text = format!(
186 "dataflow {} {} ({nodes} node(s), {failed} failed)",
187 self.result.dataflow,
188 status_word(self.result.status)
189 );
190 if !self.result.message.is_empty() {
191 text.push_str(&format!(": {}", self.result.message));
192 }
193 if self.dropped > 0 {
194 text.push_str(&format!(
195 "\n{} log line(s) were dropped because the terminal could not keep up",
196 self.dropped
197 ));
198 }
199 text
200 }
201
202 #[must_use]
204 pub fn to_json(&self) -> serde_json::Value {
205 let nodes: serde_json::Map<String, serde_json::Value> = self
206 .result
207 .node_results
208 .iter()
209 .map(|(node, cause)| {
210 (
211 node.as_str().to_owned(),
212 serde_json::json!({
213 "cause": cause.to_string(),
214 "failed": cause.is_failure(),
215 }),
216 )
217 })
218 .collect();
219 serde_json::json!({
220 "dataflow": self.result.dataflow.to_string(),
221 "status": status_word(self.result.status),
222 "message": self.result.message,
223 "failed": self.result.has_failures(),
224 "abandoned": self.abandoned,
225 "exit_code": self.exit_code(),
226 "printed_lines": self.printed,
227 "dropped_lines": self.dropped,
228 "nodes": nodes,
229 })
230 }
231}
232
233const fn status_word(status: DataflowStatus) -> &'static str {
235 match status {
236 DataflowStatus::Pending => "pending",
237 DataflowStatus::Building => "building",
238 DataflowStatus::Ready => "ready",
239 DataflowStatus::Starting => "starting",
240 DataflowStatus::Running => "running",
241 DataflowStatus::Stopping => "stopping",
242 DataflowStatus::Finished => "finished",
243 DataflowStatus::Failed => "failed",
244 _ => "unknown",
247 }
248}
249
250pub fn run(out: &mut dyn Write, args: &RunArgs) -> Result<RunReport, CliError> {
271 if args.deterministic && args.from_recording.is_none() {
272 return Err(CliError::DeterministicNeedsRecording);
273 }
274
275 let mut manifest = load_manifest(&args.manifest_path, args.exit_when_nodes_finish)?;
276 if let Some(recording) = &args.from_recording {
277 stand_down_recorded_producers(&mut manifest, recording)?;
278 }
279 let working_dir = args
280 .working_dir
281 .clone()
282 .unwrap_or_else(|| manifest_dir(&args.manifest_path));
283 let runtime_dir = crate::runtime_dir::runtime_dir(args.runtime_dir.as_deref());
284 crate::runtime_dir::ensure_dir(&runtime_dir)?;
285
286 let style = LogStyle {
287 color: args.color,
288 prefix_width: prefix_width(manifest.nodes.iter().map(|node| node.id.as_str())),
289 elapsed: true,
290 };
291 let filter = LogFilter::new().with_min_level(args.level.unwrap_or(LogLevel::Trace));
292
293 let runtime = tokio::runtime::Builder::new_multi_thread()
294 .enable_all()
295 .build()
296 .map_err(|source| CliError::io(&args.manifest_path, source))?;
297
298 let report = runtime.block_on(execute(
299 out,
300 &manifest,
301 args,
302 working_dir,
303 runtime_dir,
304 &filter,
305 style,
306 ))?;
307
308 if args.json {
309 let _ = writeln!(
310 out,
311 "{}",
312 serde_json::to_string_pretty(&report.to_json()).unwrap_or_else(|_| "{}".to_owned())
313 );
314 } else {
315 let _ = writeln!(out, "{}", report.summary());
316 }
317 Ok(report)
318}
319
320fn load_manifest(path: &Path, exit_when_nodes_finish: bool) -> Result<Manifest, CliError> {
326 let mut manifest = Manifest::from_yaml_file(path)?;
327 manifest.validate()?;
328 if exit_when_nodes_finish {
329 manifest.exit_when_nodes_finish = true;
330 }
331 Ok(manifest)
332}
333
334fn stand_down_recorded_producers(
352 manifest: &mut Manifest,
353 recording: &Path,
354) -> Result<(), CliError> {
355 let (reader, _recovery) = astrs_recording::Reader::open_or_recover(recording)?;
356 let producers: std::collections::BTreeSet<&str> = reader
357 .index()
358 .iter()
359 .map(|entry| entry.node.as_str())
360 .collect();
361 for node in &mut manifest.nodes {
362 if producers.contains(node.id.as_str()) {
363 node.path = Some(astrs_manifest::DYNAMIC_PATH_SENTINEL.to_owned());
364 }
365 }
366 Ok(())
367}
368
369fn manifest_dir(path: &Path) -> PathBuf {
371 path.parent()
372 .filter(|parent| !parent.as_os_str().is_empty())
373 .map_or_else(|| PathBuf::from("."), Path::to_path_buf)
374}
375
376async fn execute(
378 out: &mut dyn Write,
379 manifest: &Manifest,
380 args: &RunArgs,
381 working_dir: PathBuf,
382 runtime_dir: PathBuf,
383 filter: &LogFilter,
384 style: LogStyle,
385) -> Result<RunReport, CliError> {
386 let (sink, mut receiver) = TerminalLogSink::new();
387 let sink = Arc::new(sink);
388 let (cancel, cancel_rx) = watch::channel(false);
389
390 let mut options = RunOptions::new()
391 .with_working_dir(working_dir)
392 .with_runtime_dir(runtime_dir)
393 .with_build(!args.skip_build)
394 .with_report_sink(sink.clone())
395 .with_cancel(cancel_rx)
396 .with_deterministic(args.deterministic)
397 .with_replay_recording(args.from_recording.clone(), args.speed);
398 if let Some(timeout) = args.timeout {
399 options = options.with_timeout(timeout);
400 }
401 if let Some(grace) = args.grace {
402 options = options.with_finish_grace(grace);
403 }
404
405 let started = Instant::now();
406 let mut printed = 0usize;
407 let mut interrupts = 0u32;
408 let mut abandon: Option<std::pin::Pin<Box<tokio::time::Sleep>>> = None;
409
410 let mut signals = Signals::install();
411 let run_future = run_dataflow_with(manifest, options);
412 tokio::pin!(run_future);
413
414 let outcome: Option<Result<DataflowResult, astrs_daemon::DaemonError>> = loop {
415 tokio::select! {
416 biased;
420 result = &mut run_future => break Some(result),
421 () = wait_for(&mut abandon) => break None,
422 Some(item) = receiver.recv() => {
423 printed += print_one(out, &item, filter, style, started);
424 }
425 () = signals.next() => {
426 interrupts += 1;
427 let (level, message) = interrupt_notice(interrupts);
428 printed += print_one(
429 out,
430 &StreamItem::Notice { level, node: None, message },
431 filter,
432 style,
433 started,
434 );
435 if interrupts == 1 {
436 let _ = cancel.send(true);
438 } else if abandon.is_none() {
439 abandon = Some(Box::pin(tokio::time::sleep(ABANDON_AFTER)));
440 }
441 }
442 }
443 };
444
445 while let Ok(item) = receiver.try_recv() {
449 printed += print_one(out, &item, filter, style, started);
450 }
451
452 let (result, abandoned) = match outcome {
453 Some(result) => (result?, false),
454 None => (abandoned_result(manifest), true),
455 };
456 Ok(RunReport {
457 result,
458 dropped: sink.as_ref().dropped_count(),
459 printed,
460 abandoned,
461 })
462}
463
464async fn wait_for(timer: &mut Option<std::pin::Pin<Box<tokio::time::Sleep>>>) {
467 match timer.as_mut() {
468 Some(sleep) => sleep.as_mut().await,
469 None => std::future::pending().await,
470 }
471}
472
473fn interrupt_notice(count: u32) -> (LogLevel, String) {
475 if count == 1 {
476 (
477 LogLevel::Warn,
478 "interrupted: asking every node to stop (press again to give up waiting)".to_owned(),
479 )
480 } else {
481 (
482 LogLevel::Warn,
483 format!(
484 "interrupted again: abandoning the wait in {:.0}s; children are process-group \
485 killed and carry the orphan guard",
486 ABANDON_AFTER.as_secs_f64()
487 ),
488 )
489 }
490}
491
492fn print_one(
495 out: &mut dyn Write,
496 item: &StreamItem,
497 filter: &LogFilter,
498 style: LogStyle,
499 started: Instant,
500) -> usize {
501 if !filter.accepts(item) {
502 return 0;
503 }
504 if writeln!(out, "{}", render(item, style, started.elapsed())).is_err() {
505 return 0;
506 }
507 let _ = out.flush();
508 1
509}
510
511fn abandoned_result(manifest: &Manifest) -> DataflowResult {
519 let mut result = DataflowResult::new(
520 astrs_wire::DataflowId::generate(),
521 astrs_time::HlcTimestamp::new(0, 0),
522 );
523 result.status = DataflowStatus::Stopping;
524 result.message = "the wait was abandoned after a second interrupt".to_owned();
525 for node in &manifest.nodes {
526 if let Ok(id) = NodeId::new(&node.id) {
527 result.record(
528 id,
529 NodeExitCause::Killed {
530 reason: "the CLI abandoned the wait after a second interrupt".to_owned(),
531 },
532 );
533 }
534 }
535 result
536}
537
538trait DroppedCount {
541 fn dropped_count(&self) -> u64;
543}
544
545impl DroppedCount for TerminalLogSink {
546 fn dropped_count(&self) -> u64 {
547 astrs_daemon::health::ReportSink::dropped(self)
548 }
549}
550
551#[cfg(test)]
552mod tests {
553 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
554
555 use super::*;
556 use astrs_wire::DataflowId;
557
558 fn scratch(name: &str) -> PathBuf {
559 let dir = std::env::temp_dir().join(format!("astrs-cli-run-{}-{name}", std::process::id()));
560 let _ = std::fs::remove_dir_all(&dir);
561 std::fs::create_dir_all(&dir).unwrap();
562 dir
563 }
564
565 fn write_manifest(dir: &Path, yaml: &str) -> PathBuf {
566 let path = dir.join("dataflow.yml");
567 std::fs::write(&path, yaml).unwrap();
568 path
569 }
570
571 fn report(status: DataflowStatus, failures: &[(&str, NodeExitCause)]) -> RunReport {
572 let mut result = DataflowResult::new(
573 DataflowId::from_u128(1),
574 astrs_time::HlcTimestamp::new(1, 0),
575 );
576 result.status = status;
577 for (node, cause) in failures {
578 result.record(NodeId::new(*node).unwrap(), cause.clone());
579 }
580 RunReport {
581 result,
582 dropped: 0,
583 printed: 0,
584 abandoned: false,
585 }
586 }
587
588 #[test]
589 fn a_clean_finish_exits_zero() {
590 let report = report(DataflowStatus::Finished, &[("a", NodeExitCause::Success)]);
591 assert_eq!(report.exit_code(), EXIT_OK);
592 }
593
594 #[test]
595 fn a_failed_node_exits_one_even_when_the_status_says_finished() {
596 let report = report(
597 DataflowStatus::Finished,
598 &[("a", NodeExitCause::ExitCode { code: 3 })],
599 );
600 assert_eq!(report.exit_code(), EXIT_FAILED);
601 }
602
603 #[test]
604 fn a_failed_dataflow_exits_one() {
605 let report = report(
606 DataflowStatus::Failed,
607 &[("a", NodeExitCause::ExitCode { code: 3 })],
608 );
609 assert_eq!(report.exit_code(), EXIT_FAILED);
610 }
611
612 #[test]
613 fn a_non_terminal_status_exits_two() {
614 for status in [
615 DataflowStatus::Pending,
616 DataflowStatus::Building,
617 DataflowStatus::Ready,
618 DataflowStatus::Starting,
619 DataflowStatus::Running,
620 DataflowStatus::Stopping,
621 ] {
622 let report = report(status, &[]);
623 assert_eq!(report.exit_code(), EXIT_INCOMPLETE, "{status:?}");
624 }
625 }
626
627 #[test]
628 fn an_abandoned_wait_exits_two_whatever_the_status_says() {
629 let mut report = report(DataflowStatus::Finished, &[("a", NodeExitCause::Success)]);
630 report.abandoned = true;
631 assert_eq!(report.exit_code(), EXIT_INCOMPLETE);
632 }
633
634 #[test]
635 fn the_summary_names_the_status_and_the_failure_count() {
636 let report = report(
637 DataflowStatus::Failed,
638 &[
639 ("a", NodeExitCause::Success),
640 ("b", NodeExitCause::ExitCode { code: 1 }),
641 ],
642 );
643 let text = report.summary();
644 assert!(text.contains("failed"), "{text}");
645 assert!(text.contains("2 node(s)"), "{text}");
646 assert!(text.contains("1 failed"), "{text}");
647 }
648
649 #[test]
650 fn the_summary_mentions_dropped_lines_only_when_there_were_some() {
651 let mut report = report(DataflowStatus::Finished, &[]);
652 assert!(!report.summary().contains("dropped"));
653 report.dropped = 5;
654 assert!(report.summary().contains("5 log line(s)"));
655 }
656
657 #[test]
658 fn the_json_form_carries_every_node_and_the_exit_code() {
659 let report = report(
660 DataflowStatus::Failed,
661 &[("cam", NodeExitCause::ExitCode { code: 2 })],
662 );
663 let json = report.to_json();
664 assert_eq!(json["status"], "failed");
665 assert_eq!(json["exit_code"], EXIT_FAILED);
666 assert_eq!(json["nodes"]["cam"]["failed"], true);
667 }
668
669 #[test]
670 fn deterministic_without_a_recording_is_refused_before_anything_runs() {
671 let dir = scratch("deterministic");
672 let path = write_manifest(&dir, "nodes:\n - id: a\n path: /usr/bin/true\n");
673 let mut args = RunArgs::new(path);
674 args.deterministic = true;
675 let error = run(&mut Vec::new(), &args).unwrap_err();
676 assert!(
677 matches!(error, CliError::DeterministicNeedsRecording),
678 "expected DeterministicNeedsRecording, got {error}"
679 );
680 assert!(error.to_string().contains("--from-recording"), "{error}");
681 }
682
683 #[test]
684 fn a_manifest_that_does_not_parse_is_reported_by_the_cli() {
685 let dir = scratch("bad-manifest");
686 let path = write_manifest(&dir, "nodes: [oops");
687 let error = run(&mut Vec::new(), &RunArgs::new(path)).unwrap_err();
688 assert!(matches!(error, CliError::Manifest(_)), "{error}");
689 }
690
691 #[test]
692 fn a_manifest_that_fails_validation_is_reported_by_the_cli() {
693 let dir = scratch("invalid-manifest");
694 let path = write_manifest(
696 &dir,
697 "nodes:\n - id: a\n path: /usr/bin/true\n inputs:\n in: ghost/out\n",
698 );
699 let error = run(&mut Vec::new(), &RunArgs::new(path)).unwrap_err();
700 assert!(matches!(error, CliError::Validation(_)), "{error}");
701 }
702
703 #[test]
704 fn the_manifest_directory_is_the_default_working_directory() {
705 assert_eq!(
706 manifest_dir(Path::new("/a/b/dataflow.yml")),
707 PathBuf::from("/a/b")
708 );
709 assert_eq!(manifest_dir(Path::new("dataflow.yml")), PathBuf::from("."));
710 }
711
712 #[test]
713 fn exit_when_nodes_finish_overrides_the_manifest() {
714 let dir = scratch("exit-flag");
715 let path = write_manifest(&dir, "nodes:\n - id: a\n path: /usr/bin/true\n");
716 let manifest = load_manifest(&path, false).unwrap();
717 assert!(!manifest.exit_when_nodes_finish);
718 let manifest = load_manifest(&path, true).unwrap();
719 assert!(manifest.exit_when_nodes_finish);
720 }
721
722 #[test]
723 fn an_abandoned_result_records_every_node_as_killed() {
724 let manifest =
725 Manifest::from_yaml_str("nodes:\n - id: a\n path: /usr/bin/true\n").unwrap();
726 let result = abandoned_result(&manifest);
727 assert_eq!(result.status, DataflowStatus::Stopping);
728 assert_eq!(result.node_results.len(), 1);
729 assert!(result.has_failures());
730 }
731
732 #[test]
733 fn a_trivial_graph_runs_to_a_clean_finish_and_exits_zero() {
734 let dir = scratch("trivial");
735 let path = write_manifest(
736 &dir,
737 "exit_when_nodes_finish: true\nnodes:\n - id: a\n path: /bin/sh\n args: [\"-c\", \"echo hello-from-run\"]\n",
738 );
739 let mut args = RunArgs::new(path);
740 args.skip_build = true;
741 args.timeout = Some(Duration::from_secs(20));
742 args.grace = Some(Duration::from_millis(200));
743 args.runtime_dir = Some(dir.clone());
744 args.working_dir = Some(dir);
745
746 let mut out: Vec<u8> = Vec::new();
747 let report = run(&mut out, &args).expect("a report");
748 let text = String::from_utf8(out).unwrap();
749 assert_eq!(report.exit_code(), EXIT_OK, "{text}");
750 assert!(
751 text.contains("hello-from-run"),
752 "the node's own output must reach the terminal: {text}"
753 );
754 assert!(text.contains("finished"), "{text}");
755 }
756
757 #[test]
758 fn a_failing_build_line_stops_the_run_before_spawning() {
759 let dir = scratch("bad-build");
760 let path = write_manifest(
761 &dir,
762 "nodes:\n - id: a\n path: /usr/bin/true\n build: /bin/sh -c 'exit 4'\n",
763 );
764 let mut args = RunArgs::new(path);
765 args.timeout = Some(Duration::from_secs(20));
766 args.runtime_dir = Some(dir.clone());
767 args.working_dir = Some(dir);
768 let error = run(&mut Vec::new(), &args).unwrap_err();
769 assert!(matches!(error, CliError::Daemon(_)), "{error}");
770 }
771
772 #[test]
773 fn skip_build_reaches_the_spawn_a_failing_build_would_have_blocked() {
774 let dir = scratch("skip-build");
775 let path = write_manifest(
776 &dir,
777 "exit_when_nodes_finish: true\nnodes:\n - id: a\n path: /usr/bin/true\n build: /bin/sh -c 'exit 4'\n",
778 );
779 let mut args = RunArgs::new(path);
780 args.skip_build = true;
781 args.timeout = Some(Duration::from_secs(20));
782 args.grace = Some(Duration::from_millis(200));
783 args.runtime_dir = Some(dir.clone());
784 args.working_dir = Some(dir);
785 let report = run(&mut Vec::new(), &args).expect("a report");
788 assert_eq!(report.result.node_results.len(), 1);
789 }
790}