use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use astrs_daemon::{RunOptions, run_dataflow_with};
use astrs_manifest::Manifest;
use astrs_wire::{DataflowResult, DataflowStatus, LogLevel, NodeExitCause, NodeId};
use tokio::sync::watch;
use crate::command::log_stream::{
LogFilter, LogStyle, StreamItem, TerminalLogSink, prefix_width, render,
};
use crate::command::signals::Signals;
use crate::error::CliError;
pub const EXIT_OK: i32 = 0;
pub const EXIT_FAILED: i32 = 1;
pub const EXIT_INCOMPLETE: i32 = 2;
pub const ABANDON_AFTER: Duration = Duration::from_secs(2);
#[derive(Debug, Clone)]
pub struct RunArgs {
pub manifest_path: PathBuf,
pub deterministic: bool,
pub from_recording: Option<PathBuf>,
pub speed: Option<f64>,
pub exit_when_nodes_finish: bool,
pub skip_build: bool,
pub level: Option<LogLevel>,
pub working_dir: Option<PathBuf>,
pub runtime_dir: Option<PathBuf>,
pub timeout: Option<Duration>,
pub grace: Option<Duration>,
pub json: bool,
pub color: bool,
}
impl RunArgs {
#[must_use]
pub fn new(manifest_path: impl Into<PathBuf>) -> Self {
Self {
manifest_path: manifest_path.into(),
deterministic: false,
from_recording: None,
speed: None,
exit_when_nodes_finish: false,
skip_build: false,
level: None,
working_dir: None,
runtime_dir: None,
timeout: None,
grace: None,
json: false,
color: false,
}
}
}
#[derive(Debug, Clone)]
pub struct RunReport {
pub result: DataflowResult,
pub dropped: u64,
pub printed: usize,
pub abandoned: bool,
}
impl RunReport {
#[must_use]
pub fn exit_code(&self) -> i32 {
if self.abandoned {
return EXIT_INCOMPLETE;
}
match self.result.status {
DataflowStatus::Finished if !self.result.has_failures() => EXIT_OK,
DataflowStatus::Finished | DataflowStatus::Failed => EXIT_FAILED,
_ => EXIT_INCOMPLETE,
}
}
#[must_use]
pub fn summary(&self) -> String {
let nodes = self.result.node_results.len();
let failed = self
.result
.node_results
.values()
.filter(|cause| cause.is_failure())
.count();
let mut text = format!(
"dataflow {} {} ({nodes} node(s), {failed} failed)",
self.result.dataflow,
status_word(self.result.status)
);
if !self.result.message.is_empty() {
text.push_str(&format!(": {}", self.result.message));
}
if self.dropped > 0 {
text.push_str(&format!(
"\n{} log line(s) were dropped because the terminal could not keep up",
self.dropped
));
}
text
}
#[must_use]
pub fn to_json(&self) -> serde_json::Value {
let nodes: serde_json::Map<String, serde_json::Value> = self
.result
.node_results
.iter()
.map(|(node, cause)| {
(
node.as_str().to_owned(),
serde_json::json!({
"cause": cause.to_string(),
"failed": cause.is_failure(),
}),
)
})
.collect();
serde_json::json!({
"dataflow": self.result.dataflow.to_string(),
"status": status_word(self.result.status),
"message": self.result.message,
"failed": self.result.has_failures(),
"abandoned": self.abandoned,
"exit_code": self.exit_code(),
"printed_lines": self.printed,
"dropped_lines": self.dropped,
"nodes": nodes,
})
}
}
const fn status_word(status: DataflowStatus) -> &'static str {
match status {
DataflowStatus::Pending => "pending",
DataflowStatus::Building => "building",
DataflowStatus::Ready => "ready",
DataflowStatus::Starting => "starting",
DataflowStatus::Running => "running",
DataflowStatus::Stopping => "stopping",
DataflowStatus::Finished => "finished",
DataflowStatus::Failed => "failed",
_ => "unknown",
}
}
pub fn run(out: &mut dyn Write, args: &RunArgs) -> Result<RunReport, CliError> {
if args.deterministic && args.from_recording.is_none() {
return Err(CliError::DeterministicNeedsRecording);
}
let mut manifest = load_manifest(&args.manifest_path, args.exit_when_nodes_finish)?;
if let Some(recording) = &args.from_recording {
stand_down_recorded_producers(&mut manifest, recording)?;
}
let working_dir = args
.working_dir
.clone()
.unwrap_or_else(|| manifest_dir(&args.manifest_path));
let runtime_dir = crate::runtime_dir::runtime_dir(args.runtime_dir.as_deref());
crate::runtime_dir::ensure_dir(&runtime_dir)?;
let style = LogStyle {
color: args.color,
prefix_width: prefix_width(manifest.nodes.iter().map(|node| node.id.as_str())),
elapsed: true,
};
let filter = LogFilter::new().with_min_level(args.level.unwrap_or(LogLevel::Trace));
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|source| CliError::io(&args.manifest_path, source))?;
let report = runtime.block_on(execute(
out,
&manifest,
args,
working_dir,
runtime_dir,
&filter,
style,
))?;
if args.json {
let _ = writeln!(
out,
"{}",
serde_json::to_string_pretty(&report.to_json()).unwrap_or_else(|_| "{}".to_owned())
);
} else {
let _ = writeln!(out, "{}", report.summary());
}
Ok(report)
}
fn load_manifest(path: &Path, exit_when_nodes_finish: bool) -> Result<Manifest, CliError> {
let mut manifest = Manifest::from_yaml_file(path)?;
manifest.validate()?;
if exit_when_nodes_finish {
manifest.exit_when_nodes_finish = true;
}
Ok(manifest)
}
fn stand_down_recorded_producers(
manifest: &mut Manifest,
recording: &Path,
) -> Result<(), CliError> {
let (reader, _recovery) = astrs_recording::Reader::open_or_recover(recording)?;
let producers: std::collections::BTreeSet<&str> = reader
.index()
.iter()
.map(|entry| entry.node.as_str())
.collect();
for node in &mut manifest.nodes {
if producers.contains(node.id.as_str()) {
node.path = Some(astrs_manifest::DYNAMIC_PATH_SENTINEL.to_owned());
}
}
Ok(())
}
fn manifest_dir(path: &Path) -> PathBuf {
path.parent()
.filter(|parent| !parent.as_os_str().is_empty())
.map_or_else(|| PathBuf::from("."), Path::to_path_buf)
}
async fn execute(
out: &mut dyn Write,
manifest: &Manifest,
args: &RunArgs,
working_dir: PathBuf,
runtime_dir: PathBuf,
filter: &LogFilter,
style: LogStyle,
) -> Result<RunReport, CliError> {
let (sink, mut receiver) = TerminalLogSink::new();
let sink = Arc::new(sink);
let (cancel, cancel_rx) = watch::channel(false);
let mut options = RunOptions::new()
.with_working_dir(working_dir)
.with_runtime_dir(runtime_dir)
.with_build(!args.skip_build)
.with_report_sink(sink.clone())
.with_cancel(cancel_rx)
.with_deterministic(args.deterministic)
.with_replay_recording(args.from_recording.clone(), args.speed);
if let Some(timeout) = args.timeout {
options = options.with_timeout(timeout);
}
if let Some(grace) = args.grace {
options = options.with_finish_grace(grace);
}
let started = Instant::now();
let mut printed = 0usize;
let mut interrupts = 0u32;
let mut abandon: Option<std::pin::Pin<Box<tokio::time::Sleep>>> = None;
let mut signals = Signals::install();
let run_future = run_dataflow_with(manifest, options);
tokio::pin!(run_future);
let outcome: Option<Result<DataflowResult, astrs_daemon::DaemonError>> = loop {
tokio::select! {
biased;
result = &mut run_future => break Some(result),
() = wait_for(&mut abandon) => break None,
Some(item) = receiver.recv() => {
printed += print_one(out, &item, filter, style, started);
}
() = signals.next() => {
interrupts += 1;
let (level, message) = interrupt_notice(interrupts);
printed += print_one(
out,
&StreamItem::Notice { level, node: None, message },
filter,
style,
started,
);
if interrupts == 1 {
let _ = cancel.send(true);
} else if abandon.is_none() {
abandon = Some(Box::pin(tokio::time::sleep(ABANDON_AFTER)));
}
}
}
};
while let Ok(item) = receiver.try_recv() {
printed += print_one(out, &item, filter, style, started);
}
let (result, abandoned) = match outcome {
Some(result) => (result?, false),
None => (abandoned_result(manifest), true),
};
Ok(RunReport {
result,
dropped: sink.as_ref().dropped_count(),
printed,
abandoned,
})
}
async fn wait_for(timer: &mut Option<std::pin::Pin<Box<tokio::time::Sleep>>>) {
match timer.as_mut() {
Some(sleep) => sleep.as_mut().await,
None => std::future::pending().await,
}
}
fn interrupt_notice(count: u32) -> (LogLevel, String) {
if count == 1 {
(
LogLevel::Warn,
"interrupted: asking every node to stop (press again to give up waiting)".to_owned(),
)
} else {
(
LogLevel::Warn,
format!(
"interrupted again: abandoning the wait in {:.0}s; children are process-group \
killed and carry the orphan guard",
ABANDON_AFTER.as_secs_f64()
),
)
}
}
fn print_one(
out: &mut dyn Write,
item: &StreamItem,
filter: &LogFilter,
style: LogStyle,
started: Instant,
) -> usize {
if !filter.accepts(item) {
return 0;
}
if writeln!(out, "{}", render(item, style, started.elapsed())).is_err() {
return 0;
}
let _ = out.flush();
1
}
fn abandoned_result(manifest: &Manifest) -> DataflowResult {
let mut result = DataflowResult::new(
astrs_wire::DataflowId::generate(),
astrs_time::HlcTimestamp::new(0, 0),
);
result.status = DataflowStatus::Stopping;
result.message = "the wait was abandoned after a second interrupt".to_owned();
for node in &manifest.nodes {
if let Ok(id) = NodeId::new(&node.id) {
result.record(
id,
NodeExitCause::Killed {
reason: "the CLI abandoned the wait after a second interrupt".to_owned(),
},
);
}
}
result
}
trait DroppedCount {
fn dropped_count(&self) -> u64;
}
impl DroppedCount for TerminalLogSink {
fn dropped_count(&self) -> u64 {
astrs_daemon::health::ReportSink::dropped(self)
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use astrs_wire::DataflowId;
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("astrs-cli-run-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
fn write_manifest(dir: &Path, yaml: &str) -> PathBuf {
let path = dir.join("dataflow.yml");
std::fs::write(&path, yaml).unwrap();
path
}
fn report(status: DataflowStatus, failures: &[(&str, NodeExitCause)]) -> RunReport {
let mut result = DataflowResult::new(
DataflowId::from_u128(1),
astrs_time::HlcTimestamp::new(1, 0),
);
result.status = status;
for (node, cause) in failures {
result.record(NodeId::new(*node).unwrap(), cause.clone());
}
RunReport {
result,
dropped: 0,
printed: 0,
abandoned: false,
}
}
#[test]
fn a_clean_finish_exits_zero() {
let report = report(DataflowStatus::Finished, &[("a", NodeExitCause::Success)]);
assert_eq!(report.exit_code(), EXIT_OK);
}
#[test]
fn a_failed_node_exits_one_even_when_the_status_says_finished() {
let report = report(
DataflowStatus::Finished,
&[("a", NodeExitCause::ExitCode { code: 3 })],
);
assert_eq!(report.exit_code(), EXIT_FAILED);
}
#[test]
fn a_failed_dataflow_exits_one() {
let report = report(
DataflowStatus::Failed,
&[("a", NodeExitCause::ExitCode { code: 3 })],
);
assert_eq!(report.exit_code(), EXIT_FAILED);
}
#[test]
fn a_non_terminal_status_exits_two() {
for status in [
DataflowStatus::Pending,
DataflowStatus::Building,
DataflowStatus::Ready,
DataflowStatus::Starting,
DataflowStatus::Running,
DataflowStatus::Stopping,
] {
let report = report(status, &[]);
assert_eq!(report.exit_code(), EXIT_INCOMPLETE, "{status:?}");
}
}
#[test]
fn an_abandoned_wait_exits_two_whatever_the_status_says() {
let mut report = report(DataflowStatus::Finished, &[("a", NodeExitCause::Success)]);
report.abandoned = true;
assert_eq!(report.exit_code(), EXIT_INCOMPLETE);
}
#[test]
fn the_summary_names_the_status_and_the_failure_count() {
let report = report(
DataflowStatus::Failed,
&[
("a", NodeExitCause::Success),
("b", NodeExitCause::ExitCode { code: 1 }),
],
);
let text = report.summary();
assert!(text.contains("failed"), "{text}");
assert!(text.contains("2 node(s)"), "{text}");
assert!(text.contains("1 failed"), "{text}");
}
#[test]
fn the_summary_mentions_dropped_lines_only_when_there_were_some() {
let mut report = report(DataflowStatus::Finished, &[]);
assert!(!report.summary().contains("dropped"));
report.dropped = 5;
assert!(report.summary().contains("5 log line(s)"));
}
#[test]
fn the_json_form_carries_every_node_and_the_exit_code() {
let report = report(
DataflowStatus::Failed,
&[("cam", NodeExitCause::ExitCode { code: 2 })],
);
let json = report.to_json();
assert_eq!(json["status"], "failed");
assert_eq!(json["exit_code"], EXIT_FAILED);
assert_eq!(json["nodes"]["cam"]["failed"], true);
}
#[test]
fn deterministic_without_a_recording_is_refused_before_anything_runs() {
let dir = scratch("deterministic");
let path = write_manifest(&dir, "nodes:\n - id: a\n path: /usr/bin/true\n");
let mut args = RunArgs::new(path);
args.deterministic = true;
let error = run(&mut Vec::new(), &args).unwrap_err();
assert!(
matches!(error, CliError::DeterministicNeedsRecording),
"expected DeterministicNeedsRecording, got {error}"
);
assert!(error.to_string().contains("--from-recording"), "{error}");
}
#[test]
fn a_manifest_that_does_not_parse_is_reported_by_the_cli() {
let dir = scratch("bad-manifest");
let path = write_manifest(&dir, "nodes: [oops");
let error = run(&mut Vec::new(), &RunArgs::new(path)).unwrap_err();
assert!(matches!(error, CliError::Manifest(_)), "{error}");
}
#[test]
fn a_manifest_that_fails_validation_is_reported_by_the_cli() {
let dir = scratch("invalid-manifest");
let path = write_manifest(
&dir,
"nodes:\n - id: a\n path: /usr/bin/true\n inputs:\n in: ghost/out\n",
);
let error = run(&mut Vec::new(), &RunArgs::new(path)).unwrap_err();
assert!(matches!(error, CliError::Validation(_)), "{error}");
}
#[test]
fn the_manifest_directory_is_the_default_working_directory() {
assert_eq!(
manifest_dir(Path::new("/a/b/dataflow.yml")),
PathBuf::from("/a/b")
);
assert_eq!(manifest_dir(Path::new("dataflow.yml")), PathBuf::from("."));
}
#[test]
fn exit_when_nodes_finish_overrides_the_manifest() {
let dir = scratch("exit-flag");
let path = write_manifest(&dir, "nodes:\n - id: a\n path: /usr/bin/true\n");
let manifest = load_manifest(&path, false).unwrap();
assert!(!manifest.exit_when_nodes_finish);
let manifest = load_manifest(&path, true).unwrap();
assert!(manifest.exit_when_nodes_finish);
}
#[test]
fn an_abandoned_result_records_every_node_as_killed() {
let manifest =
Manifest::from_yaml_str("nodes:\n - id: a\n path: /usr/bin/true\n").unwrap();
let result = abandoned_result(&manifest);
assert_eq!(result.status, DataflowStatus::Stopping);
assert_eq!(result.node_results.len(), 1);
assert!(result.has_failures());
}
#[test]
fn a_trivial_graph_runs_to_a_clean_finish_and_exits_zero() {
let dir = scratch("trivial");
let path = write_manifest(
&dir,
"exit_when_nodes_finish: true\nnodes:\n - id: a\n path: /bin/sh\n args: [\"-c\", \"echo hello-from-run\"]\n",
);
let mut args = RunArgs::new(path);
args.skip_build = true;
args.timeout = Some(Duration::from_secs(20));
args.grace = Some(Duration::from_millis(200));
args.runtime_dir = Some(dir.clone());
args.working_dir = Some(dir);
let mut out: Vec<u8> = Vec::new();
let report = run(&mut out, &args).expect("a report");
let text = String::from_utf8(out).unwrap();
assert_eq!(report.exit_code(), EXIT_OK, "{text}");
assert!(
text.contains("hello-from-run"),
"the node's own output must reach the terminal: {text}"
);
assert!(text.contains("finished"), "{text}");
}
#[test]
fn a_failing_build_line_stops_the_run_before_spawning() {
let dir = scratch("bad-build");
let path = write_manifest(
&dir,
"nodes:\n - id: a\n path: /usr/bin/true\n build: /bin/sh -c 'exit 4'\n",
);
let mut args = RunArgs::new(path);
args.timeout = Some(Duration::from_secs(20));
args.runtime_dir = Some(dir.clone());
args.working_dir = Some(dir);
let error = run(&mut Vec::new(), &args).unwrap_err();
assert!(matches!(error, CliError::Daemon(_)), "{error}");
}
#[test]
fn skip_build_reaches_the_spawn_a_failing_build_would_have_blocked() {
let dir = scratch("skip-build");
let path = write_manifest(
&dir,
"exit_when_nodes_finish: true\nnodes:\n - id: a\n path: /usr/bin/true\n build: /bin/sh -c 'exit 4'\n",
);
let mut args = RunArgs::new(path);
args.skip_build = true;
args.timeout = Some(Duration::from_secs(20));
args.grace = Some(Duration::from_millis(200));
args.runtime_dir = Some(dir.clone());
args.working_dir = Some(dir);
let report = run(&mut Vec::new(), &args).expect("a report");
assert_eq!(report.result.node_results.len(), 1);
}
}