use core::fmt;
use std::collections::BTreeMap;
use astrs_time::HlcTimestamp;
use oxicode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use crate::common::duration::DurationMs;
use crate::ids::{DaemonId, DataId, DataflowId, NodeId, TypeUrn};
#[derive(
Debug,
Clone,
Copy,
Default,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
Encode,
Decode,
)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum DataflowStatus {
#[default]
#[oxicode(variant = 0)]
Pending,
#[oxicode(variant = 1)]
Building,
#[oxicode(variant = 2)]
Ready,
#[oxicode(variant = 3)]
Starting,
#[oxicode(variant = 4)]
Running,
#[oxicode(variant = 5)]
Stopping,
#[oxicode(variant = 6)]
Finished,
#[oxicode(variant = 7)]
Failed,
}
impl DataflowStatus {
pub const ALL: &'static [Self] = &[
Self::Pending,
Self::Building,
Self::Ready,
Self::Starting,
Self::Running,
Self::Stopping,
Self::Finished,
Self::Failed,
];
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Pending => "pending",
Self::Building => "building",
Self::Ready => "ready",
Self::Starting => "starting",
Self::Running => "running",
Self::Stopping => "stopping",
Self::Finished => "finished",
Self::Failed => "failed",
}
}
#[must_use]
pub const fn is_terminal(self) -> bool {
matches!(self, Self::Finished | Self::Failed)
}
#[must_use]
pub const fn is_active(self) -> bool {
matches!(self, Self::Starting | Self::Running | Self::Stopping)
}
}
impl fmt::Display for DataflowStatus {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(
Debug,
Clone,
Copy,
Default,
PartialEq,
Eq,
PartialOrd,
Ord,
Hash,
Serialize,
Deserialize,
Encode,
Decode,
)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum NodeRunState {
#[default]
#[oxicode(variant = 0)]
Pending,
#[oxicode(variant = 1)]
Spawning,
#[oxicode(variant = 2)]
Running,
#[oxicode(variant = 3)]
Restarting,
#[oxicode(variant = 4)]
Stopping,
#[oxicode(variant = 5)]
Finished,
#[oxicode(variant = 6)]
Failed,
}
impl NodeRunState {
pub const ALL: &'static [Self] = &[
Self::Pending,
Self::Spawning,
Self::Running,
Self::Restarting,
Self::Stopping,
Self::Finished,
Self::Failed,
];
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Pending => "pending",
Self::Spawning => "spawning",
Self::Running => "running",
Self::Restarting => "restarting",
Self::Stopping => "stopping",
Self::Finished => "finished",
Self::Failed => "failed",
}
}
#[must_use]
pub const fn is_terminal(self) -> bool {
matches!(self, Self::Finished | Self::Failed)
}
}
impl fmt::Display for NodeRunState {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum NodeExitCause {
#[oxicode(variant = 0)]
Success,
#[oxicode(variant = 1)]
ExitCode {
code: i32,
},
#[oxicode(variant = 2)]
Signal {
signal: i32,
name: String,
},
#[oxicode(variant = 3)]
Panic {
message: String,
},
#[oxicode(variant = 4)]
SpawnFailed {
message: String,
},
#[oxicode(variant = 5)]
SpawnDeadlineExceeded {
after: DurationMs,
},
#[oxicode(variant = 6)]
HealthCheckTimeout {
after: DurationMs,
},
#[oxicode(variant = 7)]
Killed {
reason: String,
},
#[oxicode(variant = 8)]
DaemonShutdown,
#[oxicode(variant = 9)]
RestartBudgetExhausted {
restarts: u32,
window: DurationMs,
},
#[oxicode(variant = 10)]
Cancelled,
#[oxicode(variant = 11)]
DaemonUnreachable {
daemon: DaemonId,
},
}
impl NodeExitCause {
#[must_use]
pub const fn is_success(&self) -> bool {
match self {
Self::Success => true,
Self::ExitCode { code } => *code == 0,
_ => false,
}
}
#[must_use]
pub const fn is_failure(&self) -> bool {
!self.is_success() && !matches!(self, Self::Cancelled | Self::DaemonShutdown)
}
#[must_use]
pub const fn kind_name(&self) -> &'static str {
match self {
Self::Success => "success",
Self::ExitCode { .. } => "exit_code",
Self::Signal { .. } => "signal",
Self::Panic { .. } => "panic",
Self::SpawnFailed { .. } => "spawn_failed",
Self::SpawnDeadlineExceeded { .. } => "spawn_deadline_exceeded",
Self::HealthCheckTimeout { .. } => "health_check_timeout",
Self::Killed { .. } => "killed",
Self::DaemonShutdown => "daemon_shutdown",
Self::RestartBudgetExhausted { .. } => "restart_budget_exhausted",
Self::Cancelled => "cancelled",
Self::DaemonUnreachable { .. } => "daemon_unreachable",
}
}
}
impl fmt::Display for NodeExitCause {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Success => f.write_str("exited successfully"),
Self::ExitCode { code } => write!(f, "exited with code {code}"),
Self::Signal { signal, name } => write!(f, "killed by signal {signal} ({name})"),
Self::Panic { message } => write!(f, "panicked: {message}"),
Self::SpawnFailed { message } => write!(f, "spawn failed: {message}"),
Self::SpawnDeadlineExceeded { after } => {
write!(f, "did not register within {after}")
}
Self::HealthCheckTimeout { after } => write!(f, "health check timed out after {after}"),
Self::Killed { reason } => write!(f, "killed by the daemon: {reason}"),
Self::DaemonShutdown => f.write_str("daemon shut down"),
Self::RestartBudgetExhausted { restarts, window } => {
write!(f, "restarted {restarts} times within {window}")
}
Self::Cancelled => f.write_str("stop requested"),
Self::DaemonUnreachable { daemon } => write!(f, "daemon {daemon} became unreachable"),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum StopCause {
#[oxicode(variant = 0)]
Requested,
#[oxicode(variant = 1)]
DataflowFinished,
#[oxicode(variant = 2)]
AllInputsClosed,
#[oxicode(variant = 3)]
DaemonShutdown,
#[oxicode(variant = 4)]
PeerFailed {
node: NodeId,
},
#[oxicode(variant = 5)]
Destroyed,
#[oxicode(variant = 6)]
Error {
message: String,
},
#[oxicode(variant = 7)]
Replaced,
}
impl StopCause {
#[must_use]
pub const fn is_graceful(&self) -> bool {
matches!(
self,
Self::Requested
| Self::DataflowFinished
| Self::AllInputsClosed
| Self::Destroyed
| Self::Replaced
)
}
}
impl fmt::Display for StopCause {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Requested => f.write_str("stop requested"),
Self::DataflowFinished => f.write_str("dataflow finished"),
Self::AllInputsClosed => f.write_str("all inputs closed"),
Self::DaemonShutdown => f.write_str("daemon shutting down"),
Self::PeerFailed { node } => write!(f, "peer node {node} failed"),
Self::Destroyed => f.write_str("dataflow destroyed"),
Self::Error { message } => write!(f, "error: {message}"),
Self::Replaced => f.write_str("superseded by a replacement"),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)]
pub struct DataflowResult {
pub dataflow: DataflowId,
pub status: DataflowStatus,
pub node_results: BTreeMap<NodeId, NodeExitCause>,
pub started_at: HlcTimestamp,
pub finished_at: Option<HlcTimestamp>,
pub message: String,
}
impl DataflowResult {
#[must_use]
pub fn new(dataflow: DataflowId, started_at: HlcTimestamp) -> Self {
Self {
dataflow,
status: DataflowStatus::Running,
node_results: BTreeMap::new(),
started_at,
finished_at: None,
message: String::new(),
}
}
pub fn record(&mut self, node: NodeId, cause: NodeExitCause) {
self.node_results.insert(node, cause);
}
pub fn finish(&mut self, finished_at: HlcTimestamp) {
self.finished_at = Some(finished_at);
self.status = if self.has_failures() {
DataflowStatus::Failed
} else {
DataflowStatus::Finished
};
}
#[must_use]
pub fn has_failures(&self) -> bool {
self.node_results.values().any(NodeExitCause::is_failure)
}
pub fn failed_nodes(&self) -> impl Iterator<Item = (&NodeId, &NodeExitCause)> {
self.node_results
.iter()
.filter(|(_, cause)| cause.is_failure())
}
#[must_use]
pub fn duration(&self) -> Option<DurationMs> {
let finished = self.finished_at?;
finished
.physical_duration_since(&self.started_at)
.map(DurationMs::from_duration)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)]
pub struct DataflowSummary {
pub id: DataflowId,
pub name: Option<String>,
pub status: DataflowStatus,
pub daemons: Vec<DaemonId>,
pub node_count: u32,
pub running_nodes: u32,
pub started_at: Option<HlcTimestamp>,
}
impl DataflowSummary {
#[must_use]
pub fn pending(id: DataflowId, node_count: u32) -> Self {
Self {
id,
name: None,
status: DataflowStatus::Pending,
daemons: Vec::new(),
node_count,
running_nodes: 0,
started_at: None,
}
}
#[must_use]
pub fn display_name(&self) -> String {
self.name.clone().unwrap_or_else(|| self.id.to_string())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)]
pub struct NodeInfo {
pub dataflow: DataflowId,
pub node: NodeId,
pub daemon: DaemonId,
pub state: NodeRunState,
pub pid: Option<u32>,
pub generation: u64,
pub restart_count: u32,
pub inputs: BTreeMap<DataId, Option<TypeUrn>>,
pub outputs: BTreeMap<DataId, Option<TypeUrn>>,
pub started_at: Option<HlcTimestamp>,
pub exit_cause: Option<NodeExitCause>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)]
pub struct DaemonInfo {
pub id: DaemonId,
pub version: crate::version::AstrsVersion,
pub address: String,
pub connected_at: HlcTimestamp,
pub node_count: u32,
pub labels: BTreeMap<String, String>,
pub reachable: bool,
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::codec::round_trip;
use crate::ids::MachineName;
fn node(name: &str) -> NodeId {
NodeId::new(name).unwrap()
}
fn daemon() -> DaemonId {
DaemonId::from_parts(Some("robot-1"), uuid::Uuid::from_u128(9)).unwrap()
}
fn causes() -> Vec<NodeExitCause> {
vec![
NodeExitCause::Success,
NodeExitCause::ExitCode { code: 0 },
NodeExitCause::ExitCode { code: -1 },
NodeExitCause::Signal {
signal: 9,
name: "SIGKILL".into(),
},
NodeExitCause::Panic {
message: "index out of bounds".into(),
},
NodeExitCause::SpawnFailed {
message: "no such file".into(),
},
NodeExitCause::SpawnDeadlineExceeded {
after: DurationMs::from_secs(10),
},
NodeExitCause::HealthCheckTimeout {
after: DurationMs::from_secs(5),
},
NodeExitCause::Killed {
reason: "finish grace expired".into(),
},
NodeExitCause::DaemonShutdown,
NodeExitCause::RestartBudgetExhausted {
restarts: 5,
window: DurationMs::from_secs(60),
},
NodeExitCause::Cancelled,
NodeExitCause::DaemonUnreachable { daemon: daemon() },
]
}
#[test]
fn every_exit_cause_round_trips_and_names_itself() {
let mut names = std::collections::BTreeSet::new();
for cause in causes() {
assert_eq!(round_trip(&cause).unwrap(), cause);
names.insert(cause.kind_name());
assert!(!cause.to_string().is_empty());
}
assert_eq!(names.len(), 12, "one name per variant");
}
#[test]
fn success_and_failure_partition_correctly() {
assert!(NodeExitCause::Success.is_success());
assert!(NodeExitCause::ExitCode { code: 0 }.is_success());
assert!(!NodeExitCause::ExitCode { code: 1 }.is_success());
assert!(NodeExitCause::ExitCode { code: 1 }.is_failure());
for cause in [NodeExitCause::Cancelled, NodeExitCause::DaemonShutdown] {
assert!(!cause.is_success(), "{cause}");
assert!(!cause.is_failure(), "{cause}");
}
}
#[test]
fn every_other_cause_is_a_failure() {
for cause in causes() {
let deliberate = matches!(
cause,
NodeExitCause::Cancelled | NodeExitCause::DaemonShutdown
);
assert_eq!(cause.is_failure(), !cause.is_success() && !deliberate);
}
}
#[test]
fn dataflow_statuses_round_trip_and_classify() {
for status in DataflowStatus::ALL.iter().copied() {
assert_eq!(round_trip(&status).unwrap(), status);
assert_eq!(status.to_string(), status.as_str());
assert!(!(status.is_terminal() && status.is_active()));
}
assert!(DataflowStatus::Finished.is_terminal());
assert!(DataflowStatus::Failed.is_terminal());
assert!(DataflowStatus::Running.is_active());
assert!(!DataflowStatus::Pending.is_active());
assert_eq!(DataflowStatus::default(), DataflowStatus::Pending);
}
#[test]
fn node_run_states_round_trip_and_classify() {
for state in NodeRunState::ALL.iter().copied() {
assert_eq!(round_trip(&state).unwrap(), state);
assert_eq!(state.to_string(), state.as_str());
}
assert!(NodeRunState::Finished.is_terminal());
assert!(NodeRunState::Failed.is_terminal());
assert!(!NodeRunState::Restarting.is_terminal());
assert_eq!(NodeRunState::default(), NodeRunState::Pending);
}
#[test]
fn stop_causes_round_trip_and_classify() {
let all = [
StopCause::Requested,
StopCause::DataflowFinished,
StopCause::AllInputsClosed,
StopCause::DaemonShutdown,
StopCause::PeerFailed { node: node("a") },
StopCause::Destroyed,
StopCause::Error {
message: "boom".into(),
},
];
for cause in all {
assert_eq!(round_trip(&cause).unwrap(), cause);
assert!(!cause.to_string().is_empty());
}
assert!(StopCause::Requested.is_graceful());
assert!(!StopCause::DaemonShutdown.is_graceful());
assert!(
!StopCause::Error {
message: String::new()
}
.is_graceful()
);
}
#[test]
fn a_clean_run_finishes_rather_than_failing() {
let mut result = DataflowResult::new(DataflowId::from_u128(1), HlcTimestamp::new(1_000, 0));
result.record(node("a"), NodeExitCause::Success);
result.record(node("b"), NodeExitCause::ExitCode { code: 0 });
result.record(node("c"), NodeExitCause::Cancelled);
result.finish(HlcTimestamp::new(3_000_000_000, 0));
assert_eq!(result.status, DataflowStatus::Finished);
assert!(!result.has_failures());
assert_eq!(result.failed_nodes().count(), 0);
assert_eq!(result.duration(), Some(DurationMs::new(2_999)));
}
#[test]
fn one_failure_fails_the_run() {
let mut result = DataflowResult::new(DataflowId::from_u128(1), HlcTimestamp::new(0, 0));
result.record(node("a"), NodeExitCause::Success);
result.record(
node("b"),
NodeExitCause::Panic {
message: "boom".into(),
},
);
result.finish(HlcTimestamp::new(1, 0));
assert_eq!(result.status, DataflowStatus::Failed);
assert!(result.has_failures());
let failed: Vec<_> = result
.failed_nodes()
.map(|(node, _)| node.clone())
.collect();
assert_eq!(failed, vec![node("b")]);
assert_eq!(round_trip(&result).unwrap(), result);
}
#[test]
fn an_unfinished_run_has_no_duration() {
let result = DataflowResult::new(DataflowId::from_u128(1), HlcTimestamp::new(5, 0));
assert_eq!(result.duration(), None);
assert_eq!(result.status, DataflowStatus::Running);
}
#[test]
fn summaries_round_trip_and_display() {
let mut summary = DataflowSummary::pending(DataflowId::from_u128(0x2A), 3);
assert_eq!(summary.display_name(), summary.id.to_string());
assert_eq!(summary.running_nodes, 0);
summary.name = Some("perception-demo".to_owned());
summary.status = DataflowStatus::Running;
summary.daemons.push(daemon());
summary.running_nodes = 3;
summary.started_at = Some(HlcTimestamp::new(1, 0));
assert_eq!(summary.display_name(), "perception-demo");
assert_eq!(round_trip(&summary).unwrap(), summary);
}
#[test]
fn node_info_round_trips() {
let info = NodeInfo {
dataflow: DataflowId::from_u128(1),
node: node("camera"),
daemon: daemon(),
state: NodeRunState::Running,
pid: Some(4242),
generation: 2,
restart_count: 1,
inputs: BTreeMap::from([(
DataId::new("trigger").unwrap(),
Some(TypeUrn::new("std/core/v1/Empty").unwrap()),
)]),
outputs: BTreeMap::from([(DataId::new("image").unwrap(), None)]),
started_at: Some(HlcTimestamp::new(7, 0)),
exit_cause: None,
};
assert_eq!(round_trip(&info).unwrap(), info);
}
#[test]
fn daemon_info_round_trips() {
let info = DaemonInfo {
id: DaemonId::generate(Some(MachineName::new("lab-1").unwrap())),
version: crate::version::AstrsVersion::from_parts(0, 1, 0),
address: "127.0.0.1:7408".to_owned(),
connected_at: HlcTimestamp::new(11, 0),
node_count: 4,
labels: BTreeMap::from([("gpu".to_owned(), "a100".to_owned())]),
reachable: true,
};
assert_eq!(round_trip(&info).unwrap(), info);
}
}