1use crate::{Context, HealthStatus, NodeState, Result};
2use async_trait::async_trait;
3use schemars::JsonSchema;
4use serde::de::DeserializeOwned;
5use serde::{Deserialize, Serialize};
6use std::fmt::Debug;
7use std::sync::Arc;
8use std::time::Duration;
9use tokio::sync::Mutex;
10use tracing::{debug, error, info, warn};
11
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
13#[serde(rename_all = "lowercase")]
14pub enum HealthPriority {
15 Critical,
16
17 #[default]
18 Normal,
19
20 Background,
21}
22
23impl HealthPriority {
24 const fn profile(&self) -> (Duration, u64, Duration) {
30 match self {
31 Self::Critical => (Duration::from_secs(1), 5, Duration::from_secs(3)),
32 Self::Normal => (Duration::from_secs(5), 60, Duration::from_secs(15)),
33 Self::Background => (Duration::from_secs(30), 120, Duration::from_secs(90)),
34 }
35 }
36
37 pub const fn interval(&self) -> Duration {
39 self.profile().0
40 }
41
42 pub const fn ttl_seconds(&self) -> u64 {
47 self.profile().1
48 }
49
50 pub const fn timeout(&self) -> Duration {
55 self.profile().2
56 }
57
58 pub const fn timeout_micros(&self) -> u64 {
60 self.timeout().as_micros() as u64
61 }
62}
63
64impl std::fmt::Display for HealthPriority {
65 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
66 match self {
67 Self::Critical => write!(f, "critical"),
68 Self::Normal => write!(f, "normal"),
69 Self::Background => write!(f, "background"),
70 }
71 }
72}
73
74#[derive(Debug, Clone)]
75pub struct HealthReportingConfig {
76 pub enabled: bool,
77
78 pub priority: HealthPriority,
79
80 pub store_history: bool,
81}
82
83impl Default for HealthReportingConfig {
84 fn default() -> Self {
85 Self {
86 enabled: true,
87 priority: HealthPriority::Normal,
88 store_history: true,
89 }
90 }
91}
92
93impl HealthReportingConfig {
94 fn with_priority(priority: HealthPriority, store_history: bool) -> Self {
97 Self {
98 enabled: true,
99 priority,
100 store_history,
101 }
102 }
103
104 pub fn critical() -> Self {
106 Self::with_priority(HealthPriority::Critical, false)
107 }
108
109 pub fn normal() -> Self {
111 Self::default()
112 }
113
114 pub fn background() -> Self {
116 Self::with_priority(HealthPriority::Background, true)
117 }
118
119 pub fn interval(&self) -> Duration {
121 self.priority.interval()
122 }
123
124 pub fn ttl_seconds(&self) -> u64 {
126 self.priority.ttl_seconds()
127 }
128}
129
130pub trait RateConfig {
131 fn rate_hz(&self) -> f32;
133
134 fn create_interval(&self) -> tokio::time::Interval {
139 let duration = Duration::from_secs_f32(1.0 / self.rate_hz());
140 tokio::time::interval(duration)
141 }
142
143 fn period(&self) -> Duration {
145 Duration::from_secs_f32(1.0 / self.rate_hz())
146 }
147}
148
149pub trait HealthStatusExt {
150 fn healthy(message: impl Into<String>) -> Self;
152 fn unhealthy(message: impl Into<String>) -> Self;
154}
155
156impl HealthStatus {
157 fn with_check(healthy: bool, failure_count: u32, message: impl Into<String>) -> Self {
161 Self {
162 healthy,
163 last_check: crate::prelude::now_micros(),
164 failure_count,
165 message: Some(message.into()),
166 }
167 }
168}
169
170impl HealthStatusExt for HealthStatus {
171 fn healthy(message: impl Into<String>) -> Self {
172 Self::with_check(true, 0, message)
173 }
174
175 fn unhealthy(message: impl Into<String>) -> Self {
176 Self::with_check(false, 1, message)
177 }
178}
179
180#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
181#[serde(rename_all = "kebab-case")]
182pub enum NodeType {
183 Imu,
184 Camera,
185 ImageClassifier,
186 ObjectDetector,
187 BehaviorExecutor,
188 LlmCommand,
189 Teleop,
190 Motor,
191 WebsocketBridge,
192 VideoStreamer,
193 Simulator,
194 Speaker,
195 Listener,
196 Rl,
197 InstanceTrack,
198 Diagnostics,
199 Servo,
200 PanTilt,
201}
202
203impl NodeType {
204 pub const ALL: &'static [NodeType] = &[
205 NodeType::Imu,
206 NodeType::Camera,
207 NodeType::ImageClassifier,
208 NodeType::ObjectDetector,
209 NodeType::BehaviorExecutor,
210 NodeType::LlmCommand,
211 NodeType::Teleop,
212 NodeType::Motor,
213 NodeType::WebsocketBridge,
214 NodeType::VideoStreamer,
215 NodeType::Simulator,
216 NodeType::Speaker,
217 NodeType::Listener,
218 NodeType::Rl,
219 NodeType::InstanceTrack,
220 NodeType::Diagnostics,
221 NodeType::Servo,
222 NodeType::PanTilt,
223 ];
224
225 pub fn as_str(&self) -> &'static str {
232 match self {
233 NodeType::Imu => "imu",
234 NodeType::Camera => "camera",
235 NodeType::ImageClassifier => "image-classifier",
236 NodeType::ObjectDetector => "object-detector",
237 NodeType::BehaviorExecutor => "behavior-executor",
238 NodeType::LlmCommand => "llm-command",
239 NodeType::Teleop => "teleop",
240 NodeType::Motor => "motor",
241 NodeType::WebsocketBridge => "websocket-bridge",
242 NodeType::VideoStreamer => "video-streamer",
243 NodeType::Simulator => "simulator",
244 NodeType::Speaker => "speaker",
245 NodeType::Listener => "listener",
246 NodeType::Rl => "rl",
247 NodeType::InstanceTrack => "instance-track",
248 NodeType::Diagnostics => "diagnostics",
249 NodeType::Servo => "servo",
250 NodeType::PanTilt => "pan-tilt",
251 }
252 }
253}
254
255impl std::fmt::Display for NodeType {
256 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
257 write!(f, "{}", self.as_str())
258 }
259}
260
261#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
262#[serde(rename_all = "kebab-case")]
263pub enum NodeTag {
264 Sensor,
265 Inertial,
266 Capture,
267 Ai,
268 Classifier,
269 Detector,
270 Vision,
271 Planner,
272 Control,
273 Nlp,
274 Input,
275 Actuator,
276 Motion,
277 Relay,
278 Websocket,
279 Streamer,
280 Video,
281 Simulator,
282 Logger,
283}
284
285#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)]
288#[serde(rename_all = "kebab-case")]
289pub enum Capability {
290 Locomotion,
291 Joints,
292 Flight,
293 Video,
294 Vision,
295 Audio,
296}
297
298#[async_trait]
299pub trait NodeImpl: Send + Sync + Debug + 'static {
300 type Config: DeserializeOwned + Send + Sync + Debug;
301
302 const NODE_TYPE: Option<NodeType> = None;
303
304 const NODE_TAGS: &'static [NodeTag] = &[];
305
306 const CRATE_NAME: Option<&'static str> = None;
314
315 const DESCRIPTION: &'static str = "";
318
319 const DEFAULT_RUN_TARGET: &'static str = "robot";
322
323 const CATALOG_TAGS: &'static [&'static str] = &[];
327
328 const CAPABILITIES: &'static [Capability] = &[];
332
333 async fn init(config: Self::Config) -> Result<Self>
344 where
345 Self: Sized;
346
347 async fn run(&mut self, ctx: &Context) -> Result<()>;
363
364 async fn shutdown(&mut self) -> Result<()> {
378 Ok(())
379 }
380
381 async fn health_check(&self) -> HealthStatus {
390 HealthStatus {
391 healthy: true,
392 last_check: crate::prelude::now_micros(),
393 failure_count: 0,
394 message: Some("Node is running".to_string()),
395 }
396 }
397
398 fn health_diagnostics(&self) -> Option<std::collections::HashMap<String, String>> {
409 None
410 }
411
412 fn preflight(&self) -> Result<()> {
436 Ok(())
437 }
438}
439
440pub fn fmt_node_with_controller(
447 f: &mut std::fmt::Formatter<'_>,
448 struct_name: &str,
449 config: &impl Debug,
450 controller_name: &str,
451 update_counter: u64,
452) -> std::fmt::Result {
453 f.debug_struct(struct_name)
454 .field("config", config)
455 .field("controller", &controller_name)
456 .field("update_counter", &update_counter)
457 .finish()
458}
459
460#[derive(Clone)]
461pub struct Node<T: NodeImpl> {
462 node: Arc<Mutex<T>>,
463 state: Arc<Mutex<NodeState>>,
464 static_diagnostics: Arc<Option<std::collections::HashMap<String, String>>>,
474}
475
476impl<T: NodeImpl> Node<T> {
477 pub async fn from_config(config: T::Config) -> Result<Self> {
485 info!("Initializing node");
486
487 let node = T::init(config).await?;
488 let static_diagnostics = node.health_diagnostics();
489
490 Ok(Self {
491 node: Arc::new(Mutex::new(node)),
492 state: Arc::new(Mutex::new(NodeState::Starting)),
493 static_diagnostics: Arc::new(static_diagnostics),
494 })
495 }
496
497 pub fn static_diagnostics(&self) -> Option<std::collections::HashMap<String, String>> {
500 (*self.static_diagnostics).clone()
501 }
502
503 pub async fn start(&self, ctx: Context) -> Result<()> {
517 {
518 let node = self.node.lock().await;
519 node.preflight()?;
520 }
521
522 {
523 let mut state = self.state.lock().await;
524 *state = NodeState::Running;
525 info!("Node transitioned to Running state");
526 }
527
528 let result = {
529 let mut node = self.node.lock().await;
530 debug!("Starting node execution");
531 node.run(&ctx).await
532 };
533
534 match result {
535 Ok(()) => {
536 info!("Node completed successfully");
537 self.shutdown().await?;
538 }
539 Err(e) => {
540 error!("Node failed with error: {}", e);
541 {
542 let mut state = self.state.lock().await;
543 *state = NodeState::Crashed;
544 }
545 self.shutdown().await?;
546 return Err(e);
547 }
548 }
549
550 Ok(())
551 }
552
553 pub async fn shutdown(&self) -> Result<()> {
557 info!("Shutting down node");
558
559 {
560 let mut state = self.state.lock().await;
561 *state = NodeState::Stopping;
562 }
563
564 let result = {
565 let mut node = self.node.lock().await;
566 node.shutdown().await
567 };
568
569 {
570 let mut state = self.state.lock().await;
571 *state = NodeState::Stopped;
572 info!("Node stopped");
573 }
574
575 if let Err(e) = result {
576 warn!("Error during shutdown: {}", e);
577 return Err(e);
578 }
579
580 Ok(())
581 }
582
583 pub async fn state(&self) -> NodeState {
585 self.state.lock().await.clone()
586 }
587
588 pub async fn health_check(&self) -> HealthStatus {
590 let node = self.node.lock().await;
591 node.health_check().await
592 }
593
594 pub async fn is_healthy(&self) -> bool {
596 matches!(self.state().await, NodeState::Running | NodeState::Starting)
597 }
598}
599
600pub async fn run_node<T: NodeImpl>(
651 config: T::Config,
652 ctx: Context,
653 health_config: HealthReportingConfig,
654) -> Result<()> {
655 crate::prelude::init_logging();
656
657 #[cfg(feature = "messaging")]
658 crate::log_publisher::attach(ctx.node_id(), ctx.clone());
659
660 let node = Node::<T>::from_config(config).await?;
661
662 #[cfg(feature = "messaging")]
663 {
664 if health_config.enabled {
665 let node_for_health = node.node.clone();
666 let static_diagnostics_for_health = node.static_diagnostics();
667 let ctx_for_health = ctx.clone();
668 let priority = health_config.priority;
669 let report_interval = health_config.interval();
670 let ttl_seconds = health_config.ttl_seconds();
671 let store_history = health_config.store_history;
672
673 tokio::spawn(async move {
674 use crate::health::HealthReporter;
675
676 let reporter = match HealthReporter::with_config_and_type_tags(
677 &ctx_for_health,
678 priority,
679 ttl_seconds,
680 store_history,
681 T::NODE_TYPE,
682 T::NODE_TAGS,
683 None,
684 )
685 .await
686 {
687 Ok(r) => r,
688 Err(e) => {
689 warn!("Health reporting disabled: {}", e);
690 return;
691 }
692 };
693
694 debug!(
695 "Health reporting enabled: priority={}, interval={:?}, ttl={}s, history={}",
696 priority, report_interval, ttl_seconds, store_history
697 );
698
699 let mut interval = tokio::time::interval(report_interval);
700
701 loop {
702 interval.tick().await;
703 debug!("Health tick - checking node health");
704
705 let msg_status = match node_for_health.try_lock() {
706 Ok(node) => {
707 let status = node.health_check().await;
708 let diagnostics = node.health_diagnostics();
709 crate::messages::system::HealthStatus {
710 timestamp: status.last_check,
711 level: if status.healthy {
712 crate::messages::system::HealthLevel::Ok
713 } else {
714 crate::messages::system::HealthLevel::Error
715 },
716 message: status.message.unwrap_or_else(|| "No message".to_string()),
717 diagnostics,
718 }
719 }
720 Err(_) => {
721 debug!("Node busy, reporting as healthy");
732 crate::messages::system::HealthStatus {
733 timestamp: crate::prelude::now_micros(),
734 level: crate::messages::system::HealthLevel::Ok,
735 message: "Node running (busy)".to_string(),
736 diagnostics: static_diagnostics_for_health.clone(),
737 }
738 }
739 };
740
741 if let Err(e) = reporter.report(msg_status).await {
742 warn!("Failed to report health: {}", e);
743 } else {
744 tracing::trace!("Health report sent successfully");
745 }
746 }
747 });
748 } else {
749 info!("Health reporting disabled");
750 }
751 }
752 #[cfg(not(feature = "messaging"))]
753 let _ = &health_config;
754
755 node.start(ctx).await
756}
757
758#[cfg(all(test, not(feature = "messaging")))]
759mod tests {
760 use super::*;
761 use crate::error::Mecha10Error;
762 use std::sync::atomic::{AtomicBool, Ordering};
763 use std::sync::Arc as StdArc;
764
765 #[derive(Debug, Default, serde::Deserialize)]
766 struct FakeConfig;
767
768 #[derive(Debug)]
769 struct OkNode {
770 ran: StdArc<AtomicBool>,
771 }
772
773 #[async_trait]
774 impl NodeImpl for OkNode {
775 type Config = FakeConfig;
776
777 async fn init(_config: Self::Config) -> Result<Self> {
778 Ok(Self {
779 ran: StdArc::new(AtomicBool::new(false)),
780 })
781 }
782
783 async fn run(&mut self, _ctx: &Context) -> Result<()> {
784 self.ran.store(true, Ordering::SeqCst);
785 Ok(())
786 }
787 }
788
789 #[derive(Debug)]
790 struct RejectingPreflightNode {
791 ran: StdArc<AtomicBool>,
792 }
793
794 #[async_trait]
795 impl NodeImpl for RejectingPreflightNode {
796 type Config = FakeConfig;
797
798 async fn init(_config: Self::Config) -> Result<Self> {
799 Ok(Self {
800 ran: StdArc::new(AtomicBool::new(false)),
801 })
802 }
803
804 async fn run(&mut self, _ctx: &Context) -> Result<()> {
805 self.ran.store(true, Ordering::SeqCst);
806 Ok(())
807 }
808
809 fn preflight(&self) -> Result<()> {
810 Err(Mecha10Error::Configuration("boom: bad node content".to_string()))
811 }
812 }
813
814 #[tokio::test]
815 async fn default_preflight_is_a_noop_and_run_still_executes() {
816 let node = Node::<OkNode>::from_config(FakeConfig).await.unwrap();
817 let ctx = Context::new("test-node-ok").await.unwrap();
818
819 let ran_handle = { node.node.lock().await.ran.clone() };
820
821 node.start(ctx).await.unwrap();
822
823 assert!(ran_handle.load(Ordering::SeqCst), "run() should have executed");
824 }
825
826 #[tokio::test]
827 async fn start_fails_fast_when_preflight_rejects_config_and_never_calls_run() {
828 let node = Node::<RejectingPreflightNode>::from_config(FakeConfig).await.unwrap();
829 let ctx = Context::new("test-node-rejecting").await.unwrap();
830
831 let ran_handle = { node.node.lock().await.ran.clone() };
832
833 let result = node.start(ctx).await;
834
835 assert!(result.is_err());
836 assert!(result.unwrap_err().to_string().contains("boom: bad node content"));
837 assert!(
838 !ran_handle.load(Ordering::SeqCst),
839 "run() must not execute when preflight() rejects the node"
840 );
841 }
842
843 #[test]
844 fn node_type_kebab_case_matches_existing_builtin_node_types_strings() {
845 let expected: &[(NodeType, &str)] = &[
846 (NodeType::Imu, "imu"),
847 (NodeType::Camera, "camera"),
848 (NodeType::ImageClassifier, "image-classifier"),
849 (NodeType::ObjectDetector, "object-detector"),
850 (NodeType::BehaviorExecutor, "behavior-executor"),
851 (NodeType::LlmCommand, "llm-command"),
852 (NodeType::Teleop, "teleop"),
853 (NodeType::Motor, "motor"),
854 (NodeType::WebsocketBridge, "websocket-bridge"),
855 (NodeType::VideoStreamer, "video-streamer"),
856 (NodeType::Simulator, "simulator"),
857 (NodeType::Speaker, "speaker"),
858 (NodeType::Listener, "listener"),
859 (NodeType::Rl, "rl"),
860 (NodeType::InstanceTrack, "instance-track"),
861 (NodeType::Diagnostics, "diagnostics"),
862 (NodeType::Servo, "servo"),
863 (NodeType::PanTilt, "pan-tilt"),
864 ];
865
866 assert_eq!(expected.len(), NodeType::ALL.len());
867
868 for (variant, kebab) in expected {
869 let json = serde_json::to_string(variant).unwrap();
870 assert_eq!(json, format!("\"{}\"", kebab));
871 assert_eq!(variant.as_str(), *kebab);
872
873 let round_tripped: NodeType = serde_json::from_str(&json).unwrap();
874 assert_eq!(round_tripped, *variant);
875 }
876 }
877
878 #[test]
879 fn node_tag_kebab_case_matches_expected_strings() {
880 let expected: &[(NodeTag, &str)] = &[
881 (NodeTag::Sensor, "sensor"),
882 (NodeTag::Inertial, "inertial"),
883 (NodeTag::Capture, "capture"),
884 (NodeTag::Ai, "ai"),
885 (NodeTag::Classifier, "classifier"),
886 (NodeTag::Detector, "detector"),
887 (NodeTag::Vision, "vision"),
888 (NodeTag::Planner, "planner"),
889 (NodeTag::Control, "control"),
890 (NodeTag::Nlp, "nlp"),
891 (NodeTag::Input, "input"),
892 (NodeTag::Actuator, "actuator"),
893 (NodeTag::Motion, "motion"),
894 (NodeTag::Relay, "relay"),
895 (NodeTag::Websocket, "websocket"),
896 (NodeTag::Streamer, "streamer"),
897 (NodeTag::Video, "video"),
898 (NodeTag::Simulator, "simulator"),
899 (NodeTag::Logger, "logger"),
900 ];
901
902 for (variant, kebab) in expected {
903 let json = serde_json::to_string(variant).unwrap();
904 assert_eq!(json, format!("\"{}\"", kebab));
905
906 let round_tripped: NodeTag = serde_json::from_str(&json).unwrap();
907 assert_eq!(round_tripped, *variant);
908 }
909 }
910
911 #[test]
912 fn capability_kebab_case_matches_expected_strings() {
913 let expected: &[(Capability, &str)] = &[
914 (Capability::Locomotion, "locomotion"),
915 (Capability::Joints, "joints"),
916 (Capability::Flight, "flight"),
917 (Capability::Video, "video"),
918 (Capability::Vision, "vision"),
919 (Capability::Audio, "audio"),
920 ];
921
922 for (variant, kebab) in expected {
923 let json = serde_json::to_string(variant).unwrap();
924 assert_eq!(json, format!("\"{}\"", kebab));
925
926 let round_tripped: Capability = serde_json::from_str(&json).unwrap();
927 assert_eq!(round_tripped, *variant);
928 }
929 }
930
931 #[test]
932 fn node_impl_defaults_node_type_to_none_and_tags_to_empty() {
933 assert_eq!(OkNode::NODE_TYPE, None);
934 assert!(OkNode::NODE_TAGS.is_empty());
935 }
936
937 #[test]
938 fn node_impl_defaults_capabilities_to_empty() {
939 assert!(OkNode::CAPABILITIES.is_empty());
940 }
941
942 #[test]
943 fn node_impl_defaults_registration_consts() {
944 assert_eq!(OkNode::CRATE_NAME, None);
945 assert_eq!(OkNode::DESCRIPTION, "");
946 assert_eq!(OkNode::DEFAULT_RUN_TARGET, "robot");
947 assert!(OkNode::CATALOG_TAGS.is_empty());
948 }
949}