1pub mod types;
56
57pub mod wire_identifier;
58
59pub mod messages;
60
61pub mod sensor;
62
63pub mod actuator;
64
65pub mod teleop;
66
67pub mod on_demand;
68
69pub mod topics;
70
71pub mod topic_utils;
72
73pub mod error;
74
75pub mod fs_utils;
76
77pub mod rwlock_collections;
78
79pub mod context;
80
81#[cfg(feature = "messaging")]
82pub mod stream;
83
84#[cfg(feature = "messaging")]
85pub mod publish;
86
87pub mod node;
88
89pub mod node_registry;
90
91pub mod config;
92
93pub mod config_watcher;
94
95pub mod model;
96
97pub mod macros;
98
99#[cfg(feature = "messaging")]
100pub mod rpc;
101
102#[cfg(feature = "messaging")]
103pub mod rpc_breaker;
104
105#[cfg(feature = "messaging")]
106pub mod rpc_streaming;
107
108pub mod discovery;
109
110pub mod schema;
111
112pub mod auth;
113
114mod redis_list_util;
115
116pub mod secrets;
117
118#[cfg(feature = "messaging")]
119pub mod health;
120
121#[cfg(feature = "messaging")]
122pub mod log_publisher;
123
124pub mod metrics;
125
126pub mod rate_limit;
127
128pub mod circuit_breaker;
129
130#[cfg(feature = "messaging")]
131pub mod dead_letter;
132
133#[cfg(feature = "schema-validation")]
134pub mod schema_validation;
135
136#[cfg(feature = "hal-compat")]
137pub mod hal_compat;
138
139pub mod dependency;
140
141pub mod kinematics;
142
143pub mod geometry;
144
145#[cfg(feature = "messaging")]
146pub mod lifecycle;
147
148pub mod topic_registry;
149
150pub mod state;
151
152pub mod recovery;
153
154#[cfg(feature = "messaging")]
155pub mod tracing_otel;
156
157pub mod profiling;
158
159pub mod priority_queue;
160
161pub mod service;
162#[cfg(feature = "messaging")]
163pub mod service_manager;
164
165pub mod aggregated;
166
167#[cfg(feature = "messaging")]
168pub mod recorder;
169
170#[cfg(feature = "messaging")]
171pub mod redis_bridge;
172
173#[cfg(feature = "messaging")]
174pub mod behavior_interrupt;
175
176#[cfg(feature = "messaging")]
177pub mod command_relay;
178
179pub mod relay_topics;
180
181pub mod node_config_base;
182
183pub mod ollama;
184
185pub mod prelude;
186
187pub use context::{Context, Receiver};
188pub use error::{Mecha10Error, Result};
189pub use messages::Message;
190pub use types::*;
191
192pub use inventory;
196
197use std::collections::HashMap;
198use std::process::Stdio;
199use std::sync::Arc;
200use thiserror::Error;
201use tokio::process::Child;
202use tokio::sync::RwLock;
203use tokio::time::{interval, Duration};
204
205use crate::rwlock_collections::RwLockMapExt;
206
207#[derive(Debug, Error)]
208pub enum RuntimeError {
209 #[error("IO error: {0}")]
210 Io(#[from] std::io::Error),
211
212 #[error("Node error: {0}")]
213 Node(String),
214
215 #[error("Configuration error: {0}")]
216 Config(String),
217
218 #[error("Process error: {0}")]
219 Process(String),
220
221 #[error("Health check failed: {0}")]
222 HealthCheck(String),
223}
224
225pub type RuntimeResult<T> = std::result::Result<T, RuntimeError>;
226
227struct ManagedNode {
228 config: NodeConfig,
229 state: NodeState,
230 process: Option<Child>,
231 restart_count: u32,
232 last_health_check: std::time::Instant,
233}
234
235impl ManagedNode {
236 fn new(config: NodeConfig) -> Self {
237 Self {
238 config,
239 state: NodeState::Stopped,
240 process: None,
241 restart_count: 0,
242 last_health_check: std::time::Instant::now(),
243 }
244 }
245
246 async fn start(&mut self) -> RuntimeResult<()> {
247 if self.state == NodeState::Running {
248 return Ok(());
249 }
250
251 self.state = NodeState::Starting;
252
253 let mut cmd = tokio::process::Command::new(&self.config.executable);
254 cmd.args(&self.config.args);
255 cmd.envs(&self.config.env);
256 cmd.stdout(Stdio::piped());
257 cmd.stderr(Stdio::piped());
258
259 let child = cmd
260 .spawn()
261 .map_err(|e| RuntimeError::Process(format!("Failed to spawn {}: {}", self.config.id, e)))?;
262
263 self.process = Some(child);
264 self.state = NodeState::Running;
265
266 tracing::info!("Started node: {}", self.config.id);
267
268 Ok(())
269 }
270
271 async fn stop(&mut self) -> RuntimeResult<()> {
272 if let Some(mut child) = self.process.take() {
273 self.state = NodeState::Stopping;
274
275 #[cfg(unix)]
276 {
277 use nix::sys::signal::{self, Signal};
278 use nix::unistd::Pid;
279
280 if let Some(pid) = child.id() {
281 let _ = signal::kill(Pid::from_raw(pid as i32), Signal::SIGTERM);
282 }
283 }
284
285 match tokio::time::timeout(Duration::from_secs(5), child.wait()).await {
286 Ok(Ok(status)) => {
287 tracing::info!("Node {} stopped: {:?}", self.config.id, status);
288 }
289 Ok(Err(e)) => {
290 tracing::error!("Error waiting for node {}: {}", self.config.id, e);
291 }
292 Err(_) => {
293 tracing::warn!("Node {} did not stop gracefully, killing", self.config.id);
294 let _ = child.kill().await;
295 }
296 }
297
298 self.state = NodeState::Stopped;
299 }
300
301 Ok(())
302 }
303
304 async fn check_health(&mut self) -> RuntimeResult<bool> {
305 if let Some(child) = &mut self.process {
306 match child.try_wait() {
307 Ok(Some(status)) => {
308 tracing::warn!("Node {} exited: {:?}", self.config.id, status);
309 self.state = NodeState::Crashed;
310 self.process = None;
311 return Ok(false);
312 }
313 Ok(None) => {
314 self.last_health_check = std::time::Instant::now();
315 return Ok(true);
316 }
317 Err(e) => {
318 tracing::error!("Error checking node {}: {}", self.config.id, e);
319 self.state = NodeState::Unhealthy;
320 return Ok(false);
321 }
322 }
323 }
324
325 Ok(false)
326 }
327
328 fn should_restart(&self) -> bool {
329 self.config.auto_restart
330 && (self.state == NodeState::Crashed || self.state == NodeState::Unhealthy)
331 && self.restart_count < 3
332 }
333}
334
335pub struct Runtime {
336 #[allow(dead_code)]
337 robot_id: String,
338
339 nodes: Arc<RwLock<HashMap<String, ManagedNode>>>,
340
341 running: Arc<RwLock<bool>>,
342}
343
344impl Runtime {
345 pub async fn new(robot_id: &str) -> RuntimeResult<Self> {
347 Ok(Self {
348 robot_id: robot_id.to_string(),
349 nodes: Arc::new(RwLock::new(HashMap::new())),
350 running: Arc::new(RwLock::new(false)),
351 })
352 }
353
354 pub async fn start_node(&mut self, config: NodeConfig) -> RuntimeResult<()> {
356 let node_id = config.id.clone();
357
358 let mut nodes = self.nodes.write().await;
359
360 if nodes.contains_key(&node_id) {
361 return Err(RuntimeError::Node(format!("Node {} already exists", node_id)));
362 }
363
364 let mut node = ManagedNode::new(config);
365 node.start().await?;
366
367 nodes.insert(node_id, node);
368
369 Ok(())
370 }
371
372 pub async fn stop_node(&mut self, node_id: &str) -> RuntimeResult<()> {
374 let mut nodes = self.nodes.write().await;
375
376 if let Some(node) = nodes.get_mut(node_id) {
377 node.stop().await?;
378 nodes.remove(node_id);
379 Ok(())
380 } else {
381 Err(RuntimeError::Node(format!("Node {} not found", node_id)))
382 }
383 }
384
385 pub async fn node_state(&self, node_id: &str) -> Option<NodeState> {
387 let nodes = self.nodes.read().await;
388 nodes.get(node_id).map(|n| n.state.clone())
389 }
390
391 pub async fn list_nodes(&self) -> Vec<String> {
393 self.nodes.cloned_keys().await
394 }
395
396 pub async fn run(&mut self) -> RuntimeResult<()> {
398 *self.running.write().await = true;
399
400 let nodes = Arc::clone(&self.nodes);
401 let running = Arc::clone(&self.running);
402
403 let health_check_task = tokio::spawn(async move {
404 let mut interval = interval(Duration::from_secs(5));
405
406 while *running.read().await {
407 interval.tick().await;
408
409 let mut nodes = nodes.write().await;
410 let node_ids: Vec<String> = nodes.keys().cloned().collect();
411
412 for node_id in node_ids {
413 if let Some(node) = nodes.get_mut(&node_id) {
414 if node.state == NodeState::Running {
415 match node.check_health().await {
416 Ok(true) => {}
417 Ok(false) => {
418 if node.should_restart() {
419 tracing::info!("Auto-restarting node: {}", node_id);
420 let _ = node.start().await;
421 node.restart_count += 1;
422 }
423 }
424 Err(e) => {
425 tracing::error!("Health check error for {}: {}", node_id, e);
426 }
427 }
428 }
429 }
430 }
431 }
432 });
433
434 tokio::signal::ctrl_c()
435 .await
436 .map_err(|e| RuntimeError::Process(format!("Error waiting for shutdown: {}", e)))?;
437
438 tracing::info!("Shutting down runtime...");
439 *self.running.write().await = false;
440
441 health_check_task.abort();
442
443 let mut nodes = self.nodes.write().await;
444 for (node_id, node) in nodes.iter_mut() {
445 tracing::info!("Stopping node: {}", node_id);
446 let _ = node.stop().await;
447 }
448
449 Ok(())
450 }
451
452 pub async fn shutdown(&mut self) -> RuntimeResult<()> {
454 *self.running.write().await = false;
455
456 let mut nodes = self.nodes.write().await;
457 for (node_id, node) in nodes.iter_mut() {
458 tracing::info!("Stopping node: {}", node_id);
459 let _ = node.stop().await;
460 }
461
462 Ok(())
463 }
464}