1pub mod types;
56
57pub mod wire_identifier;
58
59pub mod messages;
60
61pub mod sensor;
62
63pub mod actuator;
64
65pub mod teleop;
66
67pub mod topics;
68
69pub mod topic_utils;
70
71pub mod error;
72
73pub mod fs_utils;
74
75pub mod rwlock_collections;
76
77pub mod context;
78
79#[cfg(feature = "messaging")]
80pub mod stream;
81
82#[cfg(feature = "messaging")]
83pub mod publish;
84
85pub mod node;
86
87pub mod node_registry;
88
89pub mod config;
90
91pub mod config_watcher;
92
93pub mod model;
94
95pub mod macros;
96
97#[cfg(feature = "messaging")]
98pub mod rpc;
99
100#[cfg(feature = "messaging")]
101pub mod rpc_breaker;
102
103#[cfg(feature = "messaging")]
104pub mod rpc_streaming;
105
106pub mod discovery;
107
108pub mod schema;
109
110pub mod auth;
111
112mod redis_list_util;
113
114pub mod secrets;
115
116#[cfg(feature = "messaging")]
117pub mod health;
118
119#[cfg(feature = "messaging")]
120pub mod log_publisher;
121
122pub mod metrics;
123
124pub mod rate_limit;
125
126pub mod circuit_breaker;
127
128#[cfg(feature = "messaging")]
129pub mod dead_letter;
130
131#[cfg(feature = "schema-validation")]
132pub mod schema_validation;
133
134#[cfg(feature = "hal-compat")]
135pub mod hal_compat;
136
137pub mod dependency;
138
139pub mod kinematics;
140
141pub mod geometry;
142
143#[cfg(feature = "messaging")]
144pub mod lifecycle;
145
146pub mod topic_registry;
147
148pub mod state;
149
150pub mod recovery;
151
152#[cfg(feature = "messaging")]
153pub mod tracing_otel;
154
155pub mod profiling;
156
157pub mod priority_queue;
158
159pub mod service;
160#[cfg(feature = "messaging")]
161pub mod service_manager;
162
163pub mod aggregated;
164
165#[cfg(feature = "messaging")]
166pub mod recorder;
167
168#[cfg(feature = "messaging")]
169pub mod redis_bridge;
170
171#[cfg(feature = "messaging")]
172pub mod behavior_interrupt;
173
174#[cfg(feature = "messaging")]
175pub mod command_relay;
176
177pub mod relay_topics;
178
179pub mod node_config_base;
180
181pub mod ollama;
182
183pub mod prelude;
184
185pub use context::{Context, Receiver};
186pub use error::{Mecha10Error, Result};
187pub use messages::Message;
188pub use types::*;
189
190pub use inventory;
194
195use std::collections::HashMap;
196use std::process::Stdio;
197use std::sync::Arc;
198use thiserror::Error;
199use tokio::process::Child;
200use tokio::sync::RwLock;
201use tokio::time::{interval, Duration};
202
203use crate::rwlock_collections::RwLockMapExt;
204
205#[derive(Debug, Error)]
206pub enum RuntimeError {
207 #[error("IO error: {0}")]
208 Io(#[from] std::io::Error),
209
210 #[error("Node error: {0}")]
211 Node(String),
212
213 #[error("Configuration error: {0}")]
214 Config(String),
215
216 #[error("Process error: {0}")]
217 Process(String),
218
219 #[error("Health check failed: {0}")]
220 HealthCheck(String),
221}
222
223pub type RuntimeResult<T> = std::result::Result<T, RuntimeError>;
224
225struct ManagedNode {
226 config: NodeConfig,
227 state: NodeState,
228 process: Option<Child>,
229 restart_count: u32,
230 last_health_check: std::time::Instant,
231}
232
233impl ManagedNode {
234 fn new(config: NodeConfig) -> Self {
235 Self {
236 config,
237 state: NodeState::Stopped,
238 process: None,
239 restart_count: 0,
240 last_health_check: std::time::Instant::now(),
241 }
242 }
243
244 async fn start(&mut self) -> RuntimeResult<()> {
245 if self.state == NodeState::Running {
246 return Ok(());
247 }
248
249 self.state = NodeState::Starting;
250
251 let mut cmd = tokio::process::Command::new(&self.config.executable);
252 cmd.args(&self.config.args);
253 cmd.envs(&self.config.env);
254 cmd.stdout(Stdio::piped());
255 cmd.stderr(Stdio::piped());
256
257 let child = cmd
258 .spawn()
259 .map_err(|e| RuntimeError::Process(format!("Failed to spawn {}: {}", self.config.id, e)))?;
260
261 self.process = Some(child);
262 self.state = NodeState::Running;
263
264 tracing::info!("Started node: {}", self.config.id);
265
266 Ok(())
267 }
268
269 async fn stop(&mut self) -> RuntimeResult<()> {
270 if let Some(mut child) = self.process.take() {
271 self.state = NodeState::Stopping;
272
273 #[cfg(unix)]
274 {
275 use nix::sys::signal::{self, Signal};
276 use nix::unistd::Pid;
277
278 if let Some(pid) = child.id() {
279 let _ = signal::kill(Pid::from_raw(pid as i32), Signal::SIGTERM);
280 }
281 }
282
283 match tokio::time::timeout(Duration::from_secs(5), child.wait()).await {
284 Ok(Ok(status)) => {
285 tracing::info!("Node {} stopped: {:?}", self.config.id, status);
286 }
287 Ok(Err(e)) => {
288 tracing::error!("Error waiting for node {}: {}", self.config.id, e);
289 }
290 Err(_) => {
291 tracing::warn!("Node {} did not stop gracefully, killing", self.config.id);
292 let _ = child.kill().await;
293 }
294 }
295
296 self.state = NodeState::Stopped;
297 }
298
299 Ok(())
300 }
301
302 async fn check_health(&mut self) -> RuntimeResult<bool> {
303 if let Some(child) = &mut self.process {
304 match child.try_wait() {
305 Ok(Some(status)) => {
306 tracing::warn!("Node {} exited: {:?}", self.config.id, status);
307 self.state = NodeState::Crashed;
308 self.process = None;
309 return Ok(false);
310 }
311 Ok(None) => {
312 self.last_health_check = std::time::Instant::now();
313 return Ok(true);
314 }
315 Err(e) => {
316 tracing::error!("Error checking node {}: {}", self.config.id, e);
317 self.state = NodeState::Unhealthy;
318 return Ok(false);
319 }
320 }
321 }
322
323 Ok(false)
324 }
325
326 fn should_restart(&self) -> bool {
327 self.config.auto_restart
328 && (self.state == NodeState::Crashed || self.state == NodeState::Unhealthy)
329 && self.restart_count < 3
330 }
331}
332
333pub struct Runtime {
334 #[allow(dead_code)]
335 robot_id: String,
336
337 nodes: Arc<RwLock<HashMap<String, ManagedNode>>>,
338
339 running: Arc<RwLock<bool>>,
340}
341
342impl Runtime {
343 pub async fn new(robot_id: &str) -> RuntimeResult<Self> {
345 Ok(Self {
346 robot_id: robot_id.to_string(),
347 nodes: Arc::new(RwLock::new(HashMap::new())),
348 running: Arc::new(RwLock::new(false)),
349 })
350 }
351
352 pub async fn start_node(&mut self, config: NodeConfig) -> RuntimeResult<()> {
354 let node_id = config.id.clone();
355
356 let mut nodes = self.nodes.write().await;
357
358 if nodes.contains_key(&node_id) {
359 return Err(RuntimeError::Node(format!("Node {} already exists", node_id)));
360 }
361
362 let mut node = ManagedNode::new(config);
363 node.start().await?;
364
365 nodes.insert(node_id, node);
366
367 Ok(())
368 }
369
370 pub async fn stop_node(&mut self, node_id: &str) -> RuntimeResult<()> {
372 let mut nodes = self.nodes.write().await;
373
374 if let Some(node) = nodes.get_mut(node_id) {
375 node.stop().await?;
376 nodes.remove(node_id);
377 Ok(())
378 } else {
379 Err(RuntimeError::Node(format!("Node {} not found", node_id)))
380 }
381 }
382
383 pub async fn node_state(&self, node_id: &str) -> Option<NodeState> {
385 let nodes = self.nodes.read().await;
386 nodes.get(node_id).map(|n| n.state.clone())
387 }
388
389 pub async fn list_nodes(&self) -> Vec<String> {
391 self.nodes.cloned_keys().await
392 }
393
394 pub async fn run(&mut self) -> RuntimeResult<()> {
396 *self.running.write().await = true;
397
398 let nodes = Arc::clone(&self.nodes);
399 let running = Arc::clone(&self.running);
400
401 let health_check_task = tokio::spawn(async move {
402 let mut interval = interval(Duration::from_secs(5));
403
404 while *running.read().await {
405 interval.tick().await;
406
407 let mut nodes = nodes.write().await;
408 let node_ids: Vec<String> = nodes.keys().cloned().collect();
409
410 for node_id in node_ids {
411 if let Some(node) = nodes.get_mut(&node_id) {
412 if node.state == NodeState::Running {
413 match node.check_health().await {
414 Ok(true) => {}
415 Ok(false) => {
416 if node.should_restart() {
417 tracing::info!("Auto-restarting node: {}", node_id);
418 let _ = node.start().await;
419 node.restart_count += 1;
420 }
421 }
422 Err(e) => {
423 tracing::error!("Health check error for {}: {}", node_id, e);
424 }
425 }
426 }
427 }
428 }
429 }
430 });
431
432 tokio::signal::ctrl_c()
433 .await
434 .map_err(|e| RuntimeError::Process(format!("Error waiting for shutdown: {}", e)))?;
435
436 tracing::info!("Shutting down runtime...");
437 *self.running.write().await = false;
438
439 health_check_task.abort();
440
441 let mut nodes = self.nodes.write().await;
442 for (node_id, node) in nodes.iter_mut() {
443 tracing::info!("Stopping node: {}", node_id);
444 let _ = node.stop().await;
445 }
446
447 Ok(())
448 }
449
450 pub async fn shutdown(&mut self) -> RuntimeResult<()> {
452 *self.running.write().await = false;
453
454 let mut nodes = self.nodes.write().await;
455 for (node_id, node) in nodes.iter_mut() {
456 tracing::info!("Stopping node: {}", node_id);
457 let _ = node.stop().await;
458 }
459
460 Ok(())
461 }
462}