1use crate::audit::{AuditCategory, AuditEntry, AuditOutcome, AuditRecord};
14use crate::context::RuntimeContext;
15use crate::envelope::CommandEnvelope;
16use crate::error::{RuntimeError, RuntimeResult};
17use crate::handler::CommandResult;
18use crate::idempotency::{
19 IdempotencyRecord, IdempotencyStatus, IdempotencyStore, InMemoryIdempotencyStore,
20};
21use crate::lifecycle::{RuntimeLifecycle, RuntimeLifecycleEvent, RuntimeLifecycleState};
22use crate::runtime::RuntimeInstance;
23use parking_lot::{Condvar, Mutex};
24use std::sync::Arc;
25use std::time::{Duration, Instant};
26
27pub struct RuntimeController {
29 instance: Arc<RuntimeInstance>,
30 idempotency: Arc<Mutex<Box<dyn IdempotencyStore + Send + Sync>>>,
31 activity: Arc<CommandActivity>,
32}
33
34impl Clone for RuntimeController {
35 fn clone(&self) -> Self {
36 Self {
37 instance: Arc::clone(&self.instance),
38 idempotency: Arc::clone(&self.idempotency),
39 activity: Arc::clone(&self.activity),
40 }
41 }
42}
43
44#[derive(Default)]
45struct CommandActivity {
46 state: Mutex<CommandActivityState>,
47 idle: Condvar,
48}
49
50#[derive(Default)]
51struct CommandActivityState {
52 accepting: bool,
53 inflight: usize,
54}
55
56struct CommandActivityGuard {
57 activity: Arc<CommandActivity>,
58}
59
60impl Drop for CommandActivityGuard {
61 fn drop(&mut self) {
62 let mut state = self.activity.state.lock();
63 state.inflight = state.inflight.saturating_sub(1);
64 if state.inflight == 0 {
65 self.activity.idle.notify_all();
66 }
67 }
68}
69
70impl RuntimeController {
71 pub fn new(instance: RuntimeInstance) -> Self {
73 Self {
74 instance: Arc::new(instance),
75 idempotency: Arc::new(Mutex::new(Box::new(InMemoryIdempotencyStore::new()))),
76 activity: Arc::new(CommandActivity::accepting()),
77 }
78 }
79
80 pub fn with_idempotency_store(
82 instance: RuntimeInstance,
83 idempotency: Box<dyn IdempotencyStore + Send + Sync>,
84 ) -> Self {
85 Self {
86 instance: Arc::new(instance),
87 idempotency: Arc::new(Mutex::new(idempotency)),
88 activity: Arc::new(CommandActivity::accepting()),
89 }
90 }
91
92 pub fn instance(&self) -> &RuntimeInstance {
94 &self.instance
95 }
96
97 pub fn instance_arc(&self) -> Arc<RuntimeInstance> {
99 self.instance.clone()
100 }
101
102 pub fn lifecycle(&self) -> &RuntimeLifecycle {
104 self.instance.lifecycle()
105 }
106
107 pub fn idempotency_len(&self) -> usize {
109 self.idempotency.lock().len()
110 }
111
112 pub fn idempotency_contains(&self, key: &str) -> RuntimeResult<bool> {
114 Ok(self.idempotency.lock().get(key)?.is_some())
115 }
116
117 pub fn apply_lifecycle_event(
119 &self,
120 event: RuntimeLifecycleEvent,
121 ) -> RuntimeResult<RuntimeLifecycleState> {
122 match event {
123 RuntimeLifecycleEvent::ShutdownRequested => {
124 let mut activity = self.activity.state.lock();
125 let state = self.instance.lifecycle().apply(event)?;
126 activity.accepting = false;
127 Ok(state)
128 }
129 RuntimeLifecycleEvent::ShutdownCompleted => {
130 if self.inflight_commands() != 0 {
131 return Err(RuntimeError::CommandRejected);
132 }
133 self.instance.lifecycle().apply(event)
134 }
135 _ => self.instance.lifecycle().apply(event),
136 }
137 }
138
139 pub fn dispatch_command(
141 &self,
142 command: &CommandEnvelope,
143 context: &dyn RuntimeContext,
144 ) -> RuntimeResult<CommandResult> {
145 let _activity = match self.admit_dispatch(command) {
146 Ok(activity) => activity,
147 Err(rejection) => return Ok(rejection),
148 };
149 match self.reserve_idempotency(command)? {
150 Some(Ok(replay)) => Ok(replay),
151 Some(Err(err)) => Err(err),
152 None => {
153 let result = self.instance.dispatch_command(command, context);
154 self.post_dispatch(command, &result)?;
155 result
156 }
157 }
158 }
159
160 pub fn inflight_commands(&self) -> usize {
162 self.activity.state.lock().inflight
163 }
164
165 pub fn wait_for_inflight(&self, timeout: Duration) -> bool {
170 let started = Instant::now();
171 let mut activity = self.activity.state.lock();
172 while activity.inflight != 0 {
173 let remaining = timeout.saturating_sub(started.elapsed());
174 if remaining.is_zero() {
175 return false;
176 }
177 self.activity.idle.wait_for(&mut activity, remaining);
178 }
179 true
180 }
181
182 pub fn pre_dispatch(
184 &self,
185 command: &CommandEnvelope,
186 ) -> RuntimeResult<Option<RuntimeResult<CommandResult>>> {
187 if let Some(rejection) = self.check_lifecycle_readiness() {
188 let res = Ok(rejection);
189 self.record_audit(command, &res);
190 return Ok(Some(res));
191 }
192
193 self.reserve_idempotency(command)
194 }
195
196 fn reserve_idempotency(
197 &self,
198 command: &CommandEnvelope,
199 ) -> RuntimeResult<Option<RuntimeResult<CommandResult>>> {
200 if let Some(key) = command.idempotency_key.as_deref() {
201 let mut store = self.idempotency.lock();
202 if let Some(record) = store.get(key)? {
203 let payload_hash = hash_payload(&command.payload);
204 if record.request_hash != payload_hash {
205 let err = RuntimeError::IdempotencyConflict {
206 key: key.to_string(),
207 };
208 self.record_audit(command, &Err(err.clone()));
209 return Err(err);
210 }
211 match record.status {
212 IdempotencyStatus::Pending => {
213 let err = RuntimeError::IdempotencyPending {
214 key: key.to_string(),
215 };
216 self.record_audit(command, &Err(err.clone()));
217 return Err(err);
218 }
219 IdempotencyStatus::Resolved {
220 response_status,
221 ref response_body,
222 } => {
223 if response_status >= 400 {
224 store.remove(key)?;
225 return Ok(None);
226 }
227 let result = if response_body.is_empty() {
228 CommandResult::accepted(Vec::new())
229 } else {
230 serde_json::from_str::<CommandResult>(response_body).map_err(|e| {
231 RuntimeError::IdempotencyStoreIo {
232 operation: "deserialize_replay",
233 message: e.to_string(),
234 }
235 })?
236 };
237 let res = Ok(result);
238 self.record_audit(command, &res);
239 return Ok(Some(res));
240 }
241 }
242 }
243
244 let payload_hash = hash_payload(&command.payload);
246 let record = IdempotencyRecord {
247 key: key.to_string(),
248 request_hash: payload_hash,
249 status: IdempotencyStatus::Pending,
250 created_at_ms: now_ms(),
251 };
252 store.insert(record)?;
253 }
254
255 Ok(None)
256 }
257
258 pub fn post_dispatch(
260 &self,
261 command: &CommandEnvelope,
262 dispatch_result: &RuntimeResult<CommandResult>,
263 ) -> RuntimeResult<()> {
264 if let Some(key) = command.idempotency_key.as_ref() {
265 let mut store = self.idempotency.lock();
266 match dispatch_result {
267 Ok(result) => {
268 let response_body = serde_json::to_string(result).map_err(|e| {
269 RuntimeError::IdempotencyStoreIo {
270 operation: "serialize_response",
271 message: e.to_string(),
272 }
273 })?;
274 let record = IdempotencyRecord {
275 key: key.to_string(),
276 request_hash: hash_payload(&command.payload),
277 status: IdempotencyStatus::Resolved {
278 response_status: 200,
279 response_body,
280 },
281 created_at_ms: now_ms(),
282 };
283 store.insert(record)?;
284 }
285 Err(_) => {
286 store.remove(key)?;
287 }
288 }
289 }
290
291 self.emit_events_and_audit(command, dispatch_result);
292 Ok(())
293 }
294
295 fn check_lifecycle_readiness(&self) -> Option<CommandResult> {
296 match self.lifecycle().current() {
297 RuntimeLifecycleState::Running | RuntimeLifecycleState::Degraded => None,
298 RuntimeLifecycleState::Restricted => {
299 Some(CommandResult::rejected("runtime is restricted"))
300 }
301 _ => Some(CommandResult::rejected("runtime is not ready")),
302 }
303 }
304
305 fn admit_dispatch(
306 &self,
307 command: &CommandEnvelope,
308 ) -> Result<CommandActivityGuard, CommandResult> {
309 let mut activity = self.activity.state.lock();
310 let rejection = if activity.accepting {
311 self.check_lifecycle_readiness()
312 } else {
313 Some(CommandResult::rejected("runtime is not ready"))
314 };
315 if let Some(rejection) = rejection {
316 let result = Ok(rejection.clone());
317 drop(activity);
318 self.record_audit(command, &result);
319 return Err(rejection);
320 }
321 activity.inflight = activity.inflight.saturating_add(1);
322 Ok(CommandActivityGuard {
323 activity: Arc::clone(&self.activity),
324 })
325 }
326
327 fn emit_events_and_audit(
328 &self,
329 command: &CommandEnvelope,
330 dispatch_result: &RuntimeResult<CommandResult>,
331 ) {
332 if let Ok(result) = dispatch_result {
333 if result.is_accepted() {
334 let events = result
335 .events()
336 .iter()
337 .cloned()
338 .map(|event| {
339 if event.trace.is_none() {
340 if let Some(trace) = &command.trace {
341 return event.with_trace(trace.clone());
342 }
343 }
344 event
345 })
346 .collect::<Vec<_>>();
347 for event in &events {
348 let completed_at_ms = now_ms();
349 self.instance.audit_log().push_entry(
350 AuditEntry::new(
351 AuditCategory::Event,
352 event.event_id.clone(),
353 event.event_name.as_str(),
354 event.occurred_at_ms,
355 completed_at_ms,
356 AuditOutcome::Accepted,
357 )
358 .with_runtime_scope(&event.app_id, &event.node_id)
359 .with_trace(event.trace.clone()),
360 );
361 }
362 self.instance.event_bus().emit_many(events);
363 }
364 }
365 self.record_audit(command, dispatch_result);
366 }
367
368 fn record_audit(
369 &self,
370 command: &CommandEnvelope,
371 dispatch_result: &RuntimeResult<CommandResult>,
372 ) {
373 let record = match dispatch_result {
374 Ok(result) => AuditRecord {
375 command_id: command.command_id.clone(),
376 command_name: command.command_name.clone(),
377 app_id: command.app_id.clone(),
378 node_id: command.node_id.clone(),
379 timestamp_ms: command.issued_at_ms,
380 outcome: if result.is_accepted() {
381 AuditOutcome::Accepted
382 } else {
383 AuditOutcome::Rejected
384 },
385 message: result.message().map(|msg| msg.to_string()),
386 trace: command.trace.clone(),
387 },
388 Err(error) => AuditRecord {
389 command_id: command.command_id.clone(),
390 command_name: command.command_name.clone(),
391 app_id: command.app_id.clone(),
392 node_id: command.node_id.clone(),
393 timestamp_ms: command.issued_at_ms,
394 outcome: AuditOutcome::Error,
395 message: Some(format!("{error:?}")),
396 trace: command.trace.clone(),
397 },
398 };
399
400 self.instance.audit_log().push(record);
401 }
402}
403
404impl CommandActivity {
405 fn accepting() -> Self {
406 Self {
407 state: Mutex::new(CommandActivityState {
408 accepting: true,
409 inflight: 0,
410 }),
411 idle: Condvar::new(),
412 }
413 }
414}
415
416fn hash_payload(payload: &[u8]) -> String {
417 use sha2::{Digest, Sha256};
418 let mut hasher = Sha256::new();
419 hasher.update(payload);
420 let result = hasher.finalize();
421 let mut hex = String::with_capacity(result.len() * 2);
422 for byte in result {
423 hex.push_str(&format!("{:02x}", byte));
424 }
425 hex
426}
427
428fn now_ms() -> u64 {
429 std::time::SystemTime::now()
430 .duration_since(std::time::UNIX_EPOCH)
431 .map(|d| d.as_millis() as u64)
432 .unwrap_or(0)
433}
434
435#[cfg(test)]
436mod controller_tests;