1use crate::audit::{AuditCategory, AuditEntry, AuditOutcome, AuditRecord};
14use crate::context::RuntimeContext;
15use crate::envelope::CommandEnvelope;
16use crate::error::{RuntimeError, RuntimeResult};
17use crate::handler::CommandResult;
18use crate::idempotency::{IdempotencyStore, InMemoryIdempotencyStore};
19use crate::lifecycle::{RuntimeLifecycle, RuntimeLifecycleEvent, RuntimeLifecycleState};
20use crate::runtime::RuntimeInstance;
21use parking_lot::{Condvar, Mutex};
22use std::sync::Arc;
23use std::time::{Duration, Instant};
24
25mod completion;
26use completion::CompletionBuffer;
27
28pub struct RuntimeController {
30 instance: Arc<RuntimeInstance>,
31 idempotency: Arc<Mutex<Box<dyn IdempotencyStore + Send + Sync>>>,
32 activity: Arc<CommandActivity>,
33 completions: Arc<Mutex<CompletionBuffer>>,
34}
35
36impl Clone for RuntimeController {
37 fn clone(&self) -> Self {
38 Self {
39 instance: Arc::clone(&self.instance),
40 idempotency: Arc::clone(&self.idempotency),
41 activity: Arc::clone(&self.activity),
42 completions: Arc::clone(&self.completions),
43 }
44 }
45}
46
47#[derive(Default)]
48struct CommandActivity {
49 state: Mutex<CommandActivityState>,
50 idle: Condvar,
51}
52
53#[derive(Default)]
54struct CommandActivityState {
55 accepting: bool,
56 inflight: usize,
57}
58
59struct CommandActivityGuard {
60 activity: Arc<CommandActivity>,
61}
62
63impl Drop for CommandActivityGuard {
64 fn drop(&mut self) {
65 let mut state = self.activity.state.lock();
66 state.inflight = state.inflight.saturating_sub(1);
67 if state.inflight == 0 {
68 self.activity.idle.notify_all();
69 }
70 }
71}
72
73impl RuntimeController {
74 pub fn new(instance: RuntimeInstance) -> Self {
76 Self {
77 instance: Arc::new(instance),
78 idempotency: Arc::new(Mutex::new(Box::new(InMemoryIdempotencyStore::new()))),
79 activity: Arc::new(CommandActivity::accepting()),
80 completions: Arc::new(Mutex::new(CompletionBuffer::default())),
81 }
82 }
83
84 pub fn with_idempotency_store(
86 instance: RuntimeInstance,
87 idempotency: Box<dyn IdempotencyStore + Send + Sync>,
88 ) -> Self {
89 Self {
90 instance: Arc::new(instance),
91 idempotency: Arc::new(Mutex::new(idempotency)),
92 activity: Arc::new(CommandActivity::accepting()),
93 completions: Arc::new(Mutex::new(CompletionBuffer::default())),
94 }
95 }
96
97 pub fn instance(&self) -> &RuntimeInstance {
99 &self.instance
100 }
101
102 pub fn instance_arc(&self) -> Arc<RuntimeInstance> {
104 self.instance.clone()
105 }
106
107 pub fn lifecycle(&self) -> &RuntimeLifecycle {
109 self.instance.lifecycle()
110 }
111
112 pub fn idempotency_len(&self) -> usize {
114 self.idempotency.lock().len()
115 }
116
117 pub fn idempotency_contains(&self, key: &str) -> RuntimeResult<bool> {
119 Ok(self.idempotency.lock().get(key)?.is_some())
120 }
121
122 pub fn apply_lifecycle_event(
124 &self,
125 event: RuntimeLifecycleEvent,
126 ) -> RuntimeResult<RuntimeLifecycleState> {
127 match event {
128 RuntimeLifecycleEvent::ShutdownRequested => {
129 let mut activity = self.activity.state.lock();
130 let state = self.instance.lifecycle().apply(event)?;
131 activity.accepting = false;
132 Ok(state)
133 }
134 RuntimeLifecycleEvent::ShutdownCompleted => {
135 if self.inflight_commands() != 0 {
136 return Err(RuntimeError::CommandRejected);
137 }
138 self.instance.lifecycle().apply(event)
139 }
140 _ => self.instance.lifecycle().apply(event),
141 }
142 }
143
144 pub fn dispatch_command(
151 &self,
152 command: &CommandEnvelope,
153 context: &dyn RuntimeContext,
154 ) -> RuntimeResult<CommandResult> {
155 let _activity = match self.admit_dispatch(command) {
156 Ok(activity) => activity,
157 Err(rejection) => return Ok(rejection),
158 };
159 match self.reserve_idempotency(command)? {
160 Some(Ok(replay)) => Ok(replay),
161 Some(Err(err)) => Err(err),
162 None => {
163 let result = self.instance.dispatch_command(command, context);
164 self.post_dispatch(command, &result)?;
165 result
166 }
167 }
168 }
169
170 pub fn inflight_commands(&self) -> usize {
172 self.activity.state.lock().inflight
173 }
174
175 pub fn wait_for_inflight(&self, timeout: Duration) -> bool {
180 let started = Instant::now();
181 let mut activity = self.activity.state.lock();
182 while activity.inflight != 0 {
183 let remaining = timeout.saturating_sub(started.elapsed());
184 if remaining.is_zero() {
185 return false;
186 }
187 self.activity.idle.wait_for(&mut activity, remaining);
188 }
189 true
190 }
191
192 pub fn pre_dispatch(
194 &self,
195 command: &CommandEnvelope,
196 ) -> RuntimeResult<Option<RuntimeResult<CommandResult>>> {
197 if let Some(rejection) = self.check_lifecycle_readiness() {
198 let res = Ok(rejection);
199 self.record_audit(command, &res);
200 return Ok(Some(res));
201 }
202
203 self.reserve_idempotency(command)
204 }
205
206 fn reserve_idempotency(
207 &self,
208 command: &CommandEnvelope,
209 ) -> RuntimeResult<Option<RuntimeResult<CommandResult>>> {
210 let reservation = self.reserve_completion(command);
211 if let Err(error) = &reservation {
212 self.record_audit(command, &Err(error.clone()));
213 }
214 reservation
215 }
216
217 fn check_lifecycle_readiness(&self) -> Option<CommandResult> {
218 match self.lifecycle().current() {
219 RuntimeLifecycleState::Running | RuntimeLifecycleState::Degraded => None,
220 RuntimeLifecycleState::Restricted => {
221 Some(CommandResult::rejected("runtime is restricted"))
222 }
223 _ => Some(CommandResult::rejected("runtime is not ready")),
224 }
225 }
226
227 fn admit_dispatch(
228 &self,
229 command: &CommandEnvelope,
230 ) -> Result<CommandActivityGuard, CommandResult> {
231 let mut activity = self.activity.state.lock();
232 let rejection = if activity.accepting {
233 self.check_lifecycle_readiness()
234 } else {
235 Some(CommandResult::rejected("runtime is not ready"))
236 };
237 if let Some(rejection) = rejection {
238 let result = Ok(rejection.clone());
239 drop(activity);
240 self.record_audit(command, &result);
241 return Err(rejection);
242 }
243 activity.inflight = activity.inflight.saturating_add(1);
244 Ok(CommandActivityGuard {
245 activity: Arc::clone(&self.activity),
246 })
247 }
248
249 fn emit_events_and_audit(
250 &self,
251 command: &CommandEnvelope,
252 dispatch_result: &RuntimeResult<CommandResult>,
253 ) {
254 if let Ok(result) = dispatch_result {
255 if result.is_accepted() {
256 let events = result
257 .events()
258 .iter()
259 .cloned()
260 .map(|event| {
261 if event.trace.is_none() {
262 if let Some(trace) = &command.trace {
263 return event.with_trace(trace.clone());
264 }
265 }
266 event
267 })
268 .collect::<Vec<_>>();
269 for event in &events {
270 let completed_at_ms = now_ms();
271 self.instance.audit_log().push_entry(
272 AuditEntry::new(
273 AuditCategory::Event,
274 event.event_id.clone(),
275 event.event_name.as_str(),
276 event.occurred_at_ms,
277 completed_at_ms,
278 AuditOutcome::Accepted,
279 )
280 .with_runtime_scope(&event.app_id, &event.node_id)
281 .with_trace(event.trace.clone()),
282 );
283 }
284 self.instance.event_bus().emit_many(events);
285 }
286 }
287 self.record_audit(command, dispatch_result);
288 }
289
290 fn record_audit(
291 &self,
292 command: &CommandEnvelope,
293 dispatch_result: &RuntimeResult<CommandResult>,
294 ) {
295 let record = match dispatch_result {
296 Ok(result) => AuditRecord {
297 command_id: command.command_id.clone(),
298 command_name: command.command_name.clone(),
299 app_id: command.app_id.clone(),
300 node_id: command.node_id.clone(),
301 timestamp_ms: command.issued_at_ms,
302 outcome: if result.is_accepted() {
303 AuditOutcome::Accepted
304 } else {
305 AuditOutcome::Rejected
306 },
307 message: result.message().map(|msg| msg.to_string()),
308 trace: command.trace.clone(),
309 },
310 Err(error) => AuditRecord {
311 command_id: command.command_id.clone(),
312 command_name: command.command_name.clone(),
313 app_id: command.app_id.clone(),
314 node_id: command.node_id.clone(),
315 timestamp_ms: command.issued_at_ms,
316 outcome: AuditOutcome::Error,
317 message: Some(format!("{error:?}")),
318 trace: command.trace.clone(),
319 },
320 };
321
322 self.instance.audit_log().push(record);
323 }
324}
325
326impl CommandActivity {
327 fn accepting() -> Self {
328 Self {
329 state: Mutex::new(CommandActivityState {
330 accepting: true,
331 inflight: 0,
332 }),
333 idle: Condvar::new(),
334 }
335 }
336}
337
338fn now_ms() -> u64 {
339 std::time::SystemTime::now()
340 .duration_since(std::time::UNIX_EPOCH)
341 .map(|d| d.as_millis() as u64)
342 .unwrap_or(0)
343}
344
345#[cfg(test)]
346mod controller_tests;