1use std::collections::HashMap;
14use std::sync::{Arc, Mutex, RwLock as StdRwLock};
15
16use chrono::Utc;
17#[cfg(test)]
18use meerkat_core::SessionId;
19use meerkat_core::lifecycle::{InputId, RunBoundaryReceipt, RunId};
20use meerkat_core::types::HandlingMode;
21
22use crate::accept::{
23 AcceptOutcome, AdmissionPlan, AdmissionQueueAction, CoarseAdmissionFlags,
24 ExistingQueuedAdmissionAction, MachineAdmissionAuthority, RejectReason, ResolvedAdmission,
25};
26use crate::identifiers::{IdempotencyKey, LogicalRuntimeId, PolicyVersion};
27use crate::ingress_types::{
28 ContentShape, RequestId, ReservationKey, RuntimeInputProjection, RuntimeInputSemantics,
29};
30use crate::input::Input;
31use crate::input_ledger::InputLedger;
32use crate::input_state::{
33 InputAbandonReason, InputLifecycleState, InputState, InputStateHistoryEntry,
34 InputStatePersistenceRecord, InputStateSeed, InputTerminalOutcome, PolicySnapshot,
35 StoredInputState,
36};
37use crate::meerkat_machine::dsl as mm_dsl;
38use crate::policy::PolicyDecision;
39use crate::queue::InputQueue;
40use crate::runtime_event::{
41 InputLifecycleEvent, RuntimeEvent, RuntimeEventEnvelope, RuntimeStateChangeEvent,
42};
43use crate::runtime_state::RuntimeState;
44use crate::store::MachineLifecycleBindingFacts;
45use crate::traits::{RecoveryReport, ResetReport, RetireReport, RuntimeDriverError};
46
47#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
53pub enum PostAdmissionSignal {
54 None,
56 WakeLoop,
58 InterruptYielding,
65 RequestImmediateProcessing,
68}
69
70impl PostAdmissionSignal {
71 pub fn should_wake(self) -> bool {
73 self >= Self::WakeLoop
74 }
75
76 pub fn should_interrupt_yielding(self) -> bool {
78 self >= Self::InterruptYielding
79 }
80
81 pub fn should_process_immediately(self) -> bool {
83 self == Self::RequestImmediateProcessing
84 }
85}
86
87#[derive(Debug, Clone, PartialEq, Eq)]
90pub(crate) struct RuntimeControlProjection {
91 pub(crate) phase: RuntimeState,
92 pub(crate) current_run_id: Option<RunId>,
93 pub(crate) pre_run_phase: Option<RuntimeState>,
94}
95
96impl Default for RuntimeControlProjection {
97 fn default() -> Self {
98 Self {
99 phase: RuntimeState::Idle,
100 current_run_id: None,
101 pre_run_phase: None,
102 }
103 }
104}
105
106struct AdmissionValidationFacts<'a> {
107 input_kind: crate::identifiers::InputKind,
108 input_origin: &'a crate::input::InputOrigin,
109 durability: crate::input::InputDurability,
110 peer_handling_mode_valid: bool,
111 peer_response_terminal_structurally_valid: bool,
112 peer_response_terminal_observed_status: mm_dsl::PeerResponseTerminalObservedStatus,
113}
114
115#[derive(Debug, Clone, Default)]
116pub struct ReplayQueuedContributorsPlan {
117 pub queue_work_ids: Vec<InputId>,
118 pub steer_work_ids: Vec<InputId>,
119 pub notice_kind: &'static str,
120}
121
122#[derive(Clone)]
123pub(crate) struct EphemeralDriverRollbackSnapshot {
124 control_projection: RuntimeControlProjection,
125 dsl_snapshot: mm_dsl::MeerkatMachineAuthoritySnapshot,
126 ledger: InputLedger,
127 queue: InputQueue,
128 steer_queue: InputQueue,
129 events: Vec<RuntimeEventEnvelope>,
130 post_admission_signal: PostAdmissionSignal,
131 handling_mode: HashMap<InputId, HandlingMode>,
132 runtime_semantics: HashMap<InputId, RuntimeInputSemantics>,
133 primitive_projection: HashMap<InputId, RuntimeInputProjection>,
134 is_prompt_set: std::collections::HashSet<InputId>,
135 content_shape: HashMap<InputId, ContentShape>,
136 request_id: HashMap<InputId, Option<RequestId>>,
137 reservation_key: HashMap<InputId, Option<ReservationKey>>,
138 policy_snapshot: HashMap<InputId, PolicyDecision>,
139 admission_order: Vec<InputId>,
140}
141
142#[derive(Clone)]
144pub struct EphemeralRuntimeDriver {
145 runtime_id: LogicalRuntimeId,
146 control: Arc<StdRwLock<RuntimeControlProjection>>,
151 ledger: InputLedger,
152 queue: InputQueue,
153 steer_queue: InputQueue,
154 events: Vec<RuntimeEventEnvelope>,
155 post_admission_signal: PostAdmissionSignal,
160 dsl: DslAuthority,
163 handling_mode: HashMap<InputId, HandlingMode>,
168 runtime_semantics: HashMap<InputId, RuntimeInputSemantics>,
169 primitive_projection: HashMap<InputId, RuntimeInputProjection>,
170 is_prompt_set: std::collections::HashSet<InputId>,
171 content_shape: HashMap<InputId, ContentShape>,
172 request_id: HashMap<InputId, Option<RequestId>>,
173 reservation_key: HashMap<InputId, Option<ReservationKey>>,
174 policy_snapshot: HashMap<InputId, PolicyDecision>,
175 admission_order: Vec<InputId>,
179}
180
181pub(crate) type SharedIngressDslAuthority = Arc<Mutex<mm_dsl::MeerkatMachineAuthority>>;
187
188struct DslAuthority(SharedIngressDslAuthority);
189
190impl Clone for DslAuthority {
191 fn clone(&self) -> Self {
192 Self(Arc::clone(&self.0))
193 }
194}
195
196impl DslAuthority {
197 fn lock(&self) -> std::sync::MutexGuard<'_, mm_dsl::MeerkatMachineAuthority> {
198 self.0
199 .lock()
200 .unwrap_or_else(std::sync::PoisonError::into_inner)
201 }
202}
203
204pub(crate) fn new_ingress_dsl_authority() -> SharedIngressDslAuthority {
205 Arc::new(Mutex::new(
206 crate::meerkat_machine::dsl_authority::new_initialized_authority(
207 "ingress DSL authority must initialize",
208 ),
209 ))
210}
211
212fn recover_ingress_dsl_authority(
213 state: mm_dsl::MeerkatMachineState,
214) -> mm_dsl::MeerkatMachineAuthority {
215 crate::meerkat_machine::recover_projected_authority(
216 state,
217 "projected ingress DSL state must be recoverable",
218 )
219}
220
221impl EphemeralRuntimeDriver {
222 fn read_control_projection(&self) -> std::sync::RwLockReadGuard<'_, RuntimeControlProjection> {
223 match self.control.read() {
224 Ok(guard) => guard,
225 Err(poisoned) => {
226 tracing::error!("runtime control projection lock poisoned");
227 poisoned.into_inner()
228 }
229 }
230 }
231
232 fn write_control_projection(
233 &self,
234 ) -> std::sync::RwLockWriteGuard<'_, RuntimeControlProjection> {
235 match self.control.write() {
236 Ok(guard) => guard,
237 Err(poisoned) => {
238 tracing::error!("runtime control projection lock poisoned");
239 poisoned.into_inner()
240 }
241 }
242 }
243
244 pub fn new(runtime_id: LogicalRuntimeId) -> Self {
245 Self::new_with_control_and_dsl(
246 runtime_id,
247 Arc::new(StdRwLock::new(RuntimeControlProjection::default())),
248 new_ingress_dsl_authority(),
249 )
250 }
251
252 pub(crate) fn new_with_control(
253 runtime_id: LogicalRuntimeId,
254 control: Arc<StdRwLock<RuntimeControlProjection>>,
255 ) -> Self {
256 Self::new_with_control_and_dsl(runtime_id, control, new_ingress_dsl_authority())
257 }
258
259 pub(crate) fn new_with_control_and_dsl(
260 runtime_id: LogicalRuntimeId,
261 control: Arc<StdRwLock<RuntimeControlProjection>>,
262 dsl: SharedIngressDslAuthority,
263 ) -> Self {
264 Self {
265 runtime_id,
266 control,
267 ledger: InputLedger::new(),
268 queue: InputQueue::new(),
269 steer_queue: InputQueue::new(),
270 events: Vec::new(),
271 post_admission_signal: PostAdmissionSignal::None,
272 dsl: DslAuthority(dsl),
273 handling_mode: HashMap::new(),
274 runtime_semantics: HashMap::new(),
275 primitive_projection: HashMap::new(),
276 is_prompt_set: std::collections::HashSet::new(),
277 content_shape: HashMap::new(),
278 request_id: HashMap::new(),
279 reservation_key: HashMap::new(),
280 policy_snapshot: HashMap::new(),
281 admission_order: Vec::new(),
282 }
283 }
284
285 pub(crate) fn rollback_snapshot(&self) -> EphemeralDriverRollbackSnapshot {
286 EphemeralDriverRollbackSnapshot {
287 control_projection: self.read_control_projection().clone(),
288 dsl_snapshot: self.dsl.lock().snapshot(),
289 ledger: self.ledger.clone(),
290 queue: self.queue.clone(),
291 steer_queue: self.steer_queue.clone(),
292 events: self.events.clone(),
293 post_admission_signal: self.post_admission_signal,
294 handling_mode: self.handling_mode.clone(),
295 runtime_semantics: self.runtime_semantics.clone(),
296 primitive_projection: self.primitive_projection.clone(),
297 is_prompt_set: self.is_prompt_set.clone(),
298 content_shape: self.content_shape.clone(),
299 request_id: self.request_id.clone(),
300 reservation_key: self.reservation_key.clone(),
301 policy_snapshot: self.policy_snapshot.clone(),
302 admission_order: self.admission_order.clone(),
303 }
304 }
305
306 pub(crate) fn clone_with_isolated_dsl_authority(&self) -> Self {
307 let mut clone = self.clone();
308 let dsl_state = self.with_dsl_state(Clone::clone);
309 clone.dsl = DslAuthority(Arc::new(Mutex::new(recover_ingress_dsl_authority(
310 dsl_state,
311 ))));
312 clone.control = Arc::new(StdRwLock::new(self.read_control_projection().clone()));
313 clone
314 }
315
316 pub(crate) fn restore_rollback_snapshot(&mut self, snapshot: EphemeralDriverRollbackSnapshot) {
317 {
318 let mut control = self.write_control_projection();
319 *control = snapshot.control_projection;
320 }
321 {
322 let mut authority = self.dsl.lock();
323 authority.restore_snapshot(snapshot.dsl_snapshot);
324 }
325 self.ledger = snapshot.ledger;
326 self.queue = snapshot.queue;
327 self.steer_queue = snapshot.steer_queue;
328 self.events = snapshot.events;
329 self.post_admission_signal = snapshot.post_admission_signal;
330 self.handling_mode = snapshot.handling_mode;
331 self.runtime_semantics = snapshot.runtime_semantics;
332 self.primitive_projection = snapshot.primitive_projection;
333 self.is_prompt_set = snapshot.is_prompt_set;
334 self.content_shape = snapshot.content_shape;
335 self.request_id = snapshot.request_id;
336 self.reservation_key = snapshot.reservation_key;
337 self.policy_snapshot = snapshot.policy_snapshot;
338 self.admission_order = snapshot.admission_order;
339 }
340
341 pub(crate) fn shared_dsl_authority(&self) -> SharedIngressDslAuthority {
342 Arc::clone(&self.dsl.0)
343 }
344
345 pub(crate) fn session_authority_id_for_recovery(&self) -> mm_dsl::SessionId {
346 self.with_dsl_state(|state| state.session_id.clone())
347 .unwrap_or_else(|| self.contract_session_authority_id())
348 }
349
350 pub(crate) fn machine_lifecycle_binding_facts(&self) -> MachineLifecycleBindingFacts {
351 self.with_dsl_state(|state| {
352 MachineLifecycleBindingFacts::new(
353 state
354 .active_runtime_id
355 .as_ref()
356 .map(|value| value.0.clone()),
357 state.active_fence_token.map(|token| token.0),
358 state
359 .active_runtime_generation
360 .map(|generation| generation.0),
361 state
362 .active_runtime_epoch_id
363 .as_ref()
364 .map(|value| value.0.clone()),
365 )
366 })
367 }
368
369 pub(crate) fn recover_runtime_authority_from_binding_observation(
370 &mut self,
371 session_id: mm_dsl::SessionId,
372 runtime_phase: RuntimeState,
373 runtime_id: Option<&LogicalRuntimeId>,
374 active_fence_token: Option<u64>,
375 active_runtime_generation: Option<mm_dsl::Generation>,
376 active_runtime_epoch_id: Option<mm_dsl::RuntimeEpochId>,
377 ) -> Result<(), RuntimeDriverError> {
378 let silent_intent_overrides =
379 self.with_dsl_state(|state| state.silent_intent_overrides.clone());
380 self.recover_runtime_authority_from_binding_observation_with_silent_intents(
381 session_id,
382 runtime_phase,
383 runtime_id,
384 active_fence_token,
385 active_runtime_generation,
386 active_runtime_epoch_id,
387 silent_intent_overrides,
388 )
389 }
390
391 #[allow(clippy::too_many_arguments)]
392 fn recover_runtime_authority_from_binding_observation_with_silent_intents(
393 &mut self,
394 session_id: mm_dsl::SessionId,
395 runtime_phase: RuntimeState,
396 runtime_id: Option<&LogicalRuntimeId>,
397 active_fence_token: Option<u64>,
398 active_runtime_generation: Option<mm_dsl::Generation>,
399 active_runtime_epoch_id: Option<mm_dsl::RuntimeEpochId>,
400 silent_intent_overrides: std::collections::BTreeSet<String>,
401 ) -> Result<(), RuntimeDriverError> {
402 let current_run_id = self.current_run_id();
403 let pre_run_phase = self.pre_run_phase();
404 let recovered =
405 crate::meerkat_machine::dsl_authority::recover_authority_from_runtime_observation_id(
406 session_id,
407 runtime_phase,
408 runtime_id,
409 current_run_id.as_ref(),
410 pre_run_phase,
411 silent_intent_overrides,
412 active_fence_token,
413 active_runtime_generation,
414 active_runtime_epoch_id,
415 )
416 .map_err(|err| {
417 RuntimeDriverError::Internal(crate::meerkat_machine::dsl_authority::map_error(
418 err,
419 "persistent runtime recovery authority",
420 ))
421 })?;
422 {
423 let authority = self.shared_dsl_authority();
424 let mut authority = authority
425 .lock()
426 .unwrap_or_else(std::sync::PoisonError::into_inner);
427 *authority = recovered;
428 }
429 self.sync_control_projection_from_dsl_authority();
430 Ok(())
431 }
432
433 fn dsl_apply(
443 &mut self,
444 input: mm_dsl::MeerkatMachineInput,
445 context: &str,
446 ) -> Result<(), RuntimeDriverError> {
447 self.dsl_apply_effects(input, context).map(|_| ())
448 }
449
450 fn dsl_apply_effects(
451 &mut self,
452 input: mm_dsl::MeerkatMachineInput,
453 context: &str,
454 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, RuntimeDriverError> {
455 let transition = {
456 let mut authority = self.dsl.lock();
457 mm_dsl::MeerkatMachineMutator::apply(&mut *authority, input).map_err(|err| {
458 RuntimeDriverError::Internal(format!("DSL rejected {context}: {err:?}"))
459 })?
460 };
461 self.absorb_dsl_effects(transition.effects());
462 Ok(transition.into_effects())
463 }
464
465 fn dsl_preview(
466 &self,
467 input: mm_dsl::MeerkatMachineInput,
468 context: &str,
469 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, RuntimeDriverError> {
470 let state = {
471 let authority = self.dsl.lock();
472 authority.state().clone()
473 };
474 let mut preview =
475 mm_dsl::MeerkatMachineAuthority::recover_from_state(state).map_err(|err| {
476 RuntimeDriverError::Internal(format!("DSL rejected {context}: {err:?}"))
477 })?;
478 mm_dsl::MeerkatMachineMutator::apply(&mut preview, input)
479 .map(|transition| transition.into_effects())
480 .map_err(|err| RuntimeDriverError::Internal(format!("DSL rejected {context}: {err:?}")))
481 }
482
483 fn absorb_dsl_effects(&mut self, effects: &[mm_dsl::MeerkatMachineEffect]) {
490 for effect in effects {
491 if let mm_dsl::MeerkatMachineEffect::PostAdmissionSignal { signal } = effect {
492 let new_signal = match signal {
493 mm_dsl::PostAdmissionSignalKind::WakeLoop => PostAdmissionSignal::WakeLoop,
494 mm_dsl::PostAdmissionSignalKind::InterruptYielding => {
495 PostAdmissionSignal::InterruptYielding
496 }
497 mm_dsl::PostAdmissionSignalKind::RequestImmediateProcessing => {
498 PostAdmissionSignal::RequestImmediateProcessing
499 }
500 };
501 if new_signal > self.post_admission_signal {
502 self.post_admission_signal = new_signal;
503 }
504 }
505 }
506 }
507
508 pub(crate) fn absorb_post_admission_effects(
509 &mut self,
510 effects: &[mm_dsl::MeerkatMachineEffect],
511 ) {
512 self.absorb_dsl_effects(effects);
513 }
514
515 fn dsl_key(input_id: &InputId) -> String {
516 input_id.to_string()
517 }
518
519 fn with_dsl_state<R>(&self, body: impl FnOnce(&mm_dsl::MeerkatMachineState) -> R) -> R {
520 let authority = self.dsl.lock();
521 body(authority.state())
522 }
523
524 fn dsl_queue_lane(&self) -> Vec<InputId> {
527 self.lane_in_admission_order(mm_dsl::InputLane::Queue)
528 }
529
530 fn dsl_steer_lane(&self) -> Vec<InputId> {
531 self.lane_in_admission_order(mm_dsl::InputLane::Steer)
532 }
533
534 fn lane_in_admission_order(&self, lane: mm_dsl::InputLane) -> Vec<InputId> {
535 let mut candidates: Vec<(u64, InputId)> = self.with_dsl_state(|state| {
536 self.admission_order
537 .iter()
538 .filter(|id| state.input_lane.get(&Self::dsl_key(id)).copied() == Some(lane))
539 .cloned()
540 .map(|id| {
541 let seq = state
542 .input_admission_seq
543 .get(&Self::dsl_key(&id))
544 .copied()
545 .unwrap_or(u64::MAX);
546 (seq, id)
547 })
548 .collect()
549 });
550 candidates.sort_by_key(|(seq, _)| *seq);
551 candidates.into_iter().map(|(_, id)| id).collect()
552 }
553
554 pub fn input_phase(&self, input_id: &InputId) -> Option<InputLifecycleState> {
560 let key = Self::dsl_key(input_id);
561 let phase = self.with_dsl_state(|state| state.input_phases.get(&key).copied())?;
562 Some(Self::input_phase_to_lifecycle(phase))
563 }
564
565 fn input_phase_required(
566 &self,
567 input_id: &InputId,
568 context: &str,
569 ) -> Result<InputLifecycleState, RuntimeDriverError> {
570 self.input_phase(input_id).ok_or_else(|| {
571 RuntimeDriverError::Internal(format!(
572 "generated input lifecycle phase missing {context} for input {input_id}"
573 ))
574 })
575 }
576
577 fn input_phase_to_lifecycle(phase: mm_dsl::InputPhase) -> InputLifecycleState {
582 match phase {
583 mm_dsl::InputPhase::Queued => InputLifecycleState::Queued,
584 mm_dsl::InputPhase::Staged => InputLifecycleState::Staged,
585 mm_dsl::InputPhase::Applied => InputLifecycleState::Applied,
586 mm_dsl::InputPhase::AppliedPendingConsumption => {
587 InputLifecycleState::AppliedPendingConsumption
588 }
589 mm_dsl::InputPhase::Consumed => InputLifecycleState::Consumed,
590 mm_dsl::InputPhase::Superseded => InputLifecycleState::Superseded,
591 mm_dsl::InputPhase::Coalesced => InputLifecycleState::Coalesced,
592 mm_dsl::InputPhase::Abandoned => InputLifecycleState::Abandoned,
593 }
594 }
595
596 pub fn input_last_run_id(&self, input_id: &InputId) -> Option<RunId> {
598 let key = Self::dsl_key(input_id);
599 let raw = self.with_dsl_state(|state| state.input_run_associations.get(&key).cloned())?;
600 raw.0.parse::<uuid::Uuid>().ok().map(RunId::from_uuid)
601 }
602
603 pub fn input_last_boundary_sequence(&self, input_id: &InputId) -> Option<u64> {
605 let key = Self::dsl_key(input_id);
606 self.with_dsl_state(|state| state.input_boundary_sequences.get(&key).copied())
607 }
608
609 pub fn run_boundary_sequence(&self, run_id: &RunId) -> u64 {
617 let key = mm_dsl::RunId::from_domain(run_id);
618 self.with_dsl_state(|state| {
619 state
620 .live_boundary_context_sequence_by_run
621 .get(&key)
622 .copied()
623 .unwrap_or(0)
624 })
625 }
626
627 pub fn input_terminal_outcome(&self, input_id: &InputId) -> Option<InputTerminalOutcome> {
630 let key = Self::dsl_key(input_id);
631 let kind = self.with_dsl_state(|state| state.input_terminal_kind.get(&key).copied())?;
632 match kind {
633 mm_dsl::InputTerminalKind::Consumed => Some(InputTerminalOutcome::Consumed),
634 mm_dsl::InputTerminalKind::Superseded => {
635 let raw =
636 self.with_dsl_state(|state| state.input_superseded_by.get(&key).cloned())?;
637 let id = raw.parse::<uuid::Uuid>().ok().map(InputId::from_uuid)?;
638 Some(InputTerminalOutcome::Superseded { superseded_by: id })
639 }
640 mm_dsl::InputTerminalKind::Coalesced => {
641 let raw =
642 self.with_dsl_state(|state| state.input_aggregate_id.get(&key).cloned())?;
643 let id = raw.parse::<uuid::Uuid>().ok().map(InputId::from_uuid)?;
644 Some(InputTerminalOutcome::Coalesced { aggregate_id: id })
645 }
646 mm_dsl::InputTerminalKind::Abandoned => {
647 let reason = match self
648 .with_dsl_state(|state| state.input_abandon_reason.get(&key).copied())?
649 {
650 mm_dsl::InputAbandonReason::Retired => InputAbandonReason::Retired,
651 mm_dsl::InputAbandonReason::Reset => InputAbandonReason::Reset,
652 mm_dsl::InputAbandonReason::Stopped => InputAbandonReason::Stopped,
653 mm_dsl::InputAbandonReason::Destroyed => InputAbandonReason::Destroyed,
654 mm_dsl::InputAbandonReason::Cancelled => InputAbandonReason::Cancelled,
655 mm_dsl::InputAbandonReason::MaxAttemptsExhausted => {
656 let attempts = self.with_dsl_state(|state| {
657 state
658 .input_abandon_attempt_count
659 .get(&key)
660 .copied()
661 .unwrap_or(0)
662 }) as u32;
663 InputAbandonReason::MaxAttemptsExhausted { attempts }
664 }
665 };
666 Some(InputTerminalOutcome::Abandoned { reason })
667 }
668 }
669 }
670
671 pub(crate) fn input_is_terminal_by_authority(
672 &self,
673 input_id: &InputId,
674 ) -> Result<bool, RuntimeDriverError> {
675 let Some(phase) = self.input_phase(input_id) else {
676 return Err(RuntimeDriverError::Internal(format!(
677 "missing generated input lifecycle authority for '{input_id}'"
678 )));
679 };
680 crate::meerkat_machine::input_phase_behavioral_terminality_via_authority(
681 input_id,
682 phase,
683 self.input_terminal_outcome(input_id),
684 )
685 .map_err(RuntimeDriverError::Internal)
686 }
687
688 fn input_is_non_terminal_by_authority(&self, input_id: &InputId) -> bool {
689 match self.input_is_terminal_by_authority(input_id) {
690 Ok(terminal) => !terminal,
691 Err(err) => {
692 tracing::error!(
693 input_id = %input_id,
694 error = %err,
695 "generated input terminality authority rejected non-terminal filter"
696 );
697 false
698 }
699 }
700 }
701
702 pub fn input_attempt_count(&self, input_id: &InputId) -> u32 {
704 let key = Self::dsl_key(input_id);
705 self.with_dsl_state(|state| state.input_attempt_counts.get(&key).copied().unwrap_or(0))
706 as u32
707 }
708
709 pub fn input_recovery_lane(&self, input_id: &InputId) -> Option<HandlingMode> {
711 let key = Self::dsl_key(input_id);
712 let lane = self.with_dsl_state(|state| state.input_recovery_lanes.get(&key).copied())?;
713 Some(Self::handling_mode_from_admission_lane(lane))
714 }
715
716 pub fn admission_order(&self) -> Vec<InputId> {
720 let mut candidates: Vec<(u64, usize, InputId)> = self.with_dsl_state(|state| {
721 self.admission_order
722 .iter()
723 .enumerate()
724 .map(|(index, id)| {
725 let seq = state
726 .input_admission_seq
727 .get(&Self::dsl_key(id))
728 .copied()
729 .unwrap_or(u64::MAX);
730 (seq, index, id.clone())
731 })
732 .collect()
733 });
734 candidates.sort_by_key(|(seq, index, _)| (*seq, *index));
735 candidates.into_iter().map(|(_, _, id)| id).collect()
736 }
737
738 pub fn admitted_policy(&self, input_id: &InputId) -> Option<&PolicyDecision> {
740 self.policy_snapshot.get(input_id)
741 }
742
743 pub fn admitted_content_shape(&self, input_id: &InputId) -> Option<ContentShape> {
745 self.content_shape.get(input_id).copied()
746 }
747
748 pub fn admitted_request_id(&self, input_id: &InputId) -> Option<RequestId> {
750 self.request_id.get(input_id).cloned().flatten()
751 }
752
753 pub fn admitted_reservation_key(&self, input_id: &InputId) -> Option<ReservationKey> {
755 self.reservation_key.get(input_id).cloned().flatten()
756 }
757
758 pub fn admitted_handling_mode(&self, input_id: &InputId) -> Option<HandlingMode> {
760 self.handling_mode.get(input_id).copied()
761 }
762
763 pub fn admitted_runtime_semantics(&self, input_id: &InputId) -> Option<RuntimeInputSemantics> {
765 self.runtime_semantics.get(input_id).copied()
766 }
767 pub fn input_admission_sequence(&self, input_id: &InputId) -> Option<u64> {
768 let key = Self::dsl_key(input_id);
769 self.with_dsl_state(|state| state.input_admission_seq.get(&key).copied())
770 }
771
772 #[cfg(test)]
773 pub(crate) fn input_runtime_boundary(
774 &self,
775 input_id: &InputId,
776 ) -> Option<mm_dsl::RecoveredRunApplyBoundary> {
777 let key = Self::dsl_key(input_id);
778 self.with_dsl_state(|state| state.input_runtime_boundary.get(&key).copied())
779 }
780
781 #[cfg(test)]
782 pub(crate) fn input_runtime_execution_kind(
783 &self,
784 input_id: &InputId,
785 ) -> Option<mm_dsl::RecoveredRuntimeExecutionKind> {
786 let key = Self::dsl_key(input_id);
787 self.with_dsl_state(|state| state.input_runtime_execution_kind.get(&key).copied())
788 }
789
790 #[cfg(test)]
791 pub(crate) fn input_peer_response_terminal_apply_intent(
792 &self,
793 input_id: &InputId,
794 ) -> Option<mm_dsl::RecoveredPeerResponseTerminalApplyIntent> {
795 let key = Self::dsl_key(input_id);
796 self.with_dsl_state(|state| {
797 state
798 .input_runtime_peer_response_terminal_apply_intent
799 .get(&key)
800 .copied()
801 })
802 }
803
804 #[cfg(test)]
805 pub(crate) fn input_is_prompt_for_batch(&self, input_id: &InputId) -> Option<bool> {
806 let key = Self::dsl_key(input_id);
807 self.with_dsl_state(|state| state.input_is_prompt.get(&key).copied())
808 }
809
810 pub fn admitted_primitive_projection(
812 &self,
813 input_id: &InputId,
814 ) -> Option<RuntimeInputProjection> {
815 self.primitive_projection.get(input_id).cloned()
816 }
817
818 pub fn admitted_is_prompt(&self, input_id: &InputId) -> bool {
820 self.is_prompt_set.contains(input_id)
821 }
822
823 pub fn queue_lane(&self) -> Vec<InputId> {
825 self.dsl_queue_lane()
826 }
827
828 pub fn steer_lane(&self) -> Vec<InputId> {
830 self.dsl_steer_lane()
831 }
832
833 pub fn ingress_lifecycle(&self, input_id: &InputId) -> Option<InputLifecycleState> {
835 self.input_phase(input_id)
836 }
837
838 pub(crate) fn control_handle(&self) -> Arc<StdRwLock<RuntimeControlProjection>> {
839 self.control.clone()
840 }
841
842 fn control_snapshot(&self) -> RuntimeControlProjection {
843 self.read_control_projection().clone()
844 }
845
846 pub fn silent_comms_intents(&self) -> Vec<String> {
847 self.with_dsl_state(|state| state.silent_intent_overrides.iter().cloned().collect())
848 }
849
850 fn matches_silent_intent_authority(&self, input: &Input) -> bool {
851 self.with_dsl_state(|state| {
852 let intents = state
853 .silent_intent_overrides
854 .iter()
855 .cloned()
856 .collect::<Vec<_>>();
857 crate::silent_intent::matches_silent_intent(input, &intents)
858 })
859 }
860
861 fn build_projection_queue(&self, ids: &[InputId], lane: &str) -> InputQueue {
862 let mut queue = InputQueue::new();
863 for input_id in ids {
864 match self
865 .ledger
866 .get(input_id)
867 .and_then(|state| state.persisted_input.clone())
868 {
869 Some(input) => queue.enqueue(input_id.clone(), input),
870 None => {
871 tracing::error!(
872 input_id = ?input_id,
873 lane,
874 "ingress queue references input without persisted payload"
875 );
876 debug_assert!(
877 false,
878 "ingress queue projection missing persisted payload for {input_id:?} in {lane}"
879 );
880 }
881 }
882 }
883 queue
884 }
885
886 fn rebuild_queue_projections(&mut self) {
887 let queue_ids = self.dsl_queue_lane();
888 let steer_ids = self.dsl_steer_lane();
889 self.queue = self.build_projection_queue(&queue_ids, "queue");
890 self.steer_queue = self.build_projection_queue(&steer_ids, "steer_queue");
891 }
892
893 pub(crate) fn rebuild_queue_projections_after_recovery(&mut self) {
894 self.rebuild_queue_projections();
895 self.debug_assert_queue_projection_alignment();
896 }
897
898 pub(crate) fn validate_queue_projection_alignment(
899 &self,
900 context: &str,
901 ) -> Result<(), RuntimeDriverError> {
902 let physical_queue = self.queue.input_ids();
903 let machine_queue = self.dsl_queue_lane();
904 if physical_queue != machine_queue {
905 return Err(RuntimeDriverError::Internal(format!(
906 "{context}: physical queue projection diverged from machine queue lane: physical={physical_queue:?} machine={machine_queue:?}"
907 )));
908 }
909
910 let physical_steer_queue = self.steer_queue.input_ids();
911 let machine_steer_queue = self.dsl_steer_lane();
912 if physical_steer_queue != machine_steer_queue {
913 return Err(RuntimeDriverError::Internal(format!(
914 "{context}: physical steer queue projection diverged from machine steer lane: physical={physical_steer_queue:?} machine={machine_steer_queue:?}"
915 )));
916 }
917
918 Ok(())
919 }
920
921 fn debug_assert_queue_projection_alignment(&self) {
922 debug_assert_eq!(
923 self.queue.input_ids(),
924 self.dsl_queue_lane().as_slice(),
925 "physical queue must match DSL queue lane"
926 );
927 debug_assert_eq!(
928 self.steer_queue.input_ids(),
929 self.dsl_steer_lane().as_slice(),
930 "physical steer queue must match DSL steer lane"
931 );
932 }
933
934 #[allow(clippy::too_many_arguments)]
942 pub(crate) fn admit_recovered_to_ingress(
943 &mut self,
944 work_id: InputId,
945 runtime_semantics: RuntimeInputSemantics,
946 recovered_state: &InputState,
947 recovered_seed: &InputStateSeed,
948 request_id: Option<RequestId>,
949 reservation_key: Option<ReservationKey>,
950 admission_sequence_recovery: Option<mm_dsl::RecoveredInputNormalizationReasonKind>,
951 ) -> Result<(), RuntimeDriverError> {
952 let persisted_input = recovered_state.persisted_input.as_ref().ok_or_else(|| {
953 RuntimeDriverError::Internal(format!(
954 "store corruption: recovered input '{work_id}' has no persisted input; cannot validate recovered admission witness"
955 ))
956 })?;
957 let input_kind = persisted_input.kind();
958 let handling_mode = recovered_seed.recovery_lane.ok_or_else(|| {
959 RuntimeDriverError::Internal(format!(
960 "store corruption: recovered input '{work_id}' missing generated recovery lane witness"
961 ))
962 })?;
963 let content_shape = ContentShape::from_kind(input_kind);
964 let primitive_projection = crate::input::runtime_input_projection(persisted_input);
965 let is_prompt = matches!(persisted_input, Input::Prompt(_));
966
967 self.apply_recovered_admission_witness(
968 &work_id,
969 input_kind,
970 handling_mode,
971 runtime_semantics,
972 )?;
973 let runtime_grouping = matches!(
974 recovered_seed.phase,
975 InputLifecycleState::Accepted | InputLifecycleState::Queued
976 )
977 .then_some((runtime_semantics, is_prompt));
978 self.apply_recovered_lifecycle(
979 &work_id,
980 recovered_seed,
981 admission_sequence_recovery,
982 runtime_grouping,
983 )?;
984 self.register_accepted_idempotency(&work_id, recovered_state.idempotency_key.as_ref())?;
985 self.record_admission_metadata(
986 &work_id,
987 &content_shape,
988 handling_mode,
989 runtime_semantics,
990 primitive_projection,
991 is_prompt,
992 None,
993 request_id.as_ref(),
994 reservation_key.as_ref(),
995 );
996 Ok(())
997 }
998
999 fn apply_recovered_admission_witness(
1000 &mut self,
1001 work_id: &InputId,
1002 input_kind: crate::identifiers::InputKind,
1003 handling_mode: HandlingMode,
1004 runtime_semantics: RuntimeInputSemantics,
1005 ) -> Result<(), RuntimeDriverError> {
1006 let terminal_apply_intent = runtime_semantics
1007 .peer_response_terminal_apply_intent
1008 .map(mm_dsl::RecoveredPeerResponseTerminalApplyIntent::from);
1009 let runtime_boundary =
1010 mm_dsl::RecoveredRunApplyBoundary::try_from(runtime_semantics.boundary).map_err(
1011 |err| {
1012 RuntimeDriverError::Internal(format!(
1013 "store corruption: recovered input '{work_id}' has unsupported runtime boundary: {err}"
1014 ))
1015 },
1016 )?;
1017
1018 self.dsl_apply(
1019 mm_dsl::MeerkatMachineInput::RecoverAdmittedInput {
1020 input_id: Self::dsl_key(work_id),
1021 input_kind: mm_dsl::RecoveredInputKind::from(input_kind),
1022 runtime_boundary,
1023 runtime_execution_kind: mm_dsl::RecoveredRuntimeExecutionKind::from(
1024 runtime_semantics.execution_kind,
1025 ),
1026 runtime_peer_response_terminal_apply_intent: terminal_apply_intent,
1027 lane: mm_dsl::InputLane::from(handling_mode),
1028 },
1029 "RecoverAdmittedInput",
1030 )
1031 .map_err(|err| {
1032 RuntimeDriverError::Internal(format!(
1033 "store corruption: recovered input '{work_id}' rejected by generated recovered-admission authority: {err}"
1034 ))
1035 })
1036 }
1037
1038 pub(crate) fn recover_terminal_input_lifecycle(
1039 &mut self,
1040 work_id: &InputId,
1041 recovered_seed: &InputStateSeed,
1042 idempotency_key: Option<&IdempotencyKey>,
1043 ) -> Result<(), RuntimeDriverError> {
1044 let terminal = crate::meerkat_machine::input_seed_behavioral_terminality_via_authority(
1045 work_id,
1046 recovered_seed,
1047 )
1048 .map_err(RuntimeDriverError::Internal)?;
1049 if !terminal {
1050 return Err(RuntimeDriverError::Internal(format!(
1051 "terminal recovery path received non-terminal input '{work_id}'"
1052 )));
1053 }
1054 self.apply_recovered_lifecycle(work_id, recovered_seed, None, None)?;
1055 self.register_accepted_idempotency(work_id, idempotency_key)
1056 }
1057
1058 fn lifecycle_to_input_phase(lifecycle: InputLifecycleState) -> mm_dsl::InputPhase {
1059 match lifecycle {
1060 InputLifecycleState::Accepted | InputLifecycleState::Queued => {
1064 mm_dsl::InputPhase::Queued
1065 }
1066 InputLifecycleState::Staged => mm_dsl::InputPhase::Staged,
1067 InputLifecycleState::Applied => mm_dsl::InputPhase::Applied,
1068 InputLifecycleState::AppliedPendingConsumption => {
1069 mm_dsl::InputPhase::AppliedPendingConsumption
1070 }
1071 InputLifecycleState::Consumed => mm_dsl::InputPhase::Consumed,
1072 InputLifecycleState::Superseded => mm_dsl::InputPhase::Superseded,
1073 InputLifecycleState::Coalesced => mm_dsl::InputPhase::Coalesced,
1074 InputLifecycleState::Abandoned => mm_dsl::InputPhase::Abandoned,
1075 }
1076 }
1077
1078 fn apply_recovered_lifecycle(
1079 &mut self,
1080 work_id: &InputId,
1081 recovered_seed: &InputStateSeed,
1082 admission_sequence_recovery: Option<mm_dsl::RecoveredInputNormalizationReasonKind>,
1083 runtime_grouping: Option<(RuntimeInputSemantics, bool)>,
1084 ) -> Result<(), RuntimeDriverError> {
1085 let key = Self::dsl_key(work_id);
1086 let lifecycle_state = recovered_seed.phase;
1087 crate::meerkat_machine::input_seed_behavioral_terminality_via_authority(
1088 work_id,
1089 recovered_seed,
1090 )
1091 .map_err(RuntimeDriverError::Internal)?;
1092 let (terminal_kind, superseded_by, aggregate_id, abandon_reason, abandon_attempt_count) =
1093 match recovered_seed.terminal_outcome.clone() {
1094 Some(InputTerminalOutcome::Consumed) => (
1095 Some(mm_dsl::InputTerminalKind::Consumed),
1096 None,
1097 None,
1098 None,
1099 0,
1100 ),
1101 Some(InputTerminalOutcome::Superseded { superseded_by }) => (
1102 Some(mm_dsl::InputTerminalKind::Superseded),
1103 Some(superseded_by.to_string()),
1104 None,
1105 None,
1106 0,
1107 ),
1108 Some(InputTerminalOutcome::Coalesced { aggregate_id }) => (
1109 Some(mm_dsl::InputTerminalKind::Coalesced),
1110 None,
1111 Some(aggregate_id.to_string()),
1112 None,
1113 0,
1114 ),
1115 Some(InputTerminalOutcome::Abandoned { reason }) => {
1116 let abandon_attempt_count = match &reason {
1117 InputAbandonReason::MaxAttemptsExhausted { attempts } => {
1118 u64::from(*attempts)
1119 }
1120 _ => u64::from(recovered_seed.attempt_count),
1121 };
1122 (
1123 Some(mm_dsl::InputTerminalKind::Abandoned),
1124 None,
1125 None,
1126 Some(mm_dsl::InputAbandonReason::from(&reason)),
1127 abandon_attempt_count,
1128 )
1129 }
1130 None => (None, None, None, None, 0),
1131 };
1132 let recovery_lane = recovered_seed.recovery_lane.map(mm_dsl::InputLane::from);
1133 let lane = matches!(lifecycle_state, InputLifecycleState::Queued)
1134 .then_some(recovery_lane)
1135 .flatten();
1136 let (
1137 runtime_boundary,
1138 runtime_execution_kind,
1139 runtime_peer_response_terminal_apply_intent,
1140 is_prompt,
1141 ) = if let Some((runtime_semantics, is_prompt)) = runtime_grouping {
1142 let runtime_boundary = mm_dsl::RecoveredRunApplyBoundary::try_from(
1143 runtime_semantics.boundary,
1144 )
1145 .map_err(|err| {
1146 RuntimeDriverError::Internal(format!(
1147 "input '{work_id}' has unsupported runtime boundary for recovered grouping: {err}"
1148 ))
1149 })?;
1150 (
1151 Some(runtime_boundary),
1152 Some(mm_dsl::RecoveredRuntimeExecutionKind::from(
1153 runtime_semantics.execution_kind,
1154 )),
1155 runtime_semantics
1156 .peer_response_terminal_apply_intent
1157 .map(mm_dsl::RecoveredPeerResponseTerminalApplyIntent::from),
1158 is_prompt,
1159 )
1160 } else {
1161 (None, None, None, false)
1162 };
1163 self.dsl_apply(
1164 mm_dsl::MeerkatMachineInput::RecoverInputLifecycle {
1165 input_id: key,
1166 phase: Self::lifecycle_to_input_phase(lifecycle_state),
1167 terminal_kind,
1168 superseded_by,
1169 aggregate_id,
1170 abandon_reason,
1171 abandon_attempt_count,
1172 attempt_count: u64::from(recovered_seed.attempt_count),
1173 run_id: recovered_seed
1174 .last_run_id
1175 .as_ref()
1176 .map(mm_dsl::RunId::from_domain),
1177 boundary_sequence: recovered_seed.last_boundary_sequence,
1178 admission_sequence: recovered_seed.admission_sequence,
1179 admission_sequence_recovery,
1180 recovery_lane,
1181 lane,
1182 runtime_boundary,
1183 runtime_execution_kind,
1184 runtime_peer_response_terminal_apply_intent,
1185 is_prompt,
1186 },
1187 "RecoverInputLifecycle",
1188 )
1189 }
1190
1191 #[allow(clippy::too_many_arguments)]
1192 fn record_admission_metadata(
1193 &mut self,
1194 work_id: &InputId,
1195 content_shape: &ContentShape,
1196 handling_mode: HandlingMode,
1197 runtime_semantics: RuntimeInputSemantics,
1198 primitive_projection: RuntimeInputProjection,
1199 is_prompt: bool,
1200 policy: Option<&PolicyDecision>,
1201 request_id: Option<&RequestId>,
1202 reservation_key: Option<&ReservationKey>,
1203 ) {
1204 if !self.admission_order.contains(work_id) {
1205 self.admission_order.push(work_id.clone());
1206 }
1207 self.content_shape.insert(work_id.clone(), *content_shape);
1208 self.handling_mode.insert(work_id.clone(), handling_mode);
1209 self.runtime_semantics
1210 .insert(work_id.clone(), runtime_semantics);
1211 if let Some(state) = self.ledger.get_mut(work_id) {
1212 state.runtime_semantics = Some(runtime_semantics);
1213 }
1214 self.primitive_projection
1215 .insert(work_id.clone(), primitive_projection);
1216 if is_prompt {
1217 self.is_prompt_set.insert(work_id.clone());
1218 } else {
1219 self.is_prompt_set.remove(work_id);
1220 }
1221 self.request_id.insert(work_id.clone(), request_id.cloned());
1222 self.reservation_key
1223 .insert(work_id.clone(), reservation_key.cloned());
1224 if let Some(policy) = policy {
1225 self.policy_snapshot.insert(work_id.clone(), policy.clone());
1226 } else {
1227 self.policy_snapshot.remove(work_id);
1228 }
1229 }
1230
1231 fn sync_terminal_projection_from_machine(
1232 &mut self,
1233 input_id: &InputId,
1234 from_phase: InputLifecycleState,
1235 expected_phase: InputLifecycleState,
1236 reason: &'static str,
1237 ) -> Result<(), RuntimeDriverError> {
1238 let phase = self.input_phase(input_id).ok_or_else(|| {
1239 RuntimeDriverError::Internal(format!(
1240 "machine terminal projection missing input phase for {input_id}"
1241 ))
1242 })?;
1243 if phase != expected_phase {
1244 return Err(RuntimeDriverError::Internal(format!(
1245 "machine terminal projection for {input_id} was {phase:?}, expected {expected_phase:?}"
1246 )));
1247 }
1248
1249 let terminal_outcome = self.input_terminal_outcome(input_id).ok_or_else(|| {
1250 RuntimeDriverError::Internal(format!(
1251 "machine terminal projection missing terminal outcome for {input_id}"
1252 ))
1253 })?;
1254 let terminal_matches_phase = matches!(
1255 (&phase, &terminal_outcome),
1256 (
1257 InputLifecycleState::Superseded,
1258 InputTerminalOutcome::Superseded { .. }
1259 ) | (
1260 InputLifecycleState::Coalesced,
1261 InputTerminalOutcome::Coalesced { .. }
1262 ) | (
1263 InputLifecycleState::Consumed,
1264 InputTerminalOutcome::Consumed
1265 ) | (
1266 InputLifecycleState::Abandoned,
1267 InputTerminalOutcome::Abandoned { .. }
1268 )
1269 );
1270 if !terminal_matches_phase {
1271 return Err(RuntimeDriverError::Internal(format!(
1272 "machine terminal projection for {input_id} had incoherent outcome {terminal_outcome:?}"
1273 )));
1274 }
1275
1276 let Some(state) = self.ledger.get_mut(input_id) else {
1277 return Err(RuntimeDriverError::Internal(format!(
1278 "machine terminal projection missing ledger row for {input_id}"
1279 )));
1280 };
1281 let now = Utc::now();
1282 state.history.push(InputStateHistoryEntry {
1283 timestamp: now,
1284 from: from_phase,
1285 to: phase,
1286 reason: Some(reason.into()),
1287 });
1288 state.updated_at = now;
1289 Ok(())
1290 }
1291
1292 #[allow(clippy::too_many_arguments)]
1299 fn apply_persist_and_queue(
1300 &mut self,
1301 input_id: &InputId,
1302 input: &Input,
1303 content_shape: &ContentShape,
1304 handling_mode: HandlingMode,
1305 runtime_semantics: RuntimeInputSemantics,
1306 primitive_projection: RuntimeInputProjection,
1307 is_prompt: bool,
1308 policy: &PolicyDecision,
1309 mut state: InputState,
1310 queue_action: AdmissionQueueAction,
1311 existing_action: Option<&ExistingQueuedAdmissionAction>,
1312 ) -> Result<(), RuntimeDriverError> {
1313 let admission_lane = mm_dsl::InputLane::from(handling_mode);
1317 let admission_key = Self::dsl_key(input_id);
1318 let (admission_input, admission_label) = match admission_lane {
1319 mm_dsl::InputLane::Queue => (
1320 mm_dsl::MeerkatMachineInput::QueueAccepted {
1321 input_id: admission_key,
1322 },
1323 "QueueAccepted",
1324 ),
1325 mm_dsl::InputLane::Steer => (
1326 mm_dsl::MeerkatMachineInput::SteerAccepted {
1327 input_id: admission_key,
1328 },
1329 "SteerAccepted",
1330 ),
1331 };
1332 self.dsl_apply(admission_input, admission_label)?;
1333
1334 if let Some(action) = existing_action {
1340 match action {
1341 ExistingQueuedAdmissionAction::Coalesce { existing_id } => {
1342 let existing_key = Self::dsl_key(existing_id);
1343 let aggregate_key = Self::dsl_key(input_id);
1344 let from_phase = self.input_phase_required(existing_id, "before coalescing")?;
1345 self.dsl_apply(
1346 mm_dsl::MeerkatMachineInput::CoalesceInput {
1347 input_id: existing_key,
1348 aggregate_id: aggregate_key,
1349 },
1350 "CoalesceInput",
1351 )?;
1352 self.sync_terminal_projection_from_machine(
1353 existing_id,
1354 from_phase,
1355 InputLifecycleState::Coalesced,
1356 "Coalesce",
1357 )?;
1358 let _ = self.queue.remove(existing_id);
1359 let _ = self.steer_queue.remove(existing_id);
1360 }
1361 ExistingQueuedAdmissionAction::Supersede { existing_id } => {
1362 let existing_key = Self::dsl_key(existing_id);
1363 let superseded_by = Self::dsl_key(input_id);
1364 let from_phase =
1365 self.input_phase_required(existing_id, "before superseding")?;
1366 self.dsl_apply(
1367 mm_dsl::MeerkatMachineInput::SupersedeInput {
1368 input_id: existing_key,
1369 superseded_by,
1370 },
1371 "SupersedeInput",
1372 )?;
1373 self.sync_terminal_projection_from_machine(
1374 existing_id,
1375 from_phase,
1376 InputLifecycleState::Superseded,
1377 "Supersede",
1378 )?;
1379 let _ = self.queue.remove(existing_id);
1380 let _ = self.steer_queue.remove(existing_id);
1381 }
1382 }
1383 }
1384
1385 match queue_action.clone() {
1390 AdmissionQueueAction::None => {}
1391 AdmissionQueueAction::EnqueueTo { target } => {
1392 let target_lane = mm_dsl::InputLane::from(target);
1393 if target_lane != admission_lane {
1394 self.dsl_apply(
1395 mm_dsl::MeerkatMachineInput::ChangeLane {
1396 input_id: Self::dsl_key(input_id),
1397 new_lane: target_lane,
1398 },
1399 "ChangeLane",
1400 )?;
1401 }
1402 }
1403 AdmissionQueueAction::EnqueueFront { target } => {
1404 let key = Self::dsl_key(input_id);
1405 self.dsl_apply(
1406 mm_dsl::MeerkatMachineInput::PrioritizeInput {
1407 input_id: key.clone(),
1408 },
1409 "PrioritizeInput",
1410 )?;
1411 let target_lane = mm_dsl::InputLane::from(target);
1412 if target_lane != admission_lane {
1413 self.dsl_apply(
1414 mm_dsl::MeerkatMachineInput::ChangeLane {
1415 input_id: key,
1416 new_lane: target_lane,
1417 },
1418 "ChangeLane",
1419 )?;
1420 }
1421 }
1422 }
1423
1424 self.register_accepted_idempotency(input_id, input.header().idempotency_key.as_ref())?;
1425
1426 let now = Utc::now();
1427 state.persisted_input = Some(input.clone());
1428 state.policy = Some(PolicySnapshot {
1429 version: policy.policy_version,
1430 decision: policy.clone(),
1431 });
1432 state.history.push(InputStateHistoryEntry {
1433 timestamp: now,
1434 from: InputLifecycleState::Accepted,
1435 to: InputLifecycleState::Queued,
1436 reason: Some("QueueAccepted".into()),
1437 });
1438 state.updated_at = now;
1439 self.ledger.accept(state);
1440 self.record_admission_metadata(
1441 input_id,
1442 content_shape,
1443 handling_mode,
1444 runtime_semantics,
1445 primitive_projection,
1446 is_prompt,
1447 Some(policy),
1448 None,
1449 None,
1450 );
1451
1452 match queue_action {
1453 AdmissionQueueAction::None => {}
1454 AdmissionQueueAction::EnqueueTo { target } => match target {
1455 HandlingMode::Queue => self.queue.enqueue(input_id.clone(), input.clone()),
1456 HandlingMode::Steer => {
1457 self.steer_queue.enqueue(input_id.clone(), input.clone());
1458 }
1459 },
1460 AdmissionQueueAction::EnqueueFront { target } => match target {
1461 HandlingMode::Queue => {
1462 self.queue.enqueue_front(input_id.clone(), input.clone());
1463 }
1464 HandlingMode::Steer => {
1465 self.steer_queue
1466 .enqueue_front(input_id.clone(), input.clone());
1467 }
1468 },
1469 }
1470
1471 self.emit_event(RuntimeEvent::InputLifecycle(
1472 InputLifecycleEvent::Accepted {
1473 input_id: input_id.clone(),
1474 },
1475 ));
1476 self.emit_event(RuntimeEvent::InputLifecycle(InputLifecycleEvent::Queued {
1477 input_id: input_id.clone(),
1478 }));
1479
1480 Ok(())
1481 }
1482
1483 pub fn is_idle(&self) -> bool {
1484 self.runtime_phase_snapshot() == RuntimeState::Idle
1485 }
1486
1487 pub fn phase(&self) -> RuntimeState {
1488 self.runtime_phase_snapshot()
1489 }
1490
1491 fn runtime_phase_snapshot(&self) -> RuntimeState {
1492 let authority = self.shared_dsl_authority();
1493 let authority = authority
1494 .lock()
1495 .unwrap_or_else(std::sync::PoisonError::into_inner);
1496 crate::meerkat_machine::dsl_authority::runtime_phase_from_authority(&authority)
1497 }
1498
1499 pub fn current_run_id(&self) -> Option<RunId> {
1500 let authority = self.shared_dsl_authority();
1501 let authority = authority
1502 .lock()
1503 .unwrap_or_else(std::sync::PoisonError::into_inner);
1504 crate::meerkat_machine::dsl_authority::current_run_id_from_authority(&authority)
1505 }
1506
1507 pub fn pre_run_phase(&self) -> Option<RuntimeState> {
1508 let authority = self.shared_dsl_authority();
1509 let authority = authority
1510 .lock()
1511 .unwrap_or_else(std::sync::PoisonError::into_inner);
1512 crate::meerkat_machine::dsl_authority::pre_run_phase_from_authority(&authority)
1513 }
1514
1515 fn contract_session_authority_id(&self) -> mm_dsl::SessionId {
1516 mm_dsl::SessionId::from(self.runtime_id.to_string())
1517 }
1518
1519 pub(crate) fn ensure_contract_session_authority(
1520 &mut self,
1521 ) -> Result<mm_dsl::SessionId, RuntimeDriverError> {
1522 let existing = {
1523 let authority = self.shared_dsl_authority();
1524 let authority = authority
1525 .lock()
1526 .unwrap_or_else(std::sync::PoisonError::into_inner);
1527 authority.state().session_id.clone()
1528 };
1529 if let Some(session_id) = existing {
1530 return Ok(session_id);
1531 }
1532
1533 let session_id = self.contract_session_authority_id();
1534 self.dsl_apply(
1535 mm_dsl::MeerkatMachineInput::RegisterSession {
1536 session_id: session_id.clone(),
1537 },
1538 "ContractRegisterSession",
1539 )?;
1540 self.sync_control_projection_from_dsl_authority();
1541 Ok(session_id)
1542 }
1543
1544 #[doc(hidden)]
1547 pub fn contract_begin_run_authority(
1548 &mut self,
1549 run_id: RunId,
1550 ) -> Result<(), RuntimeDriverError> {
1551 let from = self.runtime_phase_snapshot();
1552 if from == RuntimeState::Running && self.current_run_id().as_ref() == Some(&run_id) {
1553 return Ok(());
1554 }
1555
1556 let session_id = self.ensure_contract_session_authority()?;
1557 if from == RuntimeState::Retired {
1558 let authority = self.shared_dsl_authority();
1559 let mut authority = authority
1560 .lock()
1561 .unwrap_or_else(std::sync::PoisonError::into_inner);
1562 authority
1563 .apply_signal(mm_dsl::MeerkatMachineSignal::DrainQueuedRun {
1564 run_id: mm_dsl::RunId::from_domain(&run_id),
1565 })
1566 .map(|_| ())
1567 .map_err(|err| {
1568 RuntimeDriverError::Internal(crate::meerkat_machine::dsl_authority::map_error(
1569 err,
1570 "ContractDrainQueuedRun",
1571 ))
1572 })?;
1573 } else {
1574 self.dsl_apply(
1575 mm_dsl::MeerkatMachineInput::Prepare {
1576 session_id,
1577 run_id: mm_dsl::RunId::from_domain(&run_id),
1578 },
1579 "ContractPrepareRun",
1580 )?;
1581 }
1582 self.sync_control_projection_from_dsl_authority();
1583 Ok(())
1584 }
1585
1586 fn set_phase(&mut self, next_phase: RuntimeState) -> RuntimeState {
1587 let mut control = self.write_control_projection();
1588 let from_phase = control.phase;
1589 control.phase = next_phase;
1590 from_phase
1591 }
1592
1593 fn transition_phase(&mut self, next_phase: RuntimeState) {
1594 let from_phase = self.set_phase(next_phase);
1595 self.emit_event(RuntimeEvent::RuntimeStateChange(RuntimeStateChangeEvent {
1596 from: from_phase,
1597 to: next_phase,
1598 }));
1599 }
1600
1601 pub(crate) fn set_control_projection(
1602 &mut self,
1603 next_phase: RuntimeState,
1604 current_run_id: Option<RunId>,
1605 pre_run_phase: Option<RuntimeState>,
1606 ) {
1607 if self.control_snapshot().phase == next_phase {
1608 self.write_control_projection().phase = next_phase;
1609 } else {
1610 self.transition_phase(next_phase);
1611 }
1612 let mut control = self.write_control_projection();
1613 control.current_run_id = current_run_id;
1614 control.pre_run_phase = pre_run_phase;
1615 }
1616
1617 pub(crate) fn sync_control_projection_from_dsl_authority(&mut self) {
1618 let (phase, current_run_id, pre_run_phase) = {
1619 let authority = self.shared_dsl_authority();
1620 let authority = authority
1621 .lock()
1622 .unwrap_or_else(std::sync::PoisonError::into_inner);
1623 (
1624 crate::meerkat_machine::dsl_authority::runtime_phase_from_authority(&authority),
1625 crate::meerkat_machine::dsl_authority::current_run_id_from_authority(&authority),
1626 crate::meerkat_machine::dsl_authority::pre_run_phase_from_authority(&authority),
1627 )
1628 };
1629 self.set_control_projection(phase, current_run_id, pre_run_phase);
1630 }
1631
1632 #[cfg(test)]
1636 #[doc(hidden)]
1637 pub(crate) fn contract_force_runtime_authority(
1638 &mut self,
1639 next_phase: RuntimeState,
1640 current_run_id: Option<RunId>,
1641 pre_run_phase: Option<RuntimeState>,
1642 ) {
1643 {
1644 let authority = self.shared_dsl_authority();
1645 let mut authority = authority
1646 .lock()
1647 .unwrap_or_else(std::sync::PoisonError::into_inner);
1648 let session_id = authority
1649 .state()
1650 .session_id
1651 .as_ref()
1652 .and_then(|session_id| {
1653 uuid::Uuid::parse_str(&session_id.0)
1654 .ok()
1655 .map(SessionId::from_uuid)
1656 });
1657 let silent_intent_overrides = authority.state().silent_intent_overrides.clone();
1658 let active_fence_token = authority
1659 .state()
1660 .active_fence_token
1661 .as_ref()
1662 .map(|token| token.0);
1663 let active_runtime_generation = authority.state().active_runtime_generation;
1664 let active_runtime_epoch_id = authority.state().active_runtime_epoch_id.clone();
1665 *authority =
1666 crate::meerkat_machine::dsl_authority::recover_authority_from_runtime_observation(
1667 &session_id.unwrap_or_default(),
1668 next_phase,
1669 Some(&self.runtime_id),
1670 current_run_id.as_ref(),
1671 pre_run_phase,
1672 silent_intent_overrides,
1673 active_fence_token,
1674 active_runtime_generation,
1675 active_runtime_epoch_id,
1676 )
1677 .expect("contract runtime authority observation must recover");
1678 }
1679 self.set_control_projection(next_phase, current_run_id, pre_run_phase);
1680 }
1681
1682 pub(crate) fn apply_runtime_executor_exited_authority(
1683 &mut self,
1684 ) -> Result<(), RuntimeDriverError> {
1685 self.dsl_apply(
1686 mm_dsl::MeerkatMachineInput::RuntimeExecutorExited,
1687 "RuntimeExecutorExited",
1688 )
1689 }
1690
1691 pub fn take_post_admission_signal(&mut self) -> PostAdmissionSignal {
1695 std::mem::replace(&mut self.post_admission_signal, PostAdmissionSignal::None)
1696 }
1697
1698 pub fn post_admission_signal(&self) -> PostAdmissionSignal {
1700 self.post_admission_signal
1701 }
1702
1703 pub fn take_wake_requested(&mut self) -> bool {
1708 self.post_admission_signal.should_wake()
1711 }
1712
1713 pub fn take_process_requested(&mut self) -> bool {
1718 let signal = std::mem::replace(&mut self.post_admission_signal, PostAdmissionSignal::None);
1719 signal.should_process_immediately()
1720 }
1721 pub fn drain_events(&mut self) -> Vec<RuntimeEventEnvelope> {
1722 std::mem::take(&mut self.events)
1723 }
1724 pub fn queue(&self) -> &InputQueue {
1725 &self.queue
1726 }
1727 pub fn steer_queue(&self) -> &InputQueue {
1728 &self.steer_queue
1729 }
1730
1731 #[cfg(test)]
1732 pub fn queue_mut(&mut self) -> &mut InputQueue {
1733 &mut self.queue
1734 }
1735
1736 #[cfg(test)]
1737 pub fn steer_queue_mut(&mut self) -> &mut InputQueue {
1738 &mut self.steer_queue
1739 }
1740
1741 #[cfg(test)]
1742 pub(crate) fn clear_admitted_runtime_semantics_for_test(&mut self, input_id: &InputId) {
1743 self.runtime_semantics.remove(input_id);
1744 if let Some(state) = self.ledger.get_mut(input_id) {
1745 state.runtime_semantics = None;
1746 }
1747 }
1748
1749 pub fn has_queued_input(&self, input_id: &InputId) -> bool {
1750 let key = Self::dsl_key(input_id);
1751 self.with_dsl_state(|state| state.input_lane.contains_key(&key))
1752 }
1753 pub fn has_queued_input_outside(&self, excluded: &[InputId]) -> bool {
1754 let excluded_keys: std::collections::HashSet<String> =
1755 excluded.iter().map(Self::dsl_key).collect();
1756 self.with_dsl_state(|state| {
1757 state
1758 .input_lane
1759 .keys()
1760 .any(|queued_key| !excluded_keys.contains(queued_key))
1761 })
1762 }
1763
1764 pub(crate) fn defer_queued_inputs_behind_backlog(
1765 &mut self,
1766 input_ids: &[InputId],
1767 ) -> Result<(), RuntimeDriverError> {
1768 for input_id in input_ids {
1769 self.dsl_apply(
1770 mm_dsl::MeerkatMachineInput::DeferInputBehindBacklog {
1771 input_id: Self::dsl_key(input_id),
1772 },
1773 "DeferInputBehindBacklog",
1774 )?;
1775 }
1776 self.rebuild_queue_projections();
1777 self.debug_assert_queue_projection_alignment();
1778 Ok(())
1779 }
1780
1781 fn existing_superseded_input(
1782 &self,
1783 input: &Input,
1784 ) -> Option<(InputId, crate::coalescing::CoalescingResult)> {
1785 let candidates: Vec<InputId> = self
1786 .dsl_queue_lane()
1787 .into_iter()
1788 .chain(self.dsl_steer_lane())
1789 .collect();
1790 candidates.into_iter().find_map(|queued_id| {
1791 let existing = self.ledger.get(&queued_id)?.persisted_input.as_ref()?;
1792 let result = crate::coalescing::check_supersession(input, existing, &self.runtime_id);
1793 match result {
1794 crate::coalescing::CoalescingResult::Supersedes { .. } => Some((queued_id, result)),
1795 crate::coalescing::CoalescingResult::Standalone => None,
1796 }
1797 })
1798 }
1799 pub fn ledger(&self) -> &InputLedger {
1800 &self.ledger
1801 }
1802 pub fn runtime_id(&self) -> &LogicalRuntimeId {
1803 &self.runtime_id
1804 }
1805 pub(crate) fn ledger_mut(&mut self) -> &mut InputLedger {
1806 &mut self.ledger
1807 }
1808 pub fn stored_input_state(&self, input_id: &InputId) -> Option<StoredInputState> {
1813 let mut state = self.ledger.get(input_id)?.clone();
1814 if state.runtime_semantics.is_none() {
1815 state.runtime_semantics = self.admitted_runtime_semantics(input_id);
1816 }
1817 let phase = self.input_phase(input_id)?;
1818 let seed = InputStateSeed {
1819 phase,
1820 last_run_id: self.input_last_run_id(input_id),
1821 last_boundary_sequence: self.input_last_boundary_sequence(input_id),
1822 admission_sequence: self.input_admission_sequence(input_id),
1823 terminal_outcome: self.input_terminal_outcome(input_id),
1824 attempt_count: self.input_attempt_count(input_id),
1825 recovery_lane: self.input_recovery_lane(input_id),
1826 };
1827 Some(StoredInputState { state, seed })
1828 }
1829
1830 pub fn input_id_for_idempotency_key(&self, idempotency_key: &str) -> Option<InputId> {
1841 let raw = self.with_dsl_state(|state| {
1842 state
1843 .admission_idempotency_inputs
1844 .get(idempotency_key)
1845 .cloned()
1846 })?;
1847 raw.parse::<uuid::Uuid>().ok().map(InputId::from_uuid)
1848 }
1849
1850 pub fn stored_input_states_snapshot(
1852 &self,
1853 ) -> Result<Vec<StoredInputState>, RuntimeDriverError> {
1854 self.ledger
1855 .iter()
1856 .map(|(input_id, state)| {
1857 let mut state = state.clone();
1858 if state.runtime_semantics.is_none() {
1859 state.runtime_semantics = self.admitted_runtime_semantics(input_id);
1860 }
1861 let phase = self.input_phase(input_id).ok_or_else(|| {
1862 RuntimeDriverError::Internal(format!(
1863 "generated input lifecycle phase missing for persisted input {input_id}"
1864 ))
1865 })?;
1866 let seed = InputStateSeed {
1867 phase,
1868 last_run_id: self.input_last_run_id(input_id),
1869 last_boundary_sequence: self.input_last_boundary_sequence(input_id),
1870 admission_sequence: self.input_admission_sequence(input_id),
1871 terminal_outcome: self.input_terminal_outcome(input_id),
1872 attempt_count: self.input_attempt_count(input_id),
1873 recovery_lane: self.input_recovery_lane(input_id),
1874 };
1875 Ok(StoredInputState { state, seed })
1876 })
1877 .collect()
1878 }
1879
1880 pub fn authorized_stored_input_states_snapshot(
1883 &self,
1884 ) -> Result<Vec<InputStatePersistenceRecord>, RuntimeDriverError> {
1885 self.stored_input_states_snapshot()?
1886 .into_iter()
1887 .map(|bundle| {
1888 InputStatePersistenceRecord::from_machine_snapshot(bundle)
1889 .map_err(RuntimeDriverError::Internal)
1890 })
1891 .collect()
1892 }
1893
1894 pub fn authorized_stored_input_state(
1896 &self,
1897 input_id: &InputId,
1898 ) -> Result<Option<InputStatePersistenceRecord>, RuntimeDriverError> {
1899 self.stored_input_state(input_id)
1900 .map(InputStatePersistenceRecord::from_machine_snapshot)
1901 .transpose()
1902 .map_err(RuntimeDriverError::Internal)
1903 }
1904
1905 pub fn recover_input_state_persistence_record(
1910 &mut self,
1911 mut bundle: StoredInputState,
1912 ) -> Result<InputStatePersistenceRecord, RuntimeDriverError> {
1913 let delta = crate::meerkat_machine::driver::machine_apply_recovered_input_normalization(
1914 &mut bundle,
1915 None,
1916 )?;
1917 let input_id = bundle.state.input_id.clone();
1918 if self.ledger.get(&input_id).is_some() {
1919 return Err(RuntimeDriverError::Internal(format!(
1920 "input-state persistence recovery record requested for duplicate input {input_id}"
1921 )));
1922 }
1923
1924 let terminal = crate::meerkat_machine::input_seed_behavioral_terminality_via_authority(
1925 &input_id,
1926 &bundle.seed,
1927 )
1928 .map_err(RuntimeDriverError::Internal)?;
1929
1930 if terminal {
1931 self.recover_terminal_input_lifecycle(
1932 &input_id,
1933 &bundle.seed,
1934 bundle.state.idempotency_key.as_ref(),
1935 )?;
1936 } else {
1937 let Some(entry) = crate::meerkat_machine::driver::machine_build_recovered_ingress_entry(
1938 &bundle.state,
1939 &bundle.seed,
1940 ) else {
1941 return Err(RuntimeDriverError::Internal(format!(
1942 "input-state persistence recovery record for '{input_id}' missing recovered admission witness"
1943 )));
1944 };
1945 self.admit_recovered_to_ingress(
1946 input_id.clone(),
1947 entry.runtime_semantics,
1948 &bundle.state,
1949 &bundle.seed,
1950 None,
1951 None,
1952 delta.admission_sequence_recovery,
1953 )?;
1954 }
1955
1956 self.ledger.recover(bundle.state);
1957 self.rebuild_queue_projections_after_recovery();
1958 self.authorized_stored_input_state(&input_id)?
1959 .ok_or_else(|| {
1960 RuntimeDriverError::Internal(format!(
1961 "generated input-state persistence recovery emitted no record for {input_id}"
1962 ))
1963 })
1964 }
1965 pub fn clear_queue_projections(&mut self) {
1968 self.queue = InputQueue::new();
1969 self.steer_queue = InputQueue::new();
1970 }
1971 pub(crate) fn dequeue_next(&mut self) -> Option<(InputId, Input)> {
1972 let queued = self
1973 .steer_queue
1974 .dequeue()
1975 .or_else(|| self.queue.dequeue())?;
1976 Some((queued.input_id, queued.input))
1977 }
1978
1979 #[cfg(any(test, debug_assertions, feature = "test-support"))]
1982 #[doc(hidden)]
1983 pub fn contract_dequeue_next_for_recovery_tests(&mut self) -> Option<(InputId, Input)> {
1984 self.dequeue_next()
1985 }
1986
1987 pub(crate) fn dequeue_batch_exact(
1988 &mut self,
1989 batch: &crate::meerkat_machine::driver::AuthorizedRuntimeLoopBatch,
1990 ) -> Result<Vec<(InputId, Input)>, RuntimeDriverError> {
1991 self.validate_queue_projection_alignment("before authorized runtime-loop dequeue")?;
1992 let (source_name, source_queue) = match batch.source() {
1993 crate::meerkat_machine::driver::RuntimeLoopBatchSource::Queue => {
1994 ("queue", &mut self.queue)
1995 }
1996 crate::meerkat_machine::driver::RuntimeLoopBatchSource::Steer => {
1997 ("steer", &mut self.steer_queue)
1998 }
1999 };
2000 source_queue
2001 .dequeue_exact_prefix(batch.input_ids())
2002 .ok_or_else(|| {
2003 RuntimeDriverError::Internal(format!(
2004 "authorized runtime batch from {source_name} did not match the physical queue prefix exactly: expected {:?}",
2005 batch.input_ids()
2006 ))
2007 })
2008 }
2009
2010 fn machine_realize_stage_batch(
2012 &mut self,
2013 input_ids: &[InputId],
2014 run_id: &RunId,
2015 ) -> Result<(), RuntimeDriverError> {
2016 for input_id in input_ids {
2017 let key = Self::dsl_key(input_id);
2018 let from_phase = self.input_phase_required(input_id, "before staging")?;
2021 self.dsl_apply(
2023 mm_dsl::MeerkatMachineInput::StageForRun {
2024 input_id: key,
2025 run_id: mm_dsl::RunId(run_id.to_string()),
2026 },
2027 "StageForRun",
2028 )?;
2029
2030 let now = Utc::now();
2031 if let Some(state) = self.ledger.get_mut(input_id) {
2032 state.history.push(InputStateHistoryEntry {
2033 timestamp: now,
2034 from: from_phase,
2035 to: InputLifecycleState::Staged,
2036 reason: Some(format!("StageForRun({run_id})")),
2037 });
2038 state.updated_at = now;
2039 }
2040 self.emit_event(RuntimeEvent::InputLifecycle(InputLifecycleEvent::Staged {
2041 input_id: input_id.clone(),
2042 run_id: run_id.clone(),
2043 }));
2044 }
2045 self.rebuild_queue_projections();
2046 self.validate_queue_projection_alignment("after authorized StageForRun")?;
2047 self.debug_assert_queue_projection_alignment();
2048
2049 Ok(())
2050 }
2051
2052 pub(crate) fn machine_realize_authorized_stage_batch(
2053 &mut self,
2054 authority: crate::meerkat_machine::driver::AuthorizedStageForRun,
2055 ) -> Result<(), RuntimeDriverError> {
2056 let (input_ids, run_id, _source) = authority.into_parts();
2057 self.machine_realize_stage_batch(&input_ids, &run_id)
2058 }
2059
2060 pub fn apply_input(
2061 &mut self,
2062 input_id: &InputId,
2063 run_id: &RunId,
2064 ) -> Result<(), RuntimeDriverError> {
2065 let key = Self::dsl_key(input_id);
2066 let from_phase = self.input_phase_required(input_id, "before applying")?;
2069 self.dsl_apply(
2070 mm_dsl::MeerkatMachineInput::MarkApplied {
2071 input_id: key.clone(),
2072 },
2073 "MarkApplied",
2074 )?;
2075 self.dsl_apply(
2076 mm_dsl::MeerkatMachineInput::MarkAppliedPendingConsumption { input_id: key },
2077 "MarkAppliedPendingConsumption",
2078 )?;
2079
2080 let now = Utc::now();
2081 if let Some(state) = self.ledger.get_mut(input_id) {
2082 state.history.push(InputStateHistoryEntry {
2083 timestamp: now,
2084 from: from_phase,
2085 to: InputLifecycleState::Applied,
2086 reason: Some(format!("MarkApplied({run_id})")),
2087 });
2088 state.history.push(InputStateHistoryEntry {
2089 timestamp: now,
2090 from: InputLifecycleState::Applied,
2091 to: InputLifecycleState::AppliedPendingConsumption,
2092 reason: Some("MarkAppliedPendingConsumption(boundary_sequence=0)".into()),
2093 });
2094 state.updated_at = now;
2095 }
2096
2097 self.emit_event(RuntimeEvent::InputLifecycle(InputLifecycleEvent::Applied {
2098 input_id: input_id.clone(),
2099 run_id: run_id.clone(),
2100 }));
2101 Ok(())
2102 }
2103
2104 pub(crate) fn consume_inputs(
2105 &mut self,
2106 input_ids: &[InputId],
2107 run_id: &RunId,
2108 ) -> Result<(), RuntimeDriverError> {
2109 for input_id in input_ids {
2110 let phase = self.input_phase(input_id);
2111 if phase != Some(InputLifecycleState::AppliedPendingConsumption) {
2112 continue;
2113 }
2114 let from_phase = phase.ok_or_else(|| {
2115 RuntimeDriverError::Internal(format!(
2116 "generated input lifecycle phase missing before consuming input {input_id}"
2117 ))
2118 })?;
2119
2120 let key = Self::dsl_key(input_id);
2121 self.dsl_apply(
2122 mm_dsl::MeerkatMachineInput::ConsumeInput { input_id: key },
2123 "ConsumeInput",
2124 )?;
2125
2126 self.sync_terminal_projection_from_machine(
2127 input_id,
2128 from_phase,
2129 InputLifecycleState::Consumed,
2130 "Consume",
2131 )?;
2132 self.events
2133 .push(self.make_envelope(RuntimeEvent::InputLifecycle(
2134 InputLifecycleEvent::Consumed {
2135 input_id: input_id.clone(),
2136 run_id: run_id.clone(),
2137 },
2138 )));
2139 }
2140 Ok(())
2141 }
2142
2143 fn machine_resolve_live_boundary_context_receipt(
2144 &mut self,
2145 run_id: &RunId,
2146 input_id: &InputId,
2147 ) -> Result<RunBoundaryReceipt, RuntimeDriverError> {
2148 let input_key = Self::dsl_key(input_id);
2149 let expected_run_id = mm_dsl::RunId::from_domain(run_id);
2150 let effects = self.dsl_apply_effects(
2151 mm_dsl::MeerkatMachineInput::ResolveLiveBoundaryContextReceipt {
2152 run_id: expected_run_id.clone(),
2153 input_id: input_key.clone(),
2154 },
2155 "ResolveLiveBoundaryContextReceipt",
2156 )?;
2157
2158 let Some((effect_run_id, effect_input_id, boundary, boundary_sequence)) =
2159 effects.into_iter().find_map(|effect| match effect {
2160 mm_dsl::MeerkatMachineEffect::LiveBoundaryContextReceiptResolved {
2161 run_id,
2162 input_id,
2163 boundary,
2164 boundary_sequence,
2165 } => Some((run_id, input_id, boundary, boundary_sequence)),
2166 _ => None,
2167 })
2168 else {
2169 return Err(RuntimeDriverError::Internal(format!(
2170 "generated machine emitted no live-boundary receipt for input {input_id}"
2171 )));
2172 };
2173
2174 if effect_run_id != expected_run_id || effect_input_id != input_key {
2175 return Err(RuntimeDriverError::Internal(format!(
2176 "generated machine emitted mismatched live-boundary receipt for input {input_id}"
2177 )));
2178 }
2179
2180 Ok(RunBoundaryReceipt {
2181 run_id: run_id.clone(),
2182 boundary: boundary.into(),
2183 contributing_input_ids: vec![input_id.clone()],
2184 conversation_digest: None,
2185 message_count: 0,
2186 sequence: boundary_sequence,
2187 })
2188 }
2189
2190 pub(crate) fn machine_realize_live_boundary_context_injected(
2191 &mut self,
2192 run_id: &RunId,
2193 input_ids: &[InputId],
2194 stage_authority: crate::meerkat_machine::driver::AuthorizedStageForRun,
2195 ) -> Result<RunBoundaryReceipt, RuntimeDriverError> {
2196 let [input_id] = input_ids else {
2197 return Err(RuntimeDriverError::Internal(format!(
2198 "generated live-boundary receipt authority requires exactly one input, got {}",
2199 input_ids.len()
2200 )));
2201 };
2202 let checkpoint = self.rollback_snapshot();
2203 let result = self
2204 .machine_resolve_live_boundary_context_receipt(run_id, input_id)
2205 .and_then(|receipt| {
2206 self.machine_realize_authorized_stage_batch(stage_authority)
2207 .and_then(|()| self.machine_realize_boundary_applied(run_id, &receipt))
2208 .and_then(|()| self.machine_realize_run_completed(run_id, input_ids))
2209 .map(|()| receipt)
2210 });
2211 match result {
2212 Ok(receipt) => Ok(receipt),
2213 Err(err) => {
2214 self.restore_rollback_snapshot(checkpoint);
2215 Err(err)
2216 }
2217 }
2218 }
2219
2220 pub fn rollback_staged(&mut self, input_ids: &[InputId]) -> Result<(), RuntimeDriverError> {
2221 for input_id in input_ids {
2222 if self.input_phase(input_id) != Some(InputLifecycleState::Staged) {
2224 continue;
2225 }
2226 let Some(_state) = self.ledger.get(input_id) else {
2227 continue;
2228 };
2229
2230 let lane = self.input_recovery_lane(input_id).ok_or_else(|| {
2231 RuntimeDriverError::Internal(format!(
2232 "generated recovery lane missing for rollback of staged input '{input_id}'"
2233 ))
2234 })?;
2235 self.dsl_apply(
2236 mm_dsl::MeerkatMachineInput::ResolveStagedRollback {
2237 input_id: Self::dsl_key(input_id),
2238 lane: mm_dsl::InputLane::from(lane),
2239 },
2240 "ResolveStagedRollback",
2241 )?;
2242
2243 match self.input_phase_required(input_id, "after staged rollback resolution")? {
2244 InputLifecycleState::Queued => {
2245 let now = Utc::now();
2246 if let Some(state) = self.ledger.get_mut(input_id) {
2247 state.history.push(InputStateHistoryEntry {
2248 timestamp: now,
2249 from: InputLifecycleState::Staged,
2250 to: InputLifecycleState::Queued,
2251 reason: Some("ResolveStagedRollback".into()),
2252 });
2253 state.updated_at = now;
2254 }
2255 }
2256 InputLifecycleState::Abandoned => {
2257 let attempts = self.input_attempt_count(input_id);
2258 tracing::warn!(
2259 input_id = %input_id,
2260 attempts,
2261 "input abandoned after generated max stage attempts decision"
2262 );
2263 self.sync_terminal_projection_from_machine(
2264 input_id,
2265 InputLifecycleState::Staged,
2266 InputLifecycleState::Abandoned,
2267 "ResolveStagedRollback->Abandon",
2268 )?;
2269 self.events
2270 .push(self.make_envelope(RuntimeEvent::InputLifecycle(
2271 InputLifecycleEvent::Abandoned {
2272 input_id: input_id.clone(),
2273 reason: InputAbandonReason::MaxAttemptsExhausted { attempts },
2274 },
2275 )));
2276 }
2277 other => {
2278 return Err(RuntimeDriverError::Internal(format!(
2279 "generated staged rollback resolution for input {input_id} produced unexpected phase {other:?}"
2280 )));
2281 }
2282 }
2283 }
2284
2285 self.rebuild_queue_projections();
2286 self.debug_assert_queue_projection_alignment();
2287 Ok(())
2288 }
2289
2290 pub(crate) fn finalize_retire(&mut self) -> RetireReport {
2291 let inputs_pending_drain = self
2292 .ledger
2293 .iter()
2294 .filter(|(id, _)| self.input_is_non_terminal_by_authority(id))
2295 .count();
2296 RetireReport {
2297 inputs_abandoned: 0,
2298 inputs_pending_drain,
2299 }
2300 }
2301
2302 pub(crate) fn reset_cleanup(&mut self) -> Result<ResetReport, RuntimeDriverError> {
2303 let abandoned = self.abandon_all_non_terminal(InputAbandonReason::Reset)?;
2304 self.queue.drain();
2305 self.steer_queue.drain();
2306 self.post_admission_signal = PostAdmissionSignal::None;
2307 self.rebuild_queue_projections();
2308 self.debug_assert_queue_projection_alignment();
2309 Ok(ResetReport {
2310 inputs_abandoned: abandoned,
2311 })
2312 }
2313
2314 pub(crate) fn destroy_cleanup(&mut self) -> Result<usize, RuntimeDriverError> {
2315 let abandoned = self.abandon_all_non_terminal(InputAbandonReason::Destroyed)?;
2316 self.queue.drain();
2317 self.steer_queue.drain();
2318 self.post_admission_signal = PostAdmissionSignal::None;
2319 self.rebuild_queue_projections();
2320 self.debug_assert_queue_projection_alignment();
2321 Ok(abandoned)
2322 }
2323
2324 pub(crate) fn stop_runtime_cleanup(&mut self) -> Result<(), RuntimeDriverError> {
2325 self.abandon_all_non_terminal(InputAbandonReason::Stopped)?;
2326 self.queue.drain();
2327 self.steer_queue.drain();
2328 Ok(())
2329 }
2330
2331 pub(crate) fn finalize_stop_runtime(&mut self) -> Result<(), RuntimeDriverError> {
2332 self.stop_runtime_cleanup()
2333 }
2334
2335 pub fn recover_ephemeral(&mut self) -> Result<RecoveryReport, RuntimeDriverError> {
2336 crate::meerkat_machine::machine_recover_ephemeral_driver(self)
2337 }
2338
2339 pub(crate) fn recycle_preserving_work(&mut self) -> Result<usize, RuntimeDriverError> {
2340 let transferred = self
2341 .ledger
2342 .iter()
2343 .filter(|(id, _)| self.input_is_non_terminal_by_authority(id))
2344 .count();
2345 let runtime_id = self.runtime_id.clone();
2346 let ledger = self.ledger.clone();
2347 let preserved_dsl = self.dsl.clone();
2348 let preserved_admission_order = std::mem::take(&mut self.admission_order);
2349 let preserved_handling_mode = std::mem::take(&mut self.handling_mode);
2350 let preserved_is_prompt = std::mem::take(&mut self.is_prompt_set);
2351 let preserved_content_shape = std::mem::take(&mut self.content_shape);
2352 let preserved_request_id = std::mem::take(&mut self.request_id);
2353 let preserved_reservation_key = std::mem::take(&mut self.reservation_key);
2354 let preserved_policy_snapshot = std::mem::take(&mut self.policy_snapshot);
2355 let control = self.control.clone();
2356
2357 *self = Self::new_with_control(runtime_id, control);
2358 self.ledger = ledger;
2359 self.dsl = preserved_dsl;
2360 self.admission_order = preserved_admission_order;
2361 self.handling_mode = preserved_handling_mode;
2362 self.is_prompt_set = preserved_is_prompt;
2363 self.content_shape = preserved_content_shape;
2364 self.request_id = preserved_request_id;
2365 self.reservation_key = preserved_reservation_key;
2366 self.policy_snapshot = preserved_policy_snapshot;
2367
2368 self.recover_ephemeral()?;
2369 self.rebuild_queue_projections();
2370 self.debug_assert_queue_projection_alignment();
2371
2372 Ok(transferred)
2373 }
2374
2375 fn emit_event(&mut self, event: RuntimeEvent) {
2376 self.events.push(self.make_envelope(event));
2377 }
2378 fn make_envelope(&self, event: RuntimeEvent) -> RuntimeEventEnvelope {
2379 RuntimeEventEnvelope {
2380 id: crate::identifiers::RuntimeEventId::new(),
2381 timestamp: chrono::Utc::now(),
2382 runtime_id: self.runtime_id.clone(),
2383 event,
2384 causation_id: None,
2385 correlation_id: None,
2386 }
2387 }
2388
2389 fn resolve_admission_plan_input(
2390 authority: &MachineAdmissionAuthority,
2391 ) -> mm_dsl::MeerkatMachineInput {
2392 authority.to_dsl_input()
2393 }
2394
2395 fn handling_mode_from_admission_lane(lane: mm_dsl::InputLane) -> HandlingMode {
2396 match lane {
2397 mm_dsl::InputLane::Queue => HandlingMode::Queue,
2398 mm_dsl::InputLane::Steer => HandlingMode::Steer,
2399 }
2400 }
2401
2402 fn admission_plan_from_machine_effect(
2403 plan: mm_dsl::AdmissionPlanKind,
2404 queue_action: mm_dsl::AdmissionQueueActionKind,
2405 lane: mm_dsl::InputLane,
2406 existing_action: mm_dsl::AdmissionExistingQueuedActionKind,
2407 existing_input_id: Option<String>,
2408 ) -> Result<AdmissionPlan, RuntimeDriverError> {
2409 if matches!(plan, mm_dsl::AdmissionPlanKind::ConsumedOnAccept) {
2410 return Ok(AdmissionPlan::ConsumedOnAccept);
2411 }
2412
2413 let target = Self::handling_mode_from_admission_lane(lane);
2414 let queue_action = match queue_action {
2415 mm_dsl::AdmissionQueueActionKind::None => AdmissionQueueAction::None,
2416 mm_dsl::AdmissionQueueActionKind::EnqueueTo => {
2417 AdmissionQueueAction::EnqueueTo { target }
2418 }
2419 mm_dsl::AdmissionQueueActionKind::EnqueueFront => {
2420 AdmissionQueueAction::EnqueueFront { target }
2421 }
2422 };
2423 let existing_action = match (existing_action, existing_input_id) {
2424 (mm_dsl::AdmissionExistingQueuedActionKind::None, None) => None,
2425 (mm_dsl::AdmissionExistingQueuedActionKind::None, Some(existing_id)) => {
2426 return Err(RuntimeDriverError::Internal(format!(
2427 "ResolveAdmissionPlan emitted existing input '{existing_id}' without existing action"
2428 )));
2429 }
2430 (mm_dsl::AdmissionExistingQueuedActionKind::Coalesce, Some(existing_id)) => {
2431 let existing_id = existing_id
2432 .parse::<uuid::Uuid>()
2433 .map(InputId::from_uuid)
2434 .map_err(|err| {
2435 RuntimeDriverError::Internal(format!(
2436 "ResolveAdmissionPlan emitted invalid coalesce target id: {err}"
2437 ))
2438 })?;
2439 Some(ExistingQueuedAdmissionAction::Coalesce { existing_id })
2440 }
2441 (mm_dsl::AdmissionExistingQueuedActionKind::Supersede, Some(existing_id)) => {
2442 let existing_id = existing_id
2443 .parse::<uuid::Uuid>()
2444 .map(InputId::from_uuid)
2445 .map_err(|err| {
2446 RuntimeDriverError::Internal(format!(
2447 "ResolveAdmissionPlan emitted invalid supersede target id: {err}"
2448 ))
2449 })?;
2450 Some(ExistingQueuedAdmissionAction::Supersede { existing_id })
2451 }
2452 (action, None) => {
2453 return Err(RuntimeDriverError::Internal(format!(
2454 "ResolveAdmissionPlan emitted {action:?} without an existing input target"
2455 )));
2456 }
2457 };
2458
2459 Ok(AdmissionPlan::Queued {
2460 persist_and_queue: true,
2461 queue_action,
2462 existing_action,
2463 })
2464 }
2465
2466 fn resolved_idempotency_from_machine_effects(
2467 input_id: &InputId,
2468 effects: Vec<mm_dsl::MeerkatMachineEffect>,
2469 ) -> Result<Option<InputId>, RuntimeDriverError> {
2470 let Some((effect_input_id, result, existing_input_id)) =
2471 effects.into_iter().find_map(|effect| match effect {
2472 mm_dsl::MeerkatMachineEffect::AdmissionIdempotencyResolved {
2473 input_id,
2474 result,
2475 existing_input_id,
2476 } => Some((input_id, result, existing_input_id)),
2477 _ => None,
2478 })
2479 else {
2480 return Err(RuntimeDriverError::Internal(
2481 "ResolveAdmissionIdempotency emitted no AdmissionIdempotencyResolved effect".into(),
2482 ));
2483 };
2484
2485 if effect_input_id != input_id.to_string() {
2486 return Err(RuntimeDriverError::Internal(format!(
2487 "ResolveAdmissionIdempotency returned input id '{effect_input_id}' for '{input_id}'"
2488 )));
2489 }
2490
2491 match (result, existing_input_id) {
2492 (mm_dsl::AdmissionIdempotencyResultKind::Accept, None) => Ok(None),
2493 (mm_dsl::AdmissionIdempotencyResultKind::Accept, Some(existing_id)) => {
2494 Err(RuntimeDriverError::Internal(format!(
2495 "ResolveAdmissionIdempotency accepted '{input_id}' but emitted existing input '{existing_id}'"
2496 )))
2497 }
2498 (mm_dsl::AdmissionIdempotencyResultKind::Deduplicated, Some(existing_id)) => {
2499 let existing_id = existing_id
2500 .parse::<uuid::Uuid>()
2501 .map(InputId::from_uuid)
2502 .map_err(|err| {
2503 RuntimeDriverError::Internal(format!(
2504 "ResolveAdmissionIdempotency emitted invalid existing input id: {err}"
2505 ))
2506 })?;
2507 Ok(Some(existing_id))
2508 }
2509 (mm_dsl::AdmissionIdempotencyResultKind::Deduplicated, None) => {
2510 Err(RuntimeDriverError::Internal(format!(
2511 "ResolveAdmissionIdempotency deduplicated '{input_id}' without an existing input"
2512 )))
2513 }
2514 }
2515 }
2516
2517 fn admission_validation_from_machine_effects(
2518 input_id: &InputId,
2519 effects: Vec<mm_dsl::MeerkatMachineEffect>,
2520 ) -> Result<Option<mm_dsl::AdmissionRejectReasonKind>, RuntimeDriverError> {
2521 let Some((effect_input_id, result, reject_reason)) =
2522 effects.into_iter().find_map(|effect| match effect {
2523 mm_dsl::MeerkatMachineEffect::AdmissionValidationResolved {
2524 input_id,
2525 result,
2526 reject_reason,
2527 } => Some((input_id, result, reject_reason)),
2528 _ => None,
2529 })
2530 else {
2531 return Err(RuntimeDriverError::Internal(
2532 "ResolveAdmissionValidation emitted no AdmissionValidationResolved effect".into(),
2533 ));
2534 };
2535
2536 if effect_input_id != input_id.to_string() {
2537 return Err(RuntimeDriverError::Internal(format!(
2538 "ResolveAdmissionValidation returned input id '{effect_input_id}' for '{input_id}'"
2539 )));
2540 }
2541
2542 match (result, reject_reason) {
2543 (mm_dsl::AdmissionValidationResultKind::Accept, None) => Ok(None),
2544 (mm_dsl::AdmissionValidationResultKind::Accept, Some(reason)) => {
2545 Err(RuntimeDriverError::Internal(format!(
2546 "ResolveAdmissionValidation accepted '{input_id}' but emitted rejection reason {reason:?}"
2547 )))
2548 }
2549 (mm_dsl::AdmissionValidationResultKind::Reject, Some(reason)) => Ok(Some(reason)),
2550 (mm_dsl::AdmissionValidationResultKind::Reject, None) => {
2551 Err(RuntimeDriverError::Internal(format!(
2552 "ResolveAdmissionValidation rejected '{input_id}' without a typed reason"
2553 )))
2554 }
2555 }
2556 }
2557
2558 fn resolve_admission_validation(
2559 &self,
2560 input_id: &InputId,
2561 facts: AdmissionValidationFacts<'_>,
2562 ) -> Result<Option<mm_dsl::AdmissionRejectReasonKind>, RuntimeDriverError> {
2563 let effects = self.dsl_preview(
2564 mm_dsl::MeerkatMachineInput::ResolveAdmissionValidation {
2565 input_id: Self::dsl_key(input_id),
2566 input_kind: mm_dsl::AdmissionInputKind::from(facts.input_kind),
2567 input_origin: mm_dsl::AdmissionInputOriginKind::from(facts.input_origin),
2568 durability: mm_dsl::InputDurabilityKind::from(facts.durability),
2569 peer_handling_mode_valid: facts.peer_handling_mode_valid,
2570 peer_response_terminal_structurally_valid: facts
2571 .peer_response_terminal_structurally_valid,
2572 peer_response_terminal_observed_status: facts
2573 .peer_response_terminal_observed_status,
2574 },
2575 "ResolveAdmissionValidation",
2576 )?;
2577 Self::admission_validation_from_machine_effects(input_id, effects)
2578 }
2579
2580 fn peer_response_terminal_observed_status(
2581 input: &Input,
2582 ) -> mm_dsl::PeerResponseTerminalObservedStatus {
2583 let Input::Peer(peer) = input else {
2584 return mm_dsl::PeerResponseTerminalObservedStatus::NotPeerTerminal;
2585 };
2586 let Some(crate::input::PeerConvention::ResponseTerminal { status, .. }) = &peer.convention
2587 else {
2588 return mm_dsl::PeerResponseTerminalObservedStatus::NotPeerTerminal;
2589 };
2590
2591 match status {
2592 meerkat_core::handles::PeerResponseTerminalProjectionStatus::Completed => {
2593 mm_dsl::PeerResponseTerminalObservedStatus::Completed
2594 }
2595 meerkat_core::handles::PeerResponseTerminalProjectionStatus::Failed => {
2596 mm_dsl::PeerResponseTerminalObservedStatus::Failed
2597 }
2598 meerkat_core::handles::PeerResponseTerminalProjectionStatus::Cancelled => {
2599 mm_dsl::PeerResponseTerminalObservedStatus::Cancelled
2600 }
2601 }
2602 }
2603
2604 fn peer_response_terminal_generated_rejection_detail(input: &Input) -> Option<String> {
2605 let Input::Peer(peer) = input else {
2606 return None;
2607 };
2608 let Some(crate::input::PeerConvention::ResponseTerminal { status, .. }) = &peer.convention
2609 else {
2610 return None;
2611 };
2612
2613 Some(format!(
2614 "peer response terminal status rejected by generated authority: {}",
2615 status.label()
2616 ))
2617 }
2618
2619 fn reject_reason_from_machine_validation(
2624 reason: mm_dsl::AdmissionRejectReasonKind,
2625 input_kind: crate::identifiers::InputKind,
2626 peer_handling_mode_detail: Option<&str>,
2627 peer_response_terminal_detail: Option<&str>,
2628 ) -> Result<RejectReason, RuntimeDriverError> {
2629 let missing_detail = || {
2630 RuntimeDriverError::Internal(format!(
2631 "ResolveAdmissionValidation emitted {reason:?} without matching validation detail"
2632 ))
2633 };
2634 match reason {
2635 mm_dsl::AdmissionRejectReasonKind::DurabilityMissing => {
2636 Ok(RejectReason::DurabilityViolation {
2637 detail: "input durability observation missing".to_owned(),
2638 })
2639 }
2640 mm_dsl::AdmissionRejectReasonKind::ExternalDerivedDurabilityForbidden => {
2641 Ok(RejectReason::DurabilityViolation {
2642 detail: "External ingress cannot submit derived inputs".to_owned(),
2643 })
2644 }
2645 mm_dsl::AdmissionRejectReasonKind::DerivedDurabilityForbiddenForInputKind => {
2646 Ok(RejectReason::DurabilityViolation {
2647 detail: format!("Derived durability forbidden for {input_kind}"),
2648 })
2649 }
2650 mm_dsl::AdmissionRejectReasonKind::PeerHandlingModeInvalid => {
2651 Ok(RejectReason::PeerHandlingModeInvalid {
2652 detail: peer_handling_mode_detail
2653 .ok_or_else(missing_detail)?
2654 .to_owned(),
2655 })
2656 }
2657 mm_dsl::AdmissionRejectReasonKind::PeerResponseTerminalInvalid => {
2658 Ok(RejectReason::PeerResponseTerminalInvalid {
2659 detail: peer_response_terminal_detail
2660 .ok_or_else(missing_detail)?
2661 .to_owned(),
2662 })
2663 }
2664 }
2665 }
2666
2667 fn reject_peer_response_terminal_observation_if_present(
2668 &mut self,
2669 input: &Input,
2670 detail: &str,
2671 ) {
2672 let Input::Peer(peer) = input else {
2673 return;
2674 };
2675 let Some(crate::input::PeerConvention::ResponseTerminal { request_id, .. }) =
2676 &peer.convention
2677 else {
2678 return;
2679 };
2680 let Ok(corr_id) = uuid::Uuid::parse_str(request_id) else {
2681 return;
2682 };
2683
2684 if let Err(error) = self.dsl_apply(
2685 mm_dsl::MeerkatMachineInput::PeerResponseRejected {
2686 corr_id: corr_id.into(),
2687 },
2688 "PeerResponseRejected(invalid peer terminal observation)",
2689 ) {
2690 tracing::debug!(
2691 request_id,
2692 detail,
2693 error = ?error,
2694 "generated peer response rejection did not match pending request"
2695 );
2696 }
2697 }
2698
2699 fn resolve_idempotency(
2700 &mut self,
2701 input_id: &InputId,
2702 idempotency_key: Option<String>,
2703 ) -> Result<Option<InputId>, RuntimeDriverError> {
2704 let effects = self.dsl_apply_effects(
2705 mm_dsl::MeerkatMachineInput::ResolveAdmissionIdempotency {
2706 input_id: Self::dsl_key(input_id),
2707 idempotency_key,
2708 },
2709 "ResolveAdmissionIdempotency",
2710 )?;
2711 Self::resolved_idempotency_from_machine_effects(input_id, effects)
2712 }
2713
2714 pub(crate) fn register_accepted_idempotency(
2715 &mut self,
2716 input_id: &InputId,
2717 idempotency_key: Option<&IdempotencyKey>,
2718 ) -> Result<(), RuntimeDriverError> {
2719 let Some(idempotency_key) = idempotency_key else {
2720 return Ok(());
2721 };
2722 self.dsl_apply(
2723 mm_dsl::MeerkatMachineInput::RegisterAcceptedIdempotency {
2724 input_id: Self::dsl_key(input_id),
2725 idempotency_key: idempotency_key.to_string(),
2726 },
2727 "RegisterAcceptedIdempotency",
2728 )
2729 }
2730
2731 fn resolved_admission_from_machine_effects(
2732 &self,
2733 input: &Input,
2734 authority: MachineAdmissionAuthority,
2735 effects: Vec<mm_dsl::MeerkatMachineEffect>,
2736 mint_execution_capability: bool,
2737 ) -> Result<ResolvedAdmission, RuntimeDriverError> {
2738 let Some(effect) = effects.into_iter().find_map(|effect| match effect {
2739 mm_dsl::MeerkatMachineEffect::AdmissionResolved {
2740 input_id,
2741 policy_version,
2742 policy_apply_mode,
2743 policy_wake_mode,
2744 policy_queue_mode,
2745 policy_consume_point,
2746 policy_drain_policy,
2747 policy_routing_disposition,
2748 lane,
2749 plan,
2750 queue_action,
2751 existing_action,
2752 existing_input_id,
2753 requires_active_pre_admission,
2754 runtime_boundary,
2755 runtime_execution_kind,
2756 runtime_peer_response_terminal_apply_intent,
2757 record_transcript,
2758 request_immediate_processing,
2759 interrupt_yielding,
2760 wake_if_idle,
2761 execution_handling_mode,
2762 live_interrupt_required,
2763 } => Some((
2764 input_id,
2765 policy_version,
2766 policy_apply_mode,
2767 policy_wake_mode,
2768 policy_queue_mode,
2769 policy_consume_point,
2770 policy_drain_policy,
2771 policy_routing_disposition,
2772 lane,
2773 plan,
2774 queue_action,
2775 existing_action,
2776 existing_input_id,
2777 requires_active_pre_admission,
2778 runtime_boundary,
2779 runtime_execution_kind,
2780 runtime_peer_response_terminal_apply_intent,
2781 record_transcript,
2782 request_immediate_processing,
2783 interrupt_yielding,
2784 wake_if_idle,
2785 execution_handling_mode,
2786 live_interrupt_required,
2787 )),
2788 _ => None,
2789 }) else {
2790 return Err(RuntimeDriverError::Internal(
2791 "ResolveAdmissionPlan emitted no AdmissionResolved effect".into(),
2792 ));
2793 };
2794
2795 let (
2796 input_id,
2797 policy_version,
2798 policy_apply_mode,
2799 policy_wake_mode,
2800 policy_queue_mode,
2801 policy_consume_point,
2802 policy_drain_policy,
2803 policy_routing_disposition,
2804 lane,
2805 plan,
2806 queue_action,
2807 existing_action,
2808 existing_input_id,
2809 requires_active_pre_admission,
2810 runtime_boundary,
2811 runtime_execution_kind,
2812 runtime_peer_response_terminal_apply_intent,
2813 record_transcript,
2814 request_immediate_processing,
2815 interrupt_yielding,
2816 wake_if_idle,
2817 execution_handling_mode,
2818 live_interrupt_required,
2819 ) = effect;
2820
2821 if input_id != authority.input_id() {
2822 return Err(RuntimeDriverError::Internal(format!(
2823 "ResolveAdmissionPlan returned input id '{input_id}' for '{}'",
2824 authority.input_id()
2825 )));
2826 }
2827
2828 let policy = PolicyDecision {
2829 apply_mode: policy_apply_mode.into(),
2830 wake_mode: policy_wake_mode.into(),
2831 queue_mode: policy_queue_mode.into(),
2832 consume_point: policy_consume_point.into(),
2833 drain_policy: policy_drain_policy.into(),
2834 routing_disposition: policy_routing_disposition.into(),
2835 record_transcript,
2836 emit_operator_content: record_transcript,
2837 policy_version: PolicyVersion(policy_version),
2838 };
2839 let runtime_semantics = RuntimeInputSemantics {
2840 boundary: runtime_boundary.into(),
2841 execution_kind: runtime_execution_kind.into(),
2842 execution_handling_mode: execution_handling_mode
2846 .map(Self::handling_mode_from_admission_lane),
2847 peer_response_terminal_apply_intent: runtime_peer_response_terminal_apply_intent
2848 .map(Into::into),
2849 live_interrupt_required,
2850 };
2851 let handling_mode = Self::handling_mode_from_admission_lane(lane);
2852 let admission_plan = Self::admission_plan_from_machine_effect(
2853 plan,
2854 queue_action,
2855 lane,
2856 existing_action,
2857 existing_input_id,
2858 )?;
2859
2860 Ok(ResolvedAdmission::from_machine_resolution(
2861 policy,
2862 handling_mode,
2863 runtime_semantics,
2864 crate::input::runtime_input_projection(input),
2865 admission_plan,
2866 CoarseAdmissionFlags {
2867 request_immediate_processing,
2868 interrupt_yielding,
2869 wake_if_idle,
2870 },
2871 requires_active_pre_admission,
2872 authority,
2873 mint_execution_capability.then_some((input_id, lane, plan)),
2874 ))
2875 }
2876
2877 fn resolve_admission_with_wake_policy(
2878 &self,
2879 input: &Input,
2880 without_wake: bool,
2881 active_turn_boundary_available: bool,
2882 ) -> Result<ResolvedAdmission, RuntimeDriverError> {
2883 let existing_superseded_id = self.existing_superseded_input(input).map(|(id, _)| id);
2884 let authority = MachineAdmissionAuthority::new(
2885 input.id().to_string(),
2886 mm_dsl::AdmissionInputKind::from(input.kind()),
2887 input.handling_mode().map(mm_dsl::InputLane::from),
2888 mm_dsl::AdmissionContinuationKind::from(input.continuation_kind()),
2889 self.matches_silent_intent_authority(input),
2890 existing_superseded_id,
2891 self.runtime_phase_snapshot() == RuntimeState::Running,
2892 active_turn_boundary_available,
2893 without_wake,
2894 );
2895 let effects = self.dsl_preview(
2896 Self::resolve_admission_plan_input(&authority),
2897 "ResolveAdmissionPlan",
2898 )?;
2899 self.resolved_admission_from_machine_effects(input, authority, effects, false)
2900 }
2901
2902 pub(crate) fn resolve_admission(
2903 &self,
2904 input: &Input,
2905 ) -> Result<ResolvedAdmission, RuntimeDriverError> {
2906 self.resolve_admission_with_wake_policy(input, false, false)
2907 }
2908
2909 pub(crate) fn resolve_admission_with_active_turn_boundary(
2910 &self,
2911 input: &Input,
2912 active_turn_boundary_available: bool,
2913 ) -> Result<ResolvedAdmission, RuntimeDriverError> {
2914 self.resolve_admission_with_wake_policy(input, false, active_turn_boundary_available)
2915 }
2916
2917 pub(crate) fn resolve_admission_without_wake_with_active_turn_boundary(
2918 &self,
2919 input: &Input,
2920 active_turn_boundary_available: bool,
2921 ) -> Result<ResolvedAdmission, RuntimeDriverError> {
2922 self.resolve_admission_with_wake_policy(input, true, active_turn_boundary_available)
2923 }
2924
2925 pub(crate) fn machine_apply_accept_with_completion_signal(
2926 &mut self,
2927 input_id: &InputId,
2928 flags: crate::accept::CoarseAdmissionFlags,
2929 ) -> Result<(), RuntimeDriverError> {
2930 self.dsl_apply(
2931 mm_dsl::MeerkatMachineInput::AcceptWithCompletion {
2932 input_id: mm_dsl::InputId::from_domain(input_id),
2933 request_immediate_processing: flags.request_immediate_processing,
2934 interrupt_yielding: flags.interrupt_yielding,
2935 wake_if_idle: flags.wake_if_idle,
2936 },
2937 "AcceptWithCompletion(RuntimeDriver)",
2938 )
2939 }
2940
2941 pub(crate) async fn accept_resolved_input(
2942 &mut self,
2943 input: Input,
2944 resolved: crate::accept::ResolvedAdmission,
2945 ) -> Result<AcceptOutcome, RuntimeDriverError> {
2946 let runtime_phase = self.runtime_phase_snapshot();
2947 let lifecycle_facts = crate::meerkat_machine::classify_runtime_lifecycle_state(
2948 runtime_phase,
2949 )
2950 .map_err(|err| {
2951 RuntimeDriverError::Internal(format!(
2952 "generated runtime lifecycle admission classification failed: {err}"
2953 ))
2954 })?;
2955 if !lifecycle_facts.can_accept_input() {
2956 return match lifecycle_facts.ingress_admission {
2957 mm_dsl::RuntimeIngressAdmission::Destroyed => Err(RuntimeDriverError::Destroyed),
2958 mm_dsl::RuntimeIngressAdmission::Open
2959 | mm_dsl::RuntimeIngressAdmission::NotReady => Err(RuntimeDriverError::NotReady {
2960 state: runtime_phase,
2961 }),
2962 };
2963 }
2964
2965 let input_id = input.id().clone();
2966 let peer_handling_mode_error =
2967 crate::peer_handling_mode::validate_peer_handling_mode(&input)
2968 .err()
2969 .map(|error| error.to_string());
2970 let peer_response_terminal_structural_error =
2971 crate::input::validate_peer_response_terminal_fact(&input)
2972 .err()
2973 .map(|error| error.to_string());
2974 let peer_response_terminal_observed_status =
2975 Self::peer_response_terminal_observed_status(&input);
2976 let peer_response_terminal_detail = peer_response_terminal_structural_error
2977 .clone()
2978 .or_else(|| Self::peer_response_terminal_generated_rejection_detail(&input));
2979 if let Some(reason) = self.resolve_admission_validation(
2980 &input_id,
2981 AdmissionValidationFacts {
2982 input_kind: input.kind(),
2983 input_origin: &input.header().source,
2984 durability: input.header().durability,
2985 peer_handling_mode_valid: peer_handling_mode_error.is_none(),
2986 peer_response_terminal_structurally_valid: peer_response_terminal_structural_error
2987 .is_none(),
2988 peer_response_terminal_observed_status,
2989 },
2990 )? {
2991 if matches!(
2992 reason,
2993 mm_dsl::AdmissionRejectReasonKind::PeerResponseTerminalInvalid
2994 ) && let Some(detail) = peer_response_terminal_detail.as_deref()
2995 {
2996 self.reject_peer_response_terminal_observation_if_present(&input, detail);
2997 }
2998 let reason = Self::reject_reason_from_machine_validation(
2999 reason,
3000 input.kind(),
3001 peer_handling_mode_error.as_deref(),
3002 peer_response_terminal_detail.as_deref(),
3003 )?;
3004 return Ok(AcceptOutcome::Rejected { reason });
3005 }
3006
3007 if resolved.authority().input_id() != input_id.to_string() {
3008 return Err(RuntimeDriverError::Internal(format!(
3009 "resolved admission authority id '{}' did not match accepted input '{input_id}'",
3010 resolved.authority().input_id()
3011 )));
3012 }
3013
3014 if let Some(existing_id) = self.resolve_idempotency(
3015 &input_id,
3016 input
3017 .header()
3018 .idempotency_key
3019 .as_ref()
3020 .map(std::string::ToString::to_string),
3021 )? {
3022 tracing::debug!(
3023 work_id = ?input_id,
3024 existing_id = ?existing_id,
3025 "input deduplicated"
3026 );
3027 self.emit_event(RuntimeEvent::InputLifecycle(
3028 InputLifecycleEvent::Deduplicated {
3029 input_id: input_id.clone(),
3030 existing_id: existing_id.clone(),
3031 },
3032 ));
3033 return Ok(AcceptOutcome::Deduplicated {
3034 input_id,
3035 existing_id,
3036 });
3037 }
3038
3039 let mut state = InputState::new_accepted(input_id.clone());
3040 state.durability = Some(input.header().durability);
3041 state.idempotency_key = input.header().idempotency_key.clone();
3042 let existing_superseded_id = self.existing_superseded_input(&input).map(|(id, _)| id);
3043 let authority = MachineAdmissionAuthority::new(
3044 input_id.to_string(),
3045 mm_dsl::AdmissionInputKind::from(input.kind()),
3046 input.handling_mode().map(mm_dsl::InputLane::from),
3047 mm_dsl::AdmissionContinuationKind::from(input.continuation_kind()),
3048 self.matches_silent_intent_authority(&input),
3049 existing_superseded_id,
3050 self.runtime_phase_snapshot() == RuntimeState::Running,
3051 resolved.authority().active_turn_boundary_available(),
3052 resolved.authority().without_wake(),
3053 );
3054 let effects = self.dsl_apply_effects(
3055 Self::resolve_admission_plan_input(&authority),
3056 "ResolveAdmissionPlan",
3057 )?;
3058 let committed_resolved =
3059 self.resolved_admission_from_machine_effects(&input, authority, effects, true)?;
3060 if !resolved.semantically_equivalent_to(&committed_resolved) {
3061 return Err(RuntimeDriverError::Internal(format!(
3062 "committed admission resolution diverged from preview: preview={resolved:?}, committed={committed_resolved:?}"
3063 )));
3064 }
3065 let (policy, handling_mode, runtime_semantics, primitive_projection, admission_plan) =
3066 committed_resolved
3067 .consume_execution_capability(&input_id)
3068 .map_err(RuntimeDriverError::Internal)?;
3069
3070 let content_shape = ContentShape::from_kind(input.kind());
3071 let is_prompt = matches!(input, Input::Prompt(_));
3072 match admission_plan {
3073 AdmissionPlan::ConsumedOnAccept => {
3074 self.dsl_apply(
3075 mm_dsl::MeerkatMachineInput::QueueAccepted {
3076 input_id: Self::dsl_key(&input_id),
3077 },
3078 "QueueAccepted(consumed_on_accept)",
3079 )?;
3080 self.dsl_apply(
3081 mm_dsl::MeerkatMachineInput::ConsumeOnAccept {
3082 input_id: Self::dsl_key(&input_id),
3083 },
3084 "ConsumeOnAccept",
3085 )?;
3086 self.register_accepted_idempotency(
3087 &input_id,
3088 input.header().idempotency_key.as_ref(),
3089 )?;
3090 let terminal_outcome = self.input_terminal_outcome(&input_id).ok_or_else(|| {
3091 RuntimeDriverError::Internal(format!(
3092 "machine terminal projection missing consume-on-accept outcome for {input_id}"
3093 ))
3094 })?;
3095 if terminal_outcome != InputTerminalOutcome::Consumed {
3096 return Err(RuntimeDriverError::Internal(format!(
3097 "machine terminal projection for consume-on-accept {input_id} was {terminal_outcome:?}"
3098 )));
3099 }
3100 let now = Utc::now();
3101 state.policy = Some(PolicySnapshot {
3102 version: policy.policy_version,
3103 decision: policy.clone(),
3104 });
3105 state.history.push(InputStateHistoryEntry {
3106 timestamp: now,
3107 from: InputLifecycleState::Accepted,
3108 to: InputLifecycleState::Consumed,
3109 reason: Some("ConsumeOnAccept (Ignore+OnAccept)".into()),
3110 });
3111 state.updated_at = now;
3112 self.ledger.accept(state);
3113 self.record_admission_metadata(
3114 &input_id,
3115 &content_shape,
3116 handling_mode,
3117 runtime_semantics,
3118 primitive_projection,
3119 is_prompt,
3120 Some(&policy),
3121 None,
3122 None,
3123 );
3124 self.emit_event(RuntimeEvent::InputLifecycle(
3125 InputLifecycleEvent::Accepted {
3126 input_id: input_id.clone(),
3127 },
3128 ));
3129 tracing::debug!(work_id = ?input_id, "input consumed on accept");
3130 }
3131 AdmissionPlan::Queued {
3132 persist_and_queue,
3133 queue_action,
3134 existing_action,
3135 } => {
3136 if persist_and_queue {
3137 self.apply_persist_and_queue(
3138 &input_id,
3139 &input,
3140 &content_shape,
3141 handling_mode,
3142 runtime_semantics,
3143 primitive_projection,
3144 is_prompt,
3145 &policy,
3146 state,
3147 queue_action,
3148 existing_action.as_ref(),
3149 )?;
3150 }
3151 }
3152 }
3153 self.rebuild_queue_projections();
3154 self.debug_assert_queue_projection_alignment();
3155
3156 let final_bundle = self.stored_input_state(&input_id).ok_or_else(|| {
3157 RuntimeDriverError::Internal(format!(
3158 "accepted input {input_id} missing generated lifecycle seed"
3159 ))
3160 })?;
3161 Ok(AcceptOutcome::Accepted {
3162 input_id,
3163 policy,
3164 state: final_bundle.state,
3165 seed: final_bundle.seed,
3166 })
3167 }
3168
3169 pub(crate) async fn preview_accept_resolved_input(
3170 &self,
3171 input: Input,
3172 resolved: &crate::accept::ResolvedAdmission,
3173 ) -> Result<AcceptOutcome, RuntimeDriverError> {
3174 let mut staged = self.clone_with_isolated_dsl_authority();
3175 staged.ensure_contract_session_authority()?;
3176 let resolved = if resolved.authority().without_wake() {
3177 staged.resolve_admission_without_wake_with_active_turn_boundary(
3178 &input,
3179 resolved.authority().active_turn_boundary_available(),
3180 )?
3181 } else {
3182 staged.resolve_admission_with_active_turn_boundary(
3183 &input,
3184 resolved.authority().active_turn_boundary_available(),
3185 )?
3186 };
3187 staged.accept_resolved_input(input, resolved).await
3188 }
3189
3190 pub fn abandon_all_non_terminal(
3191 &mut self,
3192 reason: InputAbandonReason,
3193 ) -> Result<usize, RuntimeDriverError> {
3194 let non_terminal_ids: Vec<InputId> = self
3195 .ledger
3196 .iter()
3197 .filter_map(|(id, _)| {
3198 if self.input_is_non_terminal_by_authority(id) {
3199 Some(id.clone())
3200 } else {
3201 None
3202 }
3203 })
3204 .collect();
3205 let dsl_reason = mm_dsl::InputAbandonReason::from(&reason);
3206 let mut count = 0;
3207 for id in &non_terminal_ids {
3208 let key = Self::dsl_key(id);
3209 let attempt_count = u64::from(self.input_attempt_count(id));
3210 let from_phase = self.input_phase(id).ok_or_else(|| {
3211 RuntimeDriverError::Internal(format!(
3212 "generated input lifecycle phase missing before abandoning input {id}"
3213 ))
3214 })?;
3215 self.dsl_apply(
3216 mm_dsl::MeerkatMachineInput::AbandonInput {
3217 input_id: key.clone(),
3218 reason: dsl_reason,
3219 attempt_count,
3220 },
3221 "AbandonInput",
3222 )?;
3223
3224 self.sync_terminal_projection_from_machine(
3225 id,
3226 from_phase,
3227 InputLifecycleState::Abandoned,
3228 "Abandon",
3229 )?;
3230 count += 1;
3231 self.events
3232 .push(self.make_envelope(RuntimeEvent::InputLifecycle(
3233 InputLifecycleEvent::Abandoned {
3234 input_id: id.clone(),
3235 reason: reason.clone(),
3236 },
3237 )));
3238 }
3239 Ok(count)
3240 }
3241
3242 pub(crate) fn abandon_staged_inputs(
3243 &mut self,
3244 input_ids: &[InputId],
3245 reason: InputAbandonReason,
3246 ) -> Result<usize, RuntimeDriverError> {
3247 let dsl_reason = mm_dsl::InputAbandonReason::from(&reason);
3248 let mut count = 0;
3249 for input_id in input_ids {
3250 if self.input_phase(input_id) != Some(InputLifecycleState::Staged) {
3251 continue;
3252 }
3253 let from_phase =
3254 self.input_phase_required(input_id, "before abandoning staged input")?;
3255 let attempt_count = u64::from(self.input_attempt_count(input_id));
3256 self.dsl_apply(
3257 mm_dsl::MeerkatMachineInput::AbandonInput {
3258 input_id: Self::dsl_key(input_id),
3259 reason: dsl_reason,
3260 attempt_count,
3261 },
3262 "AbandonInput(CancelledRun)",
3263 )?;
3264
3265 self.sync_terminal_projection_from_machine(
3266 input_id,
3267 from_phase,
3268 InputLifecycleState::Abandoned,
3269 "Abandon",
3270 )?;
3271 count += 1;
3272 self.events
3273 .push(self.make_envelope(RuntimeEvent::InputLifecycle(
3274 InputLifecycleEvent::Abandoned {
3275 input_id: input_id.clone(),
3276 reason: reason.clone(),
3277 },
3278 )));
3279 }
3280
3281 self.rebuild_queue_projections();
3282 self.debug_assert_queue_projection_alignment();
3283 Ok(count)
3284 }
3285
3286 pub(crate) fn abandon_pending_inputs(
3287 &mut self,
3288 reason: InputAbandonReason,
3289 ) -> Result<usize, RuntimeDriverError> {
3290 let abandoned = self.abandon_all_non_terminal(reason)?;
3291 self.queue.drain();
3292 self.steer_queue.drain();
3293 self.post_admission_signal = PostAdmissionSignal::None;
3294 self.rebuild_queue_projections();
3295 self.debug_assert_queue_projection_alignment();
3296 Ok(abandoned)
3297 }
3298
3299 pub(crate) fn machine_realize_run_completed(
3305 &mut self,
3306 run_id: &RunId,
3307 consumed_input_ids: &[InputId],
3308 ) -> Result<(), RuntimeDriverError> {
3309 self.consume_inputs(consumed_input_ids, run_id)
3310 }
3311
3312 pub(crate) fn machine_realize_run_failed(
3314 &mut self,
3315 run_id: &RunId,
3316 contributing_input_ids: &[InputId],
3317 replay_plan: &ReplayQueuedContributorsPlan,
3318 ) -> Result<(), RuntimeDriverError> {
3319 tracing::debug!(
3320 run_id = ?run_id,
3321 kind = replay_plan.notice_kind,
3322 queue = replay_plan.queue_work_ids.len(),
3323 steer = replay_plan.steer_work_ids.len(),
3324 "runtime replayed queued contributors"
3325 );
3326
3327 self.rollback_staged(contributing_input_ids)
3331 }
3332
3333 pub(crate) fn machine_realize_run_cancelled(
3335 &mut self,
3336 run_id: &RunId,
3337 contributing_input_ids: &[InputId],
3338 ) -> Result<(), RuntimeDriverError> {
3339 tracing::debug!(
3340 run_id = ?run_id,
3341 contributors = contributing_input_ids.len(),
3342 "runtime abandoned cancelled run contributors"
3343 );
3344 let _ =
3345 self.abandon_staged_inputs(contributing_input_ids, InputAbandonReason::Cancelled)?;
3346 Ok(())
3347 }
3348
3349 pub(crate) fn machine_realize_boundary_applied(
3351 &mut self,
3352 run_id: &RunId,
3353 receipt: &RunBoundaryReceipt,
3354 ) -> Result<(), RuntimeDriverError> {
3355 tracing::debug!(
3356 contributors = receipt.contributing_input_ids.len(),
3357 sequence = receipt.sequence,
3358 "runtime boundary applied"
3359 );
3360
3361 for input_id in &receipt.contributing_input_ids {
3362 let key = Self::dsl_key(input_id);
3363 if !self.with_dsl_state(|state| state.input_phases.contains_key(&key)) {
3364 continue;
3365 }
3366 if self.input_phase(input_id) != Some(InputLifecycleState::Staged) {
3370 continue;
3371 }
3372
3373 self.dsl_apply(
3374 mm_dsl::MeerkatMachineInput::MarkApplied {
3375 input_id: key.clone(),
3376 },
3377 "MarkApplied",
3378 )?;
3379 self.dsl_apply(
3380 mm_dsl::MeerkatMachineInput::MarkAppliedPendingConsumption {
3381 input_id: key.clone(),
3382 },
3383 "MarkAppliedPendingConsumption",
3384 )?;
3385 self.dsl_apply(
3390 mm_dsl::MeerkatMachineInput::RecordBoundarySeq {
3391 input_id: key,
3392 run_id: mm_dsl::RunId::from_domain(run_id),
3393 },
3394 "RecordBoundarySeq",
3395 )?;
3396
3397 let now = Utc::now();
3398 if let Some(state) = self.ledger.get_mut(input_id) {
3399 state.history.push(InputStateHistoryEntry {
3400 timestamp: now,
3401 from: InputLifecycleState::Staged,
3402 to: InputLifecycleState::Applied,
3403 reason: Some(format!("MarkApplied({run_id})")),
3404 });
3405 state.history.push(InputStateHistoryEntry {
3406 timestamp: now,
3407 from: InputLifecycleState::Applied,
3408 to: InputLifecycleState::AppliedPendingConsumption,
3409 reason: Some(format!(
3410 "MarkAppliedPendingConsumption(boundary_sequence={})",
3411 receipt.sequence
3412 )),
3413 });
3414 state.updated_at = now;
3415 }
3416 self.events
3417 .push(self.make_envelope(RuntimeEvent::InputLifecycle(
3418 InputLifecycleEvent::Applied {
3419 input_id: input_id.clone(),
3420 run_id: run_id.clone(),
3421 },
3422 )));
3423 }
3424 Ok(())
3425 }
3426}
3427
3428#[cfg_attr(not(target_arch = "wasm32"), async_trait::async_trait)]
3429#[cfg_attr(target_arch = "wasm32", async_trait::async_trait(?Send))]
3430impl crate::traits::RuntimeDriver for EphemeralRuntimeDriver {
3431 async fn accept_input(&mut self, input: Input) -> Result<AcceptOutcome, RuntimeDriverError> {
3432 let resolved = self.resolve_admission(&input)?;
3433 let flags = resolved.coarse_flags();
3434 self.ensure_contract_session_authority()?;
3435 let checkpoint = self.rollback_snapshot();
3436 let outcome = match self.accept_resolved_input(input, resolved).await {
3437 Ok(outcome) => outcome,
3438 Err(err) => {
3439 self.restore_rollback_snapshot(checkpoint);
3440 return Err(err);
3441 }
3442 };
3443 if let AcceptOutcome::Accepted { input_id, .. } = &outcome
3444 && let Err(err) = self.machine_apply_accept_with_completion_signal(input_id, flags)
3445 {
3446 self.restore_rollback_snapshot(checkpoint);
3447 return Err(err);
3448 }
3449 Ok(outcome)
3450 }
3451
3452 async fn on_runtime_event(
3453 &mut self,
3454 _event: RuntimeEventEnvelope,
3455 ) -> Result<(), RuntimeDriverError> {
3456 Ok(())
3457 }
3458
3459 async fn recover(&mut self) -> Result<RecoveryReport, RuntimeDriverError> {
3460 self.recover_ephemeral()
3461 }
3462 fn runtime_state(&self) -> RuntimeState {
3463 self.runtime_phase_snapshot()
3464 }
3465 fn input_state(&self, input_id: &InputId) -> Option<&InputState> {
3466 self.ledger.get(input_id)
3467 }
3468 fn input_phase(&self, input_id: &InputId) -> Option<InputLifecycleState> {
3469 EphemeralRuntimeDriver::input_phase(self, input_id)
3470 }
3471 fn input_last_run_id(&self, input_id: &InputId) -> Option<RunId> {
3472 EphemeralRuntimeDriver::input_last_run_id(self, input_id)
3473 }
3474 fn input_last_boundary_sequence(&self, input_id: &InputId) -> Option<u64> {
3475 EphemeralRuntimeDriver::input_last_boundary_sequence(self, input_id)
3476 }
3477 fn stored_input_state(&self, input_id: &InputId) -> Option<StoredInputState> {
3478 EphemeralRuntimeDriver::stored_input_state(self, input_id)
3479 }
3480 fn input_id_for_idempotency_key(&self, idempotency_key: &str) -> Option<InputId> {
3481 EphemeralRuntimeDriver::input_id_for_idempotency_key(self, idempotency_key)
3482 }
3483 fn active_input_ids(&self) -> Vec<InputId> {
3484 self.ledger
3485 .iter()
3486 .filter(|(id, _)| self.input_is_non_terminal_by_authority(id))
3487 .map(|(id, _)| id.clone())
3488 .collect()
3489 }
3490}
3491
3492#[cfg(test)]
3493mod tests {
3494 use super::{AdmissionValidationFacts, EphemeralRuntimeDriver};
3495 use crate::identifiers::{IdempotencyKey, LogicalRuntimeId, SupersessionKey};
3496 use crate::input::{
3497 Input, InputDurability, InputHeader, InputOrigin, InputVisibility, OperationInput,
3498 PeerConvention, PeerInput, PromptInput,
3499 };
3500 use crate::input_state::{
3501 InputAbandonReason, InputLifecycleState, InputStateSeed, InputTerminalOutcome,
3502 };
3503 use crate::meerkat_machine::dsl as mm_dsl;
3504 use crate::traits::{RuntimeDriver, RuntimeDriverError};
3505 use crate::{RuntimeState, WakeMode};
3506 use chrono::Utc;
3507 use meerkat_core::lifecycle::{InputId, RunId};
3508 use meerkat_core::ops::{OpEvent, OperationId};
3509
3510 fn peer_message_input() -> Input {
3511 Input::Peer(PeerInput {
3512 injected_context: Vec::new(),
3513 sender_taint: None,
3514 header: InputHeader {
3515 id: InputId::new(),
3516 timestamp: Utc::now(),
3517 source: InputOrigin::Peer {
3518 peer_id: "peer-1".into(),
3519 display_identity: None,
3520 runtime_id: None,
3521 },
3522 durability: InputDurability::Durable,
3523 visibility: InputVisibility::default(),
3524 idempotency_key: None,
3525 supersession_key: None,
3526 correlation_id: None,
3527 },
3528 convention: Some(PeerConvention::Message),
3529 content: "peer body".into(),
3530 payload: None,
3531 handling_mode: None,
3532 })
3533 }
3534
3535 fn prompt_input(text: &str) -> Input {
3536 Input::Prompt(PromptInput::new(text, None))
3537 }
3538
3539 fn operation_input() -> Input {
3540 let operation_id = OperationId::new();
3541 Input::Operation(OperationInput {
3542 header: InputHeader {
3543 id: InputId::new(),
3544 timestamp: Utc::now(),
3545 source: InputOrigin::System,
3546 durability: InputDurability::Derived,
3547 visibility: InputVisibility::default(),
3548 idempotency_key: None,
3549 supersession_key: None,
3550 correlation_id: None,
3551 },
3552 operation_id: operation_id.clone(),
3553 event: OpEvent::Cancelled { id: operation_id },
3554 })
3555 }
3556
3557 fn progress_input_with_supersession(label: &str, supersession_key: &str) -> Input {
3558 Input::Peer(PeerInput {
3559 injected_context: Vec::new(),
3560 sender_taint: None,
3561 header: InputHeader {
3562 id: InputId::new(),
3563 timestamp: Utc::now(),
3564 source: InputOrigin::Peer {
3565 peer_id: "peer-1".into(),
3566 display_identity: None,
3567 runtime_id: None,
3568 },
3569 durability: InputDurability::Durable,
3570 visibility: InputVisibility::default(),
3571 idempotency_key: None,
3572 supersession_key: Some(SupersessionKey::new(supersession_key)),
3573 correlation_id: None,
3574 },
3575 convention: Some(PeerConvention::ResponseProgress {
3576 request_id: format!("request-{label}"),
3577 phase: crate::input::ResponseProgressPhase::InProgress,
3578 }),
3579 content: format!("progress {label}").into(),
3580 payload: None,
3581 handling_mode: None,
3582 })
3583 }
3584
3585 fn force_control_shadow(
3586 driver: &mut EphemeralRuntimeDriver,
3587 phase: RuntimeState,
3588 current_run_id: Option<RunId>,
3589 pre_run_phase: Option<RuntimeState>,
3590 ) {
3591 let mut control = driver.write_control_projection();
3592 control.phase = phase;
3593 control.current_run_id = current_run_id;
3594 control.pre_run_phase = pre_run_phase;
3595 }
3596
3597 #[test]
3598 fn set_control_projection_does_not_write_dsl_authority() {
3599 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("projection-only"));
3600 let run_id = RunId::new();
3601
3602 driver.set_control_projection(
3603 RuntimeState::Running,
3604 Some(run_id),
3605 Some(RuntimeState::Attached),
3606 );
3607
3608 assert_eq!(
3609 driver.runtime_phase_snapshot(),
3610 RuntimeState::Idle,
3611 "control projection writes must not mutate DSL lifecycle truth",
3612 );
3613 assert_eq!(
3614 driver.current_run_id(),
3615 None,
3616 "control projection writes must not mutate DSL run binding truth",
3617 );
3618 assert_eq!(
3619 driver.control_snapshot().phase,
3620 RuntimeState::Running,
3621 "the shell projection still records the mechanical projection",
3622 );
3623 }
3624
3625 #[tokio::test]
3626 async fn direct_accept_uses_dsl_phase_not_control_projection_shadow() {
3627 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("admission-shadow"));
3628 force_control_shadow(&mut driver, RuntimeState::Stopped, None, None);
3629
3630 let outcome = driver.accept_input(peer_message_input()).await.unwrap();
3631
3632 assert!(
3633 outcome.is_accepted(),
3634 "direct RuntimeDriver admission should follow DSL phase, not a stale control shadow",
3635 );
3636 }
3637
3638 #[tokio::test]
3639 async fn priority_enqueue_order_is_assigned_by_machine() {
3640 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("priority-order"));
3641
3642 let normal_a = prompt_input("normal-a");
3643 let normal_a_id = normal_a.id().clone();
3644 driver.accept_input(normal_a).await.unwrap();
3645
3646 let normal_b = prompt_input("normal-b");
3647 let normal_b_id = normal_b.id().clone();
3648 driver.accept_input(normal_b).await.unwrap();
3649
3650 let priority = prompt_input("priority");
3651 let priority_id = priority.id().clone();
3652 driver.accept_input(priority).await.unwrap();
3653 driver
3654 .dsl_apply(
3655 mm_dsl::MeerkatMachineInput::PrioritizeInput {
3656 input_id: priority_id.to_string(),
3657 },
3658 "PrioritizeInput(test)",
3659 )
3660 .unwrap();
3661 driver.rebuild_queue_projections();
3662
3663 assert_eq!(
3664 driver.dsl_queue_lane(),
3665 vec![
3666 priority_id.clone(),
3667 normal_a_id.clone(),
3668 normal_b_id.clone()
3669 ]
3670 );
3671 let (priority_seq, normal_a_seq) = driver.with_dsl_state(|state| {
3672 (
3673 state.input_admission_seq[&priority_id.to_string()],
3674 state.input_admission_seq[&normal_a_id.to_string()],
3675 )
3676 });
3677 assert!(
3678 priority_seq < normal_a_seq,
3679 "priority order must be represented by generated admission sequence"
3680 );
3681 }
3682
3683 #[tokio::test]
3684 async fn backlog_deferral_order_is_assigned_by_machine() {
3685 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("backlog-deferral"));
3686
3687 let first = prompt_input("first");
3688 let first_id = first.id().clone();
3689 driver.accept_input(first).await.unwrap();
3690
3691 let second = prompt_input("second");
3692 let second_id = second.id().clone();
3693 driver.accept_input(second).await.unwrap();
3694
3695 driver
3696 .defer_queued_inputs_behind_backlog(std::slice::from_ref(&first_id))
3697 .unwrap();
3698
3699 assert_eq!(
3700 driver.dsl_queue_lane(),
3701 vec![second_id.clone(), first_id.clone()]
3702 );
3703 let (first_seq, second_seq) = driver.with_dsl_state(|state| {
3704 (
3705 state.input_admission_seq[&first_id.to_string()],
3706 state.input_admission_seq[&second_id.to_string()],
3707 )
3708 });
3709 assert!(
3710 first_seq > second_seq,
3711 "deferred order must be represented by generated admission sequence"
3712 );
3713 }
3714
3715 #[tokio::test]
3716 async fn authorized_batch_dequeue_requires_exact_source() {
3717 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("batch-source"));
3718 let input = prompt_input("queued");
3719 let input_id = input.id().clone();
3720 driver.accept_input(input).await.unwrap();
3721
3722 let batch = crate::meerkat_machine::driver::test_authorized_runtime_loop_batch_from_source(
3723 vec![input_id.clone()],
3724 crate::meerkat_machine::driver::RuntimeLoopBatchSource::Steer,
3725 );
3726
3727 let err = driver
3728 .dequeue_batch_exact(&batch)
3729 .expect_err("queue input must not be drained through steer authority");
3730
3731 assert!(
3732 err.to_string()
3733 .contains("authorized runtime batch from steer did not match"),
3734 "unexpected error: {err}"
3735 );
3736 assert_eq!(
3737 driver.queue().input_ids(),
3738 vec![input_id],
3739 "failed source conformance must leave the physical queue intact"
3740 );
3741 }
3742
3743 #[tokio::test]
3744 async fn authorized_batch_dequeue_requires_exact_prefix() {
3745 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("batch-prefix"));
3746 let first = prompt_input("first");
3747 let first_id = first.id().clone();
3748 driver.accept_input(first).await.unwrap();
3749 let second = prompt_input("second");
3750 let second_id = second.id().clone();
3751 driver.accept_input(second).await.unwrap();
3752
3753 let batch =
3754 crate::meerkat_machine::driver::test_authorized_runtime_loop_batch(vec![second_id]);
3755
3756 let err = driver
3757 .dequeue_batch_exact(&batch)
3758 .expect_err("later queue member must not be drained past an older prefix");
3759
3760 assert!(
3761 err.to_string()
3762 .contains("authorized runtime batch from queue did not match"),
3763 "unexpected error: {err}"
3764 );
3765 assert_eq!(
3766 driver.queue().input_ids(),
3767 vec![first_id, batch.input_ids()[0].clone()],
3768 "failed prefix conformance must leave the physical queue intact"
3769 );
3770 }
3771
3772 #[tokio::test]
3773 async fn coalesce_input_requires_generated_existing_target_authority() {
3774 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("coalesce-guard"));
3775
3776 let first = prompt_input("first");
3777 let first_id = first.id().clone();
3778 driver.accept_input(first).await.unwrap();
3779
3780 let err = driver
3781 .dsl_apply(
3782 mm_dsl::MeerkatMachineInput::CoalesceInput {
3783 input_id: first_id.to_string(),
3784 aggregate_id: InputId::new().to_string(),
3785 },
3786 "CoalesceInput",
3787 )
3788 .unwrap_err();
3789
3790 assert!(
3791 matches!(err, RuntimeDriverError::Internal(message) if message.contains("CoalesceInput")),
3792 "unauthorized coalesce should fail closed through generated guards"
3793 );
3794 }
3795
3796 #[tokio::test]
3797 async fn progress_coalesce_target_is_supplied_by_generated_admission_authority() {
3798 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("coalesce-authority"));
3799
3800 let first = progress_input_with_supersession("first", "same-window");
3801 let first_id = first.id().clone();
3802 driver.accept_input(first).await.unwrap();
3803
3804 let second = progress_input_with_supersession("second", "same-window");
3805 let second_id = second.id().clone();
3806 driver.accept_input(second).await.unwrap();
3807
3808 assert_eq!(
3809 driver.input_phase(&first_id),
3810 Some(crate::input_state::InputLifecycleState::Coalesced)
3811 );
3812 assert_eq!(
3813 driver.input_terminal_outcome(&first_id),
3814 Some(crate::input_state::InputTerminalOutcome::Coalesced {
3815 aggregate_id: second_id.clone()
3816 })
3817 );
3818 assert!(!driver.has_queued_input(&first_id));
3819 assert!(driver.has_queued_input(&second_id));
3820 driver.with_dsl_state(|state| {
3821 assert!(
3822 !state
3823 .admission_authorized_existing_actions
3824 .contains_key(&second_id.to_string())
3825 );
3826 assert!(
3827 !state
3828 .admission_authorized_existing_targets
3829 .contains_key(&second_id.to_string())
3830 );
3831 });
3832 }
3833
3834 #[tokio::test]
3835 async fn idempotency_dedup_is_resolved_by_generated_machine_map() {
3836 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("dedup-authority"));
3837 let key = IdempotencyKey::new("machine-owned-dedup");
3838
3839 let mut first = prompt_input("first");
3840 let first_id = first.id().clone();
3841 if let Input::Prompt(prompt) = &mut first {
3842 prompt.header.idempotency_key = Some(key.clone());
3843 }
3844 let first_outcome = driver.accept_input(first).await.unwrap();
3845 assert!(first_outcome.is_accepted());
3846
3847 driver.with_dsl_state(|state| {
3848 assert_eq!(
3849 state.admission_idempotency_inputs.get(&key.to_string()),
3850 Some(&first_id.to_string()),
3851 "generated machine state must own the idempotency key binding"
3852 );
3853 });
3854
3855 let mut duplicate = prompt_input("second");
3856 let duplicate_id = duplicate.id().clone();
3857 if let Input::Prompt(prompt) = &mut duplicate {
3858 prompt.header.idempotency_key = Some(key.clone());
3859 }
3860 let duplicate_outcome = driver.accept_input(duplicate).await.unwrap();
3861
3862 match duplicate_outcome {
3863 crate::accept::AcceptOutcome::Deduplicated {
3864 input_id,
3865 existing_id,
3866 } => {
3867 assert_eq!(input_id, duplicate_id);
3868 assert_eq!(existing_id, first_id);
3869 }
3870 other => panic!("expected generated deduplicated outcome, got {other:?}"),
3871 }
3872 assert!(
3873 driver.input_state(&duplicate_id).is_none(),
3874 "deduplicated inputs must not be admitted into the shell ledger"
3875 );
3876 driver.with_dsl_state(|state| {
3877 assert_eq!(
3878 state.admission_idempotency_inputs.get(&key.to_string()),
3879 Some(&first_id.to_string()),
3880 "duplicate resolution must not rewrite the generated key owner"
3881 );
3882 });
3883 }
3884
3885 #[tokio::test]
3886 async fn admission_validation_rejection_class_is_generated() {
3887 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("validation-authority"));
3888
3889 let mut input = prompt_input("derived prompt");
3890 let input_id = input.id().clone();
3891 if let Input::Prompt(prompt) = &mut input {
3892 prompt.header.durability = InputDurability::Derived;
3893 }
3894
3895 let generated_reason = driver
3896 .resolve_admission_validation(
3897 &input_id,
3898 AdmissionValidationFacts {
3899 input_kind: input.kind(),
3900 input_origin: &input.header().source,
3901 durability: input.header().durability,
3902 peer_handling_mode_valid: true,
3903 peer_response_terminal_structurally_valid: true,
3904 peer_response_terminal_observed_status:
3905 mm_dsl::PeerResponseTerminalObservedStatus::NotPeerTerminal,
3906 },
3907 )
3908 .expect("generated validation feedback should resolve");
3909 assert_eq!(
3910 generated_reason,
3911 Some(mm_dsl::AdmissionRejectReasonKind::ExternalDerivedDurabilityForbidden),
3912 "derived operator prompt must reject on the external-derived rule"
3913 );
3914
3915 let outcome = driver.accept_input(input).await.unwrap();
3916 match outcome {
3917 crate::accept::AcceptOutcome::Rejected {
3918 reason: crate::accept::RejectReason::DurabilityViolation { detail },
3919 } => {
3920 assert!(
3921 !detail.is_empty(),
3922 "shell detail should describe the raw validation error"
3923 );
3924 }
3925 other => panic!("expected generated rejection class, got {other:?}"),
3926 }
3927 assert!(
3928 driver.input_state(&input_id).is_none(),
3929 "rejected inputs must not enter the shell ledger"
3930 );
3931 driver.with_dsl_state(|state| {
3932 assert!(
3933 !state.input_phases.contains_key(&input_id.to_string()),
3934 "rejected inputs must not create lifecycle machine facts"
3935 );
3936 });
3937 }
3938
3939 #[test]
3940 fn admission_validation_durability_reasons_are_machine_emitted() {
3941 use crate::identifiers::InputKind;
3942 use mm_dsl::AdmissionRejectReasonKind as Reason;
3943
3944 let driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("validation-reasons"));
3945
3946 let operator = InputOrigin::Operator;
3947 let system = InputOrigin::System;
3948 let flow = InputOrigin::Flow {
3949 flow_id: "flow-1".into(),
3950 step_index: 0,
3951 };
3952 let peer = InputOrigin::Peer {
3953 peer_id: "peer-1".into(),
3954 display_identity: None,
3955 runtime_id: None,
3956 };
3957 let external = InputOrigin::External {
3958 source_name: "webhook".into(),
3959 };
3960
3961 let cases: Vec<(InputKind, &InputOrigin, InputDurability, Option<Reason>)> = vec![
3962 (
3964 InputKind::Prompt,
3965 &operator,
3966 InputDurability::Derived,
3967 Some(Reason::ExternalDerivedDurabilityForbidden),
3968 ),
3969 (
3970 InputKind::PeerMessage,
3971 &peer,
3972 InputDurability::Derived,
3973 Some(Reason::ExternalDerivedDurabilityForbidden),
3974 ),
3975 (
3976 InputKind::ExternalEvent,
3977 &external,
3978 InputDurability::Derived,
3979 Some(Reason::ExternalDerivedDurabilityForbidden),
3980 ),
3981 (
3982 InputKind::Continuation,
3983 &operator,
3984 InputDurability::Derived,
3985 Some(Reason::ExternalDerivedDurabilityForbidden),
3986 ),
3987 (
3989 InputKind::Prompt,
3990 &system,
3991 InputDurability::Derived,
3992 Some(Reason::DerivedDurabilityForbiddenForInputKind),
3993 ),
3994 (
3995 InputKind::PeerMessage,
3996 &system,
3997 InputDurability::Derived,
3998 Some(Reason::DerivedDurabilityForbiddenForInputKind),
3999 ),
4000 (
4001 InputKind::PeerRequest,
4002 &system,
4003 InputDurability::Derived,
4004 Some(Reason::DerivedDurabilityForbiddenForInputKind),
4005 ),
4006 (
4007 InputKind::PeerResponseTerminal,
4008 &system,
4009 InputDurability::Derived,
4010 Some(Reason::DerivedDurabilityForbiddenForInputKind),
4011 ),
4012 (
4013 InputKind::FlowStep,
4014 &flow,
4015 InputDurability::Derived,
4016 Some(Reason::DerivedDurabilityForbiddenForInputKind),
4017 ),
4018 (
4020 InputKind::PeerResponseProgress,
4021 &system,
4022 InputDurability::Derived,
4023 None,
4024 ),
4025 (
4026 InputKind::ExternalEvent,
4027 &system,
4028 InputDurability::Derived,
4029 None,
4030 ),
4031 (
4032 InputKind::Operation,
4033 &system,
4034 InputDurability::Derived,
4035 None,
4036 ),
4037 (InputKind::Prompt, &operator, InputDurability::Durable, None),
4039 (
4040 InputKind::Prompt,
4041 &operator,
4042 InputDurability::Ephemeral,
4043 None,
4044 ),
4045 ];
4046
4047 for (input_kind, input_origin, durability, expected) in cases {
4048 let input_id = InputId::new();
4049 let resolved = driver
4050 .resolve_admission_validation(
4051 &input_id,
4052 AdmissionValidationFacts {
4053 input_kind,
4054 input_origin,
4055 durability,
4056 peer_handling_mode_valid: true,
4057 peer_response_terminal_structurally_valid: true,
4058 peer_response_terminal_observed_status:
4059 mm_dsl::PeerResponseTerminalObservedStatus::NotPeerTerminal,
4060 },
4061 )
4062 .expect("generated validation must resolve");
4063 assert_eq!(
4064 resolved, expected,
4065 "machine-emitted reason mismatch for {input_kind:?}/{input_origin:?}/{durability:?}"
4066 );
4067 }
4068 }
4069
4070 #[tokio::test]
4071 async fn consume_on_accept_terminal_outcome_is_machine_owned() {
4072 let mut driver =
4073 EphemeralRuntimeDriver::new(LogicalRuntimeId::new("consume-on-accept-terminal"));
4074
4075 let input = operation_input();
4076 let input_id = input.id().clone();
4077 let outcome = driver.accept_input(input).await.unwrap();
4078
4079 match outcome {
4080 crate::accept::AcceptOutcome::Accepted { seed, .. } => {
4081 assert_eq!(seed.phase, InputLifecycleState::Consumed);
4082 assert_eq!(
4083 seed.terminal_outcome,
4084 Some(InputTerminalOutcome::Consumed),
4085 "accepted result must project terminal outcome from generated machine state"
4086 );
4087 }
4088 other => panic!("expected consume-on-accept accepted outcome, got {other:?}"),
4089 }
4090 assert_eq!(
4091 driver.input_terminal_outcome(&input_id),
4092 Some(InputTerminalOutcome::Consumed)
4093 );
4094 driver.with_dsl_state(|state| {
4095 assert_eq!(
4096 state.input_terminal_kind.get(&input_id.to_string()),
4097 Some(&mm_dsl::InputTerminalKind::Consumed),
4098 "ConsumeOnAccept must write generated terminal kind"
4099 );
4100 });
4101 }
4102
4103 #[tokio::test]
4104 async fn staged_rollback_retry_exhaustion_is_machine_owned() {
4105 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("rollback-resolution"));
4106
4107 let input = prompt_input("retry me");
4108 let input_id = input.id().clone();
4109 driver.accept_input(input).await.unwrap();
4110
4111 let run_id = RunId::new();
4112 driver
4113 .contract_begin_run_authority(run_id.clone())
4114 .expect("runtime run authority should begin through generated DSL");
4115
4116 for attempt in 1..3 {
4117 driver
4118 .machine_realize_stage_batch(std::slice::from_ref(&input_id), &run_id)
4119 .unwrap();
4120 assert_eq!(driver.input_attempt_count(&input_id), attempt);
4121
4122 driver
4123 .rollback_staged(std::slice::from_ref(&input_id))
4124 .unwrap();
4125 assert_eq!(
4126 driver.input_phase(&input_id),
4127 Some(InputLifecycleState::Queued),
4128 "machine should requeue while generated retry policy has attempts remaining"
4129 );
4130 assert_eq!(driver.input_terminal_outcome(&input_id), None);
4131 }
4132
4133 driver
4134 .machine_realize_stage_batch(std::slice::from_ref(&input_id), &run_id)
4135 .unwrap();
4136 assert_eq!(driver.input_attempt_count(&input_id), 3);
4137 driver
4138 .rollback_staged(std::slice::from_ref(&input_id))
4139 .unwrap();
4140
4141 assert_eq!(
4142 driver.input_phase(&input_id),
4143 Some(InputLifecycleState::Abandoned)
4144 );
4145 assert_eq!(
4146 driver.input_terminal_outcome(&input_id),
4147 Some(InputTerminalOutcome::Abandoned {
4148 reason: InputAbandonReason::MaxAttemptsExhausted { attempts: 3 },
4149 })
4150 );
4151 assert!(!driver.has_queued_input(&input_id));
4152 }
4153
4154 #[tokio::test]
4155 async fn stage_input_keeps_queue_projection_aligned() {
4156 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("stage-projection"));
4157 let input = prompt_input("stage me");
4158 let input_id = input.id().clone();
4159 driver.accept_input(input).await.unwrap();
4160
4161 let run_id = RunId::new();
4162 driver.contract_begin_run_authority(run_id.clone()).unwrap();
4163 driver
4164 .machine_realize_stage_batch(std::slice::from_ref(&input_id), &run_id)
4165 .unwrap();
4166
4167 assert!(driver.queue().is_empty());
4168 driver
4169 .validate_queue_projection_alignment("test post-stage")
4170 .unwrap();
4171 }
4172
4173 #[tokio::test]
4174 async fn authorized_dequeue_rejects_physical_queue_projection_drift() {
4175 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new(
4176 "queue-projection-drift-before-dequeue",
4177 ));
4178 let input = prompt_input("authorized");
4179 let input_id = input.id().clone();
4180 driver.accept_input(input).await.unwrap();
4181
4182 let drift = prompt_input("drift");
4183 let drift_id = drift.id().clone();
4184 driver.queue_mut().enqueue_front(drift_id, drift);
4185 let batch =
4186 crate::meerkat_machine::driver::test_authorized_runtime_loop_batch(vec![input_id]);
4187
4188 let err = driver
4189 .dequeue_batch_exact(&batch)
4190 .expect_err("physical queue drift must fail closed before dequeue");
4191 assert!(
4192 matches!(&err, RuntimeDriverError::Internal(message) if message.contains("physical queue projection diverged")),
4193 "unexpected queue projection error: {err:?}"
4194 );
4195 }
4196
4197 #[tokio::test]
4198 async fn authorized_dequeue_rejects_physical_steer_projection_drift() {
4199 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new(
4200 "steer-projection-drift-before-dequeue",
4201 ));
4202 let input = prompt_input("authorized");
4203 let input_id = input.id().clone();
4204 driver.accept_input(input).await.unwrap();
4205
4206 let drift = prompt_input("steer drift");
4207 let drift_id = drift.id().clone();
4208 driver.steer_queue_mut().enqueue(drift_id, drift);
4209 let batch =
4210 crate::meerkat_machine::driver::test_authorized_runtime_loop_batch(vec![input_id]);
4211
4212 let err = driver
4213 .dequeue_batch_exact(&batch)
4214 .expect_err("physical steer queue drift must fail closed before dequeue");
4215 assert!(
4216 matches!(&err, RuntimeDriverError::Internal(message) if message.contains("physical steer queue projection diverged")),
4217 "unexpected steer projection error: {err:?}"
4218 );
4219 }
4220
4221 #[tokio::test]
4222 async fn recovery_applied_stays_applied() {
4223 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("recovery-applied"));
4224
4225 let input = prompt_input("hello");
4226 let input_id = input.id().clone();
4227 driver.accept_input(input).await.unwrap();
4228
4229 let run_id = RunId::new();
4230 driver.contract_begin_run_authority(run_id.clone()).unwrap();
4231 driver
4232 .machine_realize_stage_batch(std::slice::from_ref(&input_id), &run_id)
4233 .unwrap();
4234 driver.apply_input(&input_id, &run_id).unwrap();
4235
4236 let report = driver.recover_ephemeral().unwrap();
4237 assert_eq!(report.inputs_recovered, 1);
4238 assert_eq!(
4239 driver.input_phase(&input_id),
4240 Some(InputLifecycleState::AppliedPendingConsumption)
4241 );
4242 driver
4243 .validate_queue_projection_alignment("test recovery")
4244 .unwrap();
4245 }
4246
4247 #[tokio::test]
4248 async fn missing_input_lifecycle_authority_fails_terminality_closed() {
4249 let mut driver =
4250 EphemeralRuntimeDriver::new(LogicalRuntimeId::new("missing-input-authority"));
4251
4252 let input = prompt_input("missing authority");
4253 let input_id = input.id().clone();
4254 driver.accept_input(input).await.unwrap();
4255 {
4256 let mut authority = driver.dsl.lock();
4257 let mut state = authority.state().clone();
4258 state.input_phases.remove(&input_id.to_string());
4259 *authority = super::recover_ingress_dsl_authority(state);
4260 }
4261
4262 let err = driver
4263 .input_is_terminal_by_authority(&input_id)
4264 .expect_err("missing machine lifecycle authority must fail closed");
4265 assert!(
4266 matches!(&err, RuntimeDriverError::Internal(message) if message.contains("missing generated input lifecycle authority")),
4267 "unexpected terminality error: {err:?}"
4268 );
4269 assert!(
4270 driver.active_input_ids().is_empty(),
4271 "active-input projection must not fabricate non-terminal truth without generated authority"
4272 );
4273
4274 let err = driver
4275 .machine_realize_stage_batch(std::slice::from_ref(&input_id), &RunId::new())
4276 .expect_err("staging must not synthesize a queued phase without machine authority");
4277 assert!(
4278 matches!(&err, RuntimeDriverError::Internal(message) if message.contains("generated input lifecycle phase missing before staging")),
4279 "unexpected staging error: {err:?}"
4280 );
4281 }
4282
4283 #[tokio::test]
4284 async fn cancelled_peer_response_terminal_rejects_and_cleans_pending_via_machine() {
4285 let mut driver =
4286 EphemeralRuntimeDriver::new(LogicalRuntimeId::new("cancelled-peer-terminal"));
4287 let peer_id = "550e8400-e29b-41d4-a716-446655440000";
4288 let request_uuid = uuid::Uuid::parse_str("018f6f79-7a82-7c4e-a552-a3b86f9630f1").unwrap();
4289 let request_id = meerkat_core::PeerCorrelationId::from_uuid(request_uuid);
4290 driver
4291 .dsl_apply(
4292 mm_dsl::MeerkatMachineInput::PeerRequestSent {
4293 corr_id: request_id.into(),
4294 },
4295 "PeerRequestSent(test)",
4296 )
4297 .unwrap();
4298
4299 let input = Input::Peer(PeerInput {
4300 injected_context: Vec::new(),
4301 sender_taint: None,
4302 header: InputHeader {
4303 id: InputId::new(),
4304 timestamp: Utc::now(),
4305 source: InputOrigin::Peer {
4306 peer_id: peer_id.into(),
4307 display_identity: Some("Analyst".into()),
4308 runtime_id: None,
4309 },
4310 durability: InputDurability::Durable,
4311 visibility: InputVisibility::default(),
4312 idempotency_key: None,
4313 supersession_key: None,
4314 correlation_id: None,
4315 },
4316 convention: Some(PeerConvention::ResponseTerminal {
4317 request_id: request_uuid.to_string(),
4318 status: meerkat_core::handles::PeerResponseTerminalProjectionStatus::Cancelled,
4319 }),
4320 content: meerkat_core::types::ContentInput::Text(String::new()),
4321 payload: Some(serde_json::json!({"ok": false})),
4322 handling_mode: None,
4323 });
4324
4325 let outcome = driver.accept_input(input).await.unwrap();
4326 match outcome {
4327 crate::accept::AcceptOutcome::Rejected {
4328 reason: crate::accept::RejectReason::PeerResponseTerminalInvalid { detail },
4329 } => assert!(detail.contains("rejected by generated authority")),
4330 other => panic!("expected generated peer terminal rejection, got {other:?}"),
4331 }
4332 driver.with_dsl_state(|state| {
4333 assert!(
4334 !state
4335 .pending_peer_requests
4336 .contains_key(&mm_dsl::PeerCorrelationId::from(request_uuid)),
4337 "invalid terminal observation must clean pending peer truth through generated authority"
4338 );
4339 });
4340 }
4341
4342 #[tokio::test]
4343 async fn abandon_all_non_terminal_projects_generated_terminal_outcome() {
4344 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("abandon-projection"));
4345 let input = prompt_input("abandon me");
4346 let input_id = input.id().clone();
4347 driver.accept_input(input).await.unwrap();
4348
4349 let abandoned = driver
4350 .abandon_all_non_terminal(InputAbandonReason::Stopped)
4351 .unwrap();
4352
4353 assert_eq!(abandoned, 1);
4354 assert_eq!(
4355 driver.input_terminal_outcome(&input_id),
4356 Some(InputTerminalOutcome::Abandoned {
4357 reason: InputAbandonReason::Stopped
4358 }),
4359 "generated machine projection is the only terminal-outcome owner"
4360 );
4361 }
4362
4363 #[test]
4364 fn recovered_terminal_lifecycle_requires_terminal_witness() {
4365 let mut driver =
4366 EphemeralRuntimeDriver::new(LogicalRuntimeId::new("recover-terminal-witness"));
4367 let input_id = InputId::new();
4368 let seed = InputStateSeed {
4369 phase: InputLifecycleState::Consumed,
4370 last_run_id: None,
4371 last_boundary_sequence: None,
4372 admission_sequence: None,
4373 terminal_outcome: None,
4374 attempt_count: 0,
4375 recovery_lane: None,
4376 };
4377
4378 let err = driver
4379 .recover_terminal_input_lifecycle(&input_id, &seed, None)
4380 .expect_err("terminal recovery without terminal outcome must fail closed");
4381 assert!(
4382 matches!(&err, RuntimeDriverError::Internal(message) if message.contains("behavioral input terminality")),
4383 "unexpected recovery error: {err:?}"
4384 );
4385
4386 let generated_err = driver
4387 .dsl_apply(
4388 mm_dsl::MeerkatMachineInput::RecoverInputLifecycle {
4389 input_id: input_id.to_string(),
4390 phase: mm_dsl::InputPhase::Consumed,
4391 terminal_kind: None,
4392 superseded_by: None,
4393 aggregate_id: None,
4394 abandon_reason: None,
4395 abandon_attempt_count: 0,
4396 attempt_count: 0,
4397 run_id: None,
4398 boundary_sequence: None,
4399 admission_sequence: None,
4400 admission_sequence_recovery: None,
4401 recovery_lane: None,
4402 lane: None,
4403 runtime_boundary: None,
4404 runtime_execution_kind: None,
4405 runtime_peer_response_terminal_apply_intent: None,
4406 is_prompt: false,
4407 },
4408 "RecoverInputLifecycle(test)",
4409 )
4410 .expect_err("generated recovery authority must reject missing terminal kind");
4411 assert!(
4412 matches!(&generated_err, RuntimeDriverError::Internal(message) if message.contains("RecoverInputLifecycle")),
4413 "unexpected generated recovery error: {generated_err:?}"
4414 );
4415 }
4416
4417 #[test]
4418 fn recovered_max_attempts_terminal_reason_owns_attempt_payload() {
4419 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("recover-max-attempts"));
4420 let input_id = InputId::new();
4421 let split_seed = InputStateSeed {
4422 phase: InputLifecycleState::Abandoned,
4423 last_run_id: None,
4424 last_boundary_sequence: None,
4425 admission_sequence: None,
4426 terminal_outcome: Some(InputTerminalOutcome::Abandoned {
4427 reason: InputAbandonReason::MaxAttemptsExhausted { attempts: 3 },
4428 }),
4429 attempt_count: 2,
4430 recovery_lane: None,
4431 };
4432
4433 let err = driver
4434 .recover_terminal_input_lifecycle(&input_id, &split_seed, None)
4435 .expect_err("max-attempts recovery must reject a split attempt witness");
4436 assert!(
4437 matches!(&err, RuntimeDriverError::Internal(message) if message.contains("RecoverInputLifecycle")),
4438 "unexpected recovery error: {err:?}"
4439 );
4440
4441 let below_policy_seed = InputStateSeed {
4442 phase: InputLifecycleState::Abandoned,
4443 last_run_id: None,
4444 last_boundary_sequence: None,
4445 admission_sequence: None,
4446 terminal_outcome: Some(InputTerminalOutcome::Abandoned {
4447 reason: InputAbandonReason::MaxAttemptsExhausted { attempts: 2 },
4448 }),
4449 attempt_count: 2,
4450 recovery_lane: None,
4451 };
4452
4453 let err = driver
4454 .recover_terminal_input_lifecycle(&input_id, &below_policy_seed, None)
4455 .expect_err("max-attempts recovery must reject attempts below machine policy");
4456 assert!(
4457 matches!(&err, RuntimeDriverError::Internal(message) if message.contains("RecoverInputLifecycle")),
4458 "unexpected recovery policy error: {err:?}"
4459 );
4460
4461 let err = crate::meerkat_machine::authorize_stored_input_state_seed(
4462 &input_id,
4463 &below_policy_seed,
4464 )
4465 .expect_err("stored max-attempts seed must reject attempts below machine policy");
4466 assert!(
4467 err.contains("stored input-state seed"),
4468 "unexpected stored seed policy error: {err:?}"
4469 );
4470 }
4471
4472 #[test]
4473 fn resolve_admission_uses_generated_machine_phase_not_control_projection() {
4474 let mut driver = EphemeralRuntimeDriver::new(LogicalRuntimeId::new("phase-drift"));
4475 force_control_shadow(
4476 &mut driver,
4477 RuntimeState::Running,
4478 Some(RunId::new()),
4479 Some(RuntimeState::Attached),
4480 );
4481
4482 let input = peer_message_input();
4483 let projected = driver.resolve_admission(&input).unwrap();
4484 assert!(projected.requires_active_runtime_pre_admission());
4485 let flags = projected.coarse_flags();
4486 assert_eq!(projected.policy().wake_mode, WakeMode::WakeIfIdle);
4487 assert!(!flags.interrupt_yielding);
4488 assert!(!flags.request_immediate_processing);
4489 }
4490}