1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24 CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31 CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32 OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33 OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34 OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35 OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36 WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43fn deserialize_required_operation_source<'de, D>(
54 deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57 D: serde::Deserializer<'de>,
58{
59 <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65 status: OperationStatus,
66 kind: OperationKind,
67 #[serde(deserialize_with = "deserialize_required_operation_source")]
68 operation_source: Option<OperationSource>,
69 peer_ready: bool,
70 progress_count: u32,
71 watcher_count: u32,
72 terminal_outcome: Option<OperationTerminalOutcome>,
73 #[serde(default, skip_serializing_if = "Option::is_none")]
74 completion_sequence: Option<CompletionSeq>,
75 terminal_buffered: bool,
76}
77
78#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81 seq: CompletionSeq,
82 kind: OperationKind,
83 terminal_outcome: OperationTerminalOutcome,
84}
85
86#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89 operations: HashMap<OperationId, OperationCanonicalState>,
90 completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91 completed_order: VecDeque<OperationId>,
92 max_completed: usize,
93 max_concurrent: Option<usize>,
94 active_count: usize,
95 wait_request_id: Option<WaitRequestId>,
96 wait_operation_ids: Vec<OperationId>,
97 next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101 pub fn max_completed(&self) -> usize {
103 self.max_completed
104 }
105
106 pub fn max_concurrent(&self) -> Option<usize> {
108 self.max_concurrent
109 }
110
111 pub fn operation_count(&self) -> usize {
113 self.operations.len()
114 }
115
116 pub fn completion_feed_count(&self) -> usize {
118 self.completion_feed_entries.len()
119 }
120}
121
122#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134 pub epoch_id: meerkat_core::RuntimeEpochId,
136 pub authority_state: RegistryCanonicalState,
138 pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140 pub completion_entries: Vec<CompletionEntry>,
143 pub cursors: meerkat_core::EpochCursorSnapshot,
145}
146
147#[derive(Debug)]
148pub struct OpsLifecyclePersistenceRequest {
149 snapshot: PersistedOpsSnapshot,
150 result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
151}
152
153impl OpsLifecyclePersistenceRequest {
154 pub fn snapshot(&self) -> &PersistedOpsSnapshot {
155 &self.snapshot
156 }
157
158 pub fn complete(self, result: Result<(), OpsLifecycleError>) {
159 let _ = self.result_tx.send(result);
160 }
161}
162
163#[derive(Debug)]
172struct FeedBufferInner {
173 entries: VecDeque<CompletionEntry>,
174 watermark: CompletionSeq,
175 max_retained: usize,
176}
177
178#[derive(Debug)]
183struct FeedBuffer {
184 inner: RwLock<FeedBufferInner>,
185 watermark_atomic: AtomicU64,
187 notify: tokio::sync::Notify,
189}
190
191impl FeedBuffer {
192 fn new(max_retained: usize) -> Self {
193 Self {
194 inner: RwLock::new(FeedBufferInner {
195 entries: VecDeque::new(),
196 watermark: 0,
197 max_retained,
198 }),
199 watermark_atomic: AtomicU64::new(0),
200 notify: tokio::sync::Notify::new(),
201 }
202 }
203
204 fn push(&self, entry: CompletionEntry) {
205 let mut inner = self
206 .inner
207 .write()
208 .unwrap_or_else(std::sync::PoisonError::into_inner);
209 let seq = entry.seq;
210 inner.entries.push_back(entry);
211 inner.watermark = seq;
212
213 while inner.entries.len() > inner.max_retained {
215 inner.entries.pop_front();
216 }
217
218 drop(inner);
219
220 self.watermark_atomic.store(seq, Ordering::Release);
221 self.notify.notify_waiters();
222 }
223}
224
225#[derive(Debug, Clone)]
230pub struct RuntimeCompletionFeed {
231 buffer: Arc<FeedBuffer>,
232}
233
234impl CompletionFeed for RuntimeCompletionFeed {
235 fn watermark(&self) -> CompletionSeq {
236 self.buffer.watermark_atomic.load(Ordering::Acquire)
237 }
238
239 fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
240 let inner = self
241 .buffer
242 .inner
243 .read()
244 .unwrap_or_else(std::sync::PoisonError::into_inner);
245 let entries: Vec<CompletionEntry> = inner
246 .entries
247 .iter()
248 .filter(|e| e.seq > after_seq)
249 .cloned()
250 .collect();
251 let watermark = inner.watermark;
252 CompletionBatch { entries, watermark }
253 }
254
255 fn wait_for_advance(
256 &self,
257 after_seq: CompletionSeq,
258 ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
259 Box::pin(async move {
260 loop {
261 let notified = self.buffer.notify.notified();
266 let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
267 if current > after_seq {
268 return current;
269 }
270 notified.await;
271 }
272 })
273 }
274}
275
276#[derive(Debug)]
281struct OperationCompletionNotifier {
282 tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
283}
284
285impl OperationCompletionNotifier {
286 fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
287 Self { tx }
288 }
289
290 fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
291 let _ = self.tx.send(outcome.clone());
292 }
293}
294
295fn operation_completion_watch_from_receiver(
296 rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
297) -> OperationCompletionWatch {
298 Box::pin(async move {
299 rx.await
300 .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
301 })
302}
303
304fn resolved_operation_completion_watch(
305 outcome: OperationTerminalOutcome,
306) -> OperationCompletionWatch {
307 Box::pin(async move { Ok(outcome) })
308}
309
310#[derive(Debug)]
314struct ShellRecord {
315 spec: OperationSpec,
316 peer_handle: Option<OperationPeerHandle>,
317 watchers: Vec<OperationCompletionNotifier>,
321 created_at: Instant,
323 started_at: Option<Instant>,
324 completed_at: Option<Instant>,
325 created_at_wall: SystemTime,
327}
328
329#[derive(Debug)]
330struct PendingWaitState {
331 wait_request_id: WaitRequestId,
332 sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
333}
334
335enum WaitAllAuthorityPlan {
336 AlreadySatisfied(WaitAllSatisfied),
337 ActivateBarrier,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq)]
341enum RecoveredOperationRecordDisposition {
342 Retain,
343 Discard,
344}
345
346impl ShellRecord {
347 fn new(spec: OperationSpec) -> Self {
348 Self {
349 spec,
350 peer_handle: None,
351 watchers: Vec::new(),
352 created_at: Instant::now(),
353 started_at: None,
354 completed_at: None,
355 created_at_wall: SystemTime::now(),
356 }
357 }
358
359 fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
360 wall_anchor
361 .duration_since(UNIX_EPOCH)
362 .map(|d| d.as_millis() as u64)
363 .unwrap_or(0)
364 }
365
366 fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
367 let offset = instant.saturating_duration_since(self.created_at);
370 let wall = self.created_at_wall + offset;
371 Self::epoch_millis(&wall)
372 }
373
374 fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
376 for watcher in std::mem::take(&mut self.watchers) {
377 watcher.notify_after_generated_terminal(outcome);
378 }
379 }
380
381 fn mark_completed(&mut self) {
383 self.completed_at = Some(Instant::now());
384 }
385}
386
387#[derive(Debug)]
392struct ShellState {
393 dsl: DslAuthority,
395 records: HashMap<OperationId, ShellRecord>,
397 pending_wait: Option<PendingWaitState>,
399 completed_order: VecDeque<OperationId>,
401 max_completed: usize,
403 max_concurrent: Option<usize>,
405 wait_request_id: Option<WaitRequestId>,
411 feed_buffer: Arc<FeedBuffer>,
413 persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
415 persistence_sealed: bool,
417 persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
419 persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
421 owner_retired: bool,
427}
428
429struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
435
436impl std::fmt::Debug for DslAuthority {
437 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
438 f.debug_struct("DslAuthority")
439 .field("state", self.0.state())
440 .finish()
441 }
442}
443
444fn new_ops_dsl_authority() -> DslAuthority {
448 DslAuthority(Box::new(
449 crate::meerkat_machine::dsl_authority::new_initialized_authority(
450 "ops lifecycle DSL Initialize must be accepted",
451 ),
452 ))
453}
454
455impl ShellState {
456 fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
457 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
458 let dsl = new_ops_dsl_authority();
459 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
460 let feed_capacity = max_completed.saturating_mul(4).max(1024);
461 tracing::info!(
462 feed_capacity,
463 "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
464 );
465 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
466 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
467 Self {
468 dsl,
469 records: HashMap::new(),
470 pending_wait: None,
471 completed_order: VecDeque::new(),
472 max_completed,
473 max_concurrent,
474 wait_request_id: None,
475 feed_buffer,
480 persist_tx: None,
481 persistence_sealed: false,
482 persist_epoch_id: None,
483 persist_cursor_state: None,
484 owner_retired: false,
485 }
486 }
487
488 fn ensure_owner_active(&self) -> Result<(), OpsLifecycleError> {
489 if self.owner_retired {
490 Err(OpsLifecycleError::OwnerRetired)
491 } else {
492 Ok(())
493 }
494 }
495
496 fn dsl_apply(
503 &mut self,
504 input: mm_dsl::MeerkatMachineInput,
505 context: &str,
506 ) -> Result<(), OpsLifecycleError> {
507 self.dsl_apply_raw(input).map_err(|err| {
508 OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
509 })
510 }
511
512 fn dsl_apply_raw(
519 &mut self,
520 input: mm_dsl::MeerkatMachineInput,
521 ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
522 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
523 }
524
525 fn dsl_apply_with_effects(
526 &mut self,
527 input: mm_dsl::MeerkatMachineInput,
528 context: &str,
529 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
530 let transition =
531 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
532 OpsLifecycleError::Internal(format!(
533 "DSL rejected ops transition ({context}): {err:?}"
534 ))
535 })?;
536 Ok(transition.into_effects())
537 }
538
539 fn checked_terminal_payload(
544 kind: mm_dsl::OperationTerminalOutcomeKind,
545 payload: &OperationTerminalOutcome,
546 authority: &str,
547 operation_id: &str,
548 ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
549 if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
550 return Err(OpsLifecycleError::Internal(format!(
551 "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
552 )));
553 }
554 Ok(payload.clone())
555 }
556
557 fn status(&self, id: &OperationId) -> Option<OperationStatus> {
559 let id_key = mm_dsl::OperationId::from_domain(id).0;
560 self.dsl
561 .0
562 .state()
563 .op_statuses
564 .get(&id_key)
565 .copied()
566 .map(OperationStatus::from)
567 }
568
569 fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
570 self.status(id).ok_or_else(|| {
571 OpsLifecycleError::Internal(format!(
572 "generated op lifecycle authority missing status for {id}"
573 ))
574 })
575 }
576
577 fn kind(&self, id: &OperationId) -> Option<OperationKind> {
579 let id_key = mm_dsl::OperationId::from_domain(id).0;
580 self.dsl
581 .0
582 .state()
583 .op_kinds
584 .get(&id_key)
585 .copied()
586 .map(OperationKind::from)
587 }
588
589 fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
590 self.kind(id).ok_or_else(|| {
591 OpsLifecycleError::Internal(format!(
592 "generated op lifecycle authority missing kind for {id}"
593 ))
594 })
595 }
596
597 fn operation_source(
598 &self,
599 id: &OperationId,
600 ) -> Result<Option<OperationSource>, OpsLifecycleError> {
601 let id_key = mm_dsl::OperationId::from_domain(id).0;
602 self.dsl
603 .0
604 .state()
605 .op_sources
606 .get(&id_key)
607 .map(|source| {
608 source.to_domain().map_err(|error| {
609 OpsLifecycleError::Internal(format!(
610 "generated operation source authority has invalid source for {id}: {error}"
611 ))
612 })
613 })
614 .transpose()
615 }
616
617 fn child_session_id_from_operation_source(
618 operation_source: Option<&OperationSource>,
619 ) -> Option<SessionId> {
620 match operation_source {
621 Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
622 Some(OperationSource::BackendPeer { .. } | OperationSource::DetachedJob { .. })
623 | None => None,
624 }
625 }
626
627 fn align_spec_child_session_id_to_source(
628 spec: &mut OperationSpec,
629 operation_source: Option<&OperationSource>,
630 ) {
631 spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
632 }
633
634 fn peer_ready(&self, id: &OperationId) -> Option<bool> {
636 let id_key = mm_dsl::OperationId::from_domain(id).0;
637 self.dsl.0.state().op_peer_ready.get(&id_key).copied()
638 }
639
640 fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
641 self.peer_ready(id).ok_or_else(|| {
642 OpsLifecycleError::Internal(format!(
643 "generated op peer wiring authority missing peer-ready fact for {id}"
644 ))
645 })
646 }
647
648 fn progress_count(&self, id: &OperationId) -> Option<u32> {
650 let id_key = mm_dsl::OperationId::from_domain(id).0;
651 self.dsl
652 .0
653 .state()
654 .op_progress_counts
655 .get(&id_key)
656 .map(|v| (*v).min(u32::MAX as u64) as u32)
657 }
658
659 fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
660 self.progress_count(id).ok_or_else(|| {
661 OpsLifecycleError::Internal(format!(
662 "generated op progress authority missing progress count for {id}"
663 ))
664 })
665 }
666
667 fn terminal_outcome(
671 &self,
672 id: &OperationId,
673 ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
674 let id_key = mm_dsl::OperationId::from_domain(id).0;
675 let state = self.dsl.0.state();
676 let status = self.status(id);
677 let terminal = match status {
678 Some(status) => Self::operation_status_is_terminal(id, status)?,
679 None => false,
680 };
681 let kind = state.op_terminal_outcomes.get(&id_key).copied();
682 let Some(kind) = kind else {
683 if terminal {
684 return Err(OpsLifecycleError::Internal(format!(
685 "generated op terminal authority missing terminal outcome for {id}"
686 )));
687 }
688 return Ok(None);
689 };
690 if !terminal {
691 return Err(OpsLifecycleError::Internal(format!(
692 "generated op terminal authority has terminal outcome for non-terminal {id}"
693 )));
694 }
695 let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
696 OpsLifecycleError::Internal(format!(
697 "generated op terminal authority missing terminal payload for {id}"
698 ))
699 })?;
700 Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
701 .map(Some)
702 }
703
704 fn contains(&self, id: &OperationId) -> bool {
706 let id_key = mm_dsl::OperationId::from_domain(id).0;
707 self.dsl.0.state().op_statuses.contains_key(&id_key)
708 }
709
710 fn active_count(&self) -> usize {
712 self.dsl.0.state().active_op_count as usize
713 }
714
715 fn operation_count(&self) -> usize {
717 self.dsl.0.state().op_statuses.len()
718 }
719
720 fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
722 let mut ids = BTreeSet::new();
723 let state = self.dsl.0.state();
724 Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
725 Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
726 Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
727 Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
728 Self::collect_operation_id_keys(
729 &mut ids,
730 "op_progress_counts",
731 state.op_progress_counts.keys(),
732 )?;
733 Self::collect_operation_id_keys(
734 &mut ids,
735 "op_terminal_outcomes",
736 state.op_terminal_outcomes.keys(),
737 )?;
738 Self::collect_operation_id_keys(
739 &mut ids,
740 "op_terminal_payload",
741 state.op_terminal_payload.keys(),
742 )?;
743 Self::collect_operation_id_keys(
744 &mut ids,
745 "op_completion_seq",
746 state.op_completion_seq.keys(),
747 )?;
748 ids.extend(self.records.keys().cloned());
749 Ok(ids.into_iter().collect())
750 }
751
752 fn collect_operation_id_keys<'a, I>(
753 ids: &mut BTreeSet<OperationId>,
754 field: &str,
755 keys: I,
756 ) -> Result<(), OpsLifecycleError>
757 where
758 I: IntoIterator<Item = &'a String>,
759 {
760 for key in keys {
761 let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
762 OpsLifecycleError::Internal(format!(
763 "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
764 ))
765 })?;
766 ids.insert(id);
767 }
768 Ok(())
769 }
770
771 fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
772 let id_key = mm_dsl::OperationId::from_domain(id).0;
773 let state = self.dsl.0.state();
774 state.op_statuses.contains_key(&id_key)
775 || state.op_kinds.contains_key(&id_key)
776 || state.op_sources.contains_key(&id_key)
777 || state.op_peer_ready.contains_key(&id_key)
778 || state.op_progress_counts.contains_key(&id_key)
779 || state.op_terminal_outcomes.contains_key(&id_key)
780 || state.op_terminal_payload.contains_key(&id_key)
781 || state.op_completion_seq.contains_key(&id_key)
782 }
783
784 fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
786 let id_key = mm_dsl::OperationId::from_domain(id).0;
787 self.dsl.0.state().op_completion_seq.get(&id_key).copied()
788 }
789
790 fn completion_feed_authority_entries(
791 &self,
792 ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
793 let state = self.dsl.0.state();
794 let sequence_keys: BTreeSet<String> =
795 state.completion_feed_sequences.keys().cloned().collect();
796 let companion_domains: [(&str, BTreeSet<String>); 3] = [
797 (
798 "completion_feed_kinds",
799 state.completion_feed_kinds.keys().cloned().collect(),
800 ),
801 (
802 "completion_feed_terminal_outcomes",
803 state
804 .completion_feed_terminal_outcomes
805 .keys()
806 .cloned()
807 .collect(),
808 ),
809 (
810 "completion_feed_terminal_payload",
811 state
812 .completion_feed_terminal_payload
813 .keys()
814 .cloned()
815 .collect(),
816 ),
817 ];
818 for (field, keys) in companion_domains {
819 if keys != sequence_keys {
820 return Err(OpsLifecycleError::Internal(format!(
821 "generated completion feed authority has mismatched {field} domain"
822 )));
823 }
824 }
825
826 let mut entries = HashMap::new();
827 for (id_key, seq) in &state.completion_feed_sequences {
828 if !state.completion_sequence_claims.contains(seq) {
829 return Err(OpsLifecycleError::Internal(format!(
830 "generated completion feed authority sequence {seq} for {id_key} is not claimed"
831 )));
832 }
833 let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
834 OpsLifecycleError::Internal(format!(
835 "generated completion feed authority used invalid operation id key {id_key}: {error}"
836 ))
837 })?;
838 let kind = state
839 .completion_feed_kinds
840 .get(id_key)
841 .copied()
842 .map(OperationKind::from)
843 .ok_or_else(|| {
844 OpsLifecycleError::Internal(format!(
845 "generated completion feed authority missing kind for {id_key}"
846 ))
847 })?;
848 let outcome_kind = state
849 .completion_feed_terminal_outcomes
850 .get(id_key)
851 .copied()
852 .ok_or_else(|| {
853 OpsLifecycleError::Internal(format!(
854 "generated completion feed authority missing terminal outcome for {id_key}"
855 ))
856 })?;
857 let payload = state
858 .completion_feed_terminal_payload
859 .get(id_key)
860 .ok_or_else(|| {
861 OpsLifecycleError::Internal(format!(
862 "generated completion feed authority missing terminal payload for {id_key}"
863 ))
864 })?;
865 let terminal_outcome = Self::checked_terminal_payload(
866 outcome_kind,
867 payload,
868 "generated completion feed authority",
869 id_key,
870 )?;
871 entries.insert(
872 operation_id,
873 CompletionFeedCanonicalState {
874 seq: *seq,
875 kind,
876 terminal_outcome,
877 },
878 );
879 }
880 Ok(entries)
881 }
882
883 fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
884 let state = self.dsl.0.state();
885 match consumer {
886 CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
887 CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
888 CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
889 }
890 }
891
892 fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
893 meerkat_core::EpochCursorSnapshot {
894 agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
895 runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
896 runtime_last_injected_seq: self
897 .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
898 }
899 }
900
901 fn snapshot(
903 &self,
904 id: &OperationId,
905 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
906 let Some(shell) = self.records.get(id) else {
907 if self.has_generated_operation_record_fact(id) {
908 return Err(OpsLifecycleError::Internal(format!(
909 "generated op lifecycle authority has operation facts without shell projection record for {id}"
910 )));
911 }
912 return Ok(None);
913 };
914 let kind = self.require_kind(id)?;
915 let status = self.require_status(id)?;
916 let terminal = Self::operation_status_is_terminal(id, status)?;
917 let public_result_class = Self::operation_public_result_class(id, status)?;
918 let peer_ready = self.require_peer_ready(id)?;
919 let progress_count = self.require_progress_count(id)?;
920 let operation_source = self.operation_source(id)?;
921 let terminal_outcome = self.terminal_outcome(id)?;
922
923 let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
924 let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
925 let completed_at_ms = shell
926 .completed_at
927 .map(|i| shell.epoch_millis_for_instant(i));
928 let elapsed_ms = shell.completed_at.map(|completed| {
929 completed
930 .saturating_duration_since(shell.created_at)
931 .as_millis() as u64
932 });
933
934 Ok(Some(OperationLifecycleSnapshot {
935 id: shell.spec.id.clone(),
936 kind,
937 owner_session_id: shell.spec.owner_session_id.clone(),
938 display_name: shell.spec.display_name.clone(),
939 source_label: shell.spec.source_label.clone(),
940 child_session_id: Self::child_session_id_from_operation_source(
941 operation_source.as_ref(),
942 ),
943 operation_source,
944 expect_peer_channel: shell.spec.expect_peer_channel,
945 status,
946 terminal,
947 public_result_class,
948 peer_ready,
949 progress_count,
950 watcher_count: shell.watchers.len() as u32,
951 terminal_outcome,
952 peer_handle: shell.peer_handle.clone(),
953 created_at_ms,
954 started_at_ms,
955 completed_at_ms,
956 elapsed_ms,
957 }))
958 }
959
960 fn finalize_terminal(
965 &mut self,
966 id: &OperationId,
967 ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
968 let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
969 OpsLifecycleError::Internal(format!(
970 "generated op terminal transition did not mint terminal outcome for {id}"
971 ))
972 })?;
973 let kind = self.require_kind(id)?;
974
975 if let Some(shell) = self.records.get_mut(id) {
977 shell.notify_watchers(&outcome);
978 shell.mark_completed();
979 }
980
981 if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
982 {
983 self.dsl_apply(
984 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
985 operation_id: mm_dsl::OperationId::from_domain(id).0,
986 },
987 "CollectCompletedOp",
988 )?;
989 self.records.remove(id);
990 self.completed_order.retain(|queued| queued != id);
991 return Ok(None);
992 }
993
994 let mut completion_entry = None;
995 if Self::operation_completion_feed_class(id, kind)?
996 == mm_dsl::OperationCompletionFeedClass::Emit
997 {
998 let feed_authority = self
999 .completion_feed_authority_entries()?
1000 .remove(id)
1001 .ok_or_else(|| {
1002 OpsLifecycleError::Internal(format!(
1003 "generated op terminal transition did not mint completion feed authority for {id}"
1004 ))
1005 })?;
1006 if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
1007 return Err(OpsLifecycleError::Internal(format!(
1008 "generated completion feed authority drifted from terminal op authority for {id}"
1009 )));
1010 }
1011 let seq = self.completion_sequence(id).ok_or_else(|| {
1012 OpsLifecycleError::Internal(format!(
1013 "generated op terminal transition did not mint completion sequence for {id}"
1014 ))
1015 })?;
1016 if feed_authority.seq != seq {
1017 return Err(OpsLifecycleError::Internal(format!(
1018 "generated completion feed authority sequence drifted for {id}"
1019 )));
1020 }
1021 let display_name = self
1022 .records
1023 .get(id)
1024 .map(|r| r.spec.display_name.clone())
1025 .unwrap_or_default();
1026 let completed_at_ms = self
1027 .records
1028 .get(id)
1029 .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1030 completion_entry = Some(CompletionEntry {
1031 seq: feed_authority.seq,
1032 operation_id: id.clone(),
1033 kind: feed_authority.kind,
1034 display_name,
1035 terminal_outcome: feed_authority.terminal_outcome,
1036 completed_at_ms,
1037 });
1038 }
1039
1040 self.completed_order.push_back(id.clone());
1042 while self.completed_order.len() > self.max_completed {
1043 if let Some(evicted) = self.completed_order.pop_front() {
1044 self.dsl_apply(
1045 mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1046 operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1047 },
1048 "EvictCompletedOp",
1049 )?;
1050 self.records.remove(&evicted);
1051 }
1052 }
1053
1054 self.maybe_satisfy_wait()?;
1059 Ok(completion_entry)
1060 }
1061
1062 fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1063 if let Some(entry) = entry {
1064 self.feed_buffer.push(entry);
1065 }
1066 }
1067
1068 fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1070 self.dsl
1071 .0
1072 .state()
1073 .wait_operation_ids
1074 .iter()
1075 .map(|key| {
1076 serde_json::from_str::<OperationId>(key).map_err(|error| {
1077 OpsLifecycleError::Internal(format!(
1078 "generated wait operation identity authority used invalid operation id key {key}: {error}"
1079 ))
1080 })
1081 })
1082 .collect()
1083 }
1084
1085 #[cfg(test)]
1087 fn wait_active(&self) -> bool {
1088 self.dsl.0.state().wait_active
1089 }
1090
1091 fn wait_all_satisfied_from_effects(
1092 effects: &[mm_dsl::MeerkatMachineEffect],
1093 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1094 let mut satisfied = None;
1095 for effect in effects {
1096 let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1097 wait_request_id,
1098 run_id,
1099 operation_ids,
1100 } = effect
1101 else {
1102 continue;
1103 };
1104 if satisfied.is_some() {
1105 return Err(OpsLifecycleError::Internal(
1106 "generated wait_all authority emitted multiple satisfaction effects".into(),
1107 ));
1108 }
1109 let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1110 OpsLifecycleError::Internal(format!(
1111 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1112 wait_request_id.0
1113 ))
1114 })?;
1115 let mut ids = Vec::with_capacity(operation_ids.len());
1116 for operation_id in operation_ids {
1117 ids.push(
1118 serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1119 OpsLifecycleError::Internal(format!(
1120 "generated wait_all authority emitted invalid operation id '{}': {err}",
1121 operation_id.0
1122 ))
1123 })?,
1124 );
1125 }
1126 satisfied = Some(WaitAllSatisfied {
1127 wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1128 run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1129 OpsLifecycleError::Internal(format!(
1130 "generated wait_all authority emitted invalid run id '{}': {err}",
1131 run_id.0
1132 ))
1133 })?),
1134 operation_ids: ids,
1135 });
1136 }
1137 Ok(satisfied)
1138 }
1139
1140 fn parse_wait_all_operation_id(
1141 raw: &str,
1142 context: &str,
1143 ) -> Result<OperationId, OpsLifecycleError> {
1144 serde_json::from_str::<OperationId>(raw).map_err(|err| {
1145 OpsLifecycleError::Internal(format!(
1146 "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1147 ))
1148 })
1149 }
1150
1151 fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1152 let mut seen = HashSet::new();
1153 operation_ids
1154 .iter()
1155 .find(|operation_id| !seen.insert((*operation_id).clone()))
1156 .cloned()
1157 }
1158
1159 fn wait_all_admission_error_from_effects(
1160 wait_request_id: &WaitRequestId,
1161 effects: &[mm_dsl::MeerkatMachineEffect],
1162 ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1163 let mut admission = None;
1164 for effect in effects {
1165 let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1166 wait_request_id: resolved_wait_request_id,
1167 result,
1168 reject_reason,
1169 rejected_operation_id,
1170 } = effect
1171 else {
1172 continue;
1173 };
1174 if admission.is_some() {
1175 return Err(OpsLifecycleError::Internal(
1176 "generated wait_all authority emitted multiple admission results".into(),
1177 ));
1178 }
1179 let resolved_uuid =
1180 uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1181 OpsLifecycleError::Internal(format!(
1182 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1183 resolved_wait_request_id.0
1184 ))
1185 })?;
1186 let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1187 if &resolved_wait_request_id != wait_request_id {
1188 return Err(OpsLifecycleError::Internal(format!(
1189 "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1190 )));
1191 }
1192 admission = Some(match result {
1193 mm_dsl::WaitAllAdmissionResultKind::Accept => {
1194 if reject_reason.is_some() || rejected_operation_id.is_some() {
1195 return Err(OpsLifecycleError::Internal(
1196 "generated wait_all authority accepted with rejection payload".into(),
1197 ));
1198 }
1199 None
1200 }
1201 mm_dsl::WaitAllAdmissionResultKind::Reject => {
1202 let reason = reject_reason.ok_or_else(|| {
1203 OpsLifecycleError::Internal(
1204 "generated wait_all authority rejected without reason".into(),
1205 )
1206 })?;
1207 let error = match reason {
1208 mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1209 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1210 OpsLifecycleError::Internal(
1211 "generated wait_all authority rejected duplicate without operation id"
1212 .into(),
1213 )
1214 })?;
1215 OpsLifecycleError::DuplicateWaitOperation(
1216 Self::parse_wait_all_operation_id(raw, "duplicate")?,
1217 )
1218 }
1219 mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1220 if rejected_operation_id.is_some() {
1221 return Err(OpsLifecycleError::Internal(
1222 "generated wait_all authority rejected active wait with operation id"
1223 .into(),
1224 ));
1225 }
1226 OpsLifecycleError::WaitAlreadyActive
1227 }
1228 mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1229 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1230 OpsLifecycleError::Internal(
1231 "generated wait_all authority rejected missing operation without operation id"
1232 .into(),
1233 )
1234 })?;
1235 OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1236 raw, "missing",
1237 )?)
1238 }
1239 };
1240 Some(error)
1241 }
1242 });
1243 }
1244 admission.ok_or_else(|| {
1245 OpsLifecycleError::Internal(
1246 "generated wait_all authority emitted no admission result".into(),
1247 )
1248 })
1249 }
1250
1251 fn resolve_wait_all_admission(
1252 &mut self,
1253 wait_request_id: &WaitRequestId,
1254 operation_ids: &[OperationId],
1255 dsl_ids: &BTreeSet<String>,
1256 dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1257 operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1258 operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1259 ) -> Result<(), OpsLifecycleError> {
1260 let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1261 .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1262 let not_found = operation_ids
1263 .iter()
1264 .find(|operation_id| !self.contains(operation_id))
1265 .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1266 let dsl_id_sequence: Vec<String> = operation_ids
1267 .iter()
1268 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1269 .collect();
1270 let effects = self.dsl_apply_with_effects(
1271 mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1272 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1273 operation_id_sequence: dsl_id_sequence,
1274 operation_ids: dsl_ids.clone(),
1275 operation_id_tokens: dsl_id_tokens.clone(),
1276 operation_token_by_id: operation_token_by_id.clone(),
1277 operation_id_by_token: operation_id_by_token.clone(),
1278 duplicate_operation_id: duplicate,
1279 not_found_operation_id: not_found,
1280 },
1281 "ResolveWaitAllAdmission",
1282 )?;
1283 if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1284 {
1285 return Err(error);
1286 }
1287 Ok(())
1288 }
1289
1290 fn try_satisfy_wait_all_authority(
1291 &mut self,
1292 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1293 let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1294 return Ok(None);
1295 };
1296 let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1297 return Err(OpsLifecycleError::Internal(
1298 "generated wait_all authority has active wait without run id".into(),
1299 ));
1300 };
1301 let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1302 let transition = match mm_dsl::MeerkatMachineMutator::apply(
1303 &mut *self.dsl.0,
1304 mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1305 wait_request_id: dsl_wait_request_id,
1306 run_id: dsl_run_id,
1307 operation_id_tokens: dsl_operation_id_tokens,
1308 },
1309 ) {
1310 Ok(transition) => transition,
1311 Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1312 Err(err) => {
1313 return Err(OpsLifecycleError::Internal(format!(
1314 "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1315 )));
1316 }
1317 };
1318 Self::wait_all_satisfied_from_effects(transition.effects())?
1319 .ok_or_else(|| {
1320 OpsLifecycleError::Internal(
1321 "generated wait_all authority accepted satisfaction without effect".into(),
1322 )
1323 })
1324 .map(Some)
1325 }
1326
1327 fn begin_wait_all_authority(
1328 &mut self,
1329 run_id: &RunId,
1330 wait_request_id: &WaitRequestId,
1331 operation_ids: &[OperationId],
1332 ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1333 let mut dsl_ids = BTreeSet::new();
1334 let mut dsl_id_tokens = BTreeSet::new();
1335 let mut operation_token_by_id = BTreeMap::new();
1336 let mut operation_id_by_token = BTreeMap::new();
1337 for id in operation_ids {
1338 let token = mm_dsl::OperationId::from_domain(id);
1339 let raw_id = token.0.clone();
1340 dsl_ids.insert(raw_id.clone());
1341 dsl_id_tokens.insert(token.clone());
1342 operation_token_by_id.insert(raw_id.clone(), token.clone());
1343 operation_id_by_token.insert(token, raw_id);
1344 }
1345 self.resolve_wait_all_admission(
1346 wait_request_id,
1347 operation_ids,
1348 &dsl_ids,
1349 &dsl_id_tokens,
1350 &operation_token_by_id,
1351 &operation_id_by_token,
1352 )?;
1353 self.dsl_apply(
1354 mm_dsl::MeerkatMachineInput::RequestWaitAll {
1355 run_id: mm_dsl::RunId::from_domain(run_id),
1356 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1357 operation_id_sequence: operation_ids
1358 .iter()
1359 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1360 .collect(),
1361 operation_ids: dsl_ids,
1362 operation_id_tokens: dsl_id_tokens,
1363 operation_token_by_id,
1364 operation_id_by_token,
1365 },
1366 "RequestWaitAll",
1367 )?;
1368 if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1369 return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1370 }
1371 Ok(WaitAllAuthorityPlan::ActivateBarrier)
1372 }
1373
1374 fn owner_termination_targets(
1375 &self,
1376 ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1377 let mut targets = Vec::new();
1378 for id in self.operation_ids()? {
1379 let status = self.require_status(&id)?;
1380 if !Self::operation_status_is_terminal(&id, status)? {
1381 targets.push((id, status));
1382 }
1383 }
1384 Ok(targets)
1385 }
1386
1387 fn operation_status_is_terminal(
1388 operation_id: &OperationId,
1389 status: OperationStatus,
1390 ) -> Result<bool, OpsLifecycleError> {
1391 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1392 let effects = Self::apply_stateless_classifier(
1393 mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1394 operation_id: operation_id_key.clone(),
1395 status: mm_dsl::OperationStatus::from(status),
1396 },
1397 "ClassifyOperationTerminality",
1398 )?;
1399 let mut terminal = None;
1400 for effect in effects {
1401 match effect {
1402 mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1403 if operation_id == operation_id_key =>
1404 {
1405 terminal = Some(true);
1406 }
1407 mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1408 if operation_id == operation_id_key =>
1409 {
1410 terminal = Some(false);
1411 }
1412 other => {
1413 return Err(OpsLifecycleError::Internal(format!(
1414 "unexpected generated operation terminality effect: {other:?}"
1415 )));
1416 }
1417 }
1418 }
1419 terminal.ok_or_else(|| {
1420 OpsLifecycleError::Internal(format!(
1421 "generated operation terminality authority emitted no effect for {operation_id}"
1422 ))
1423 })
1424 }
1425
1426 fn operation_public_result_class(
1427 operation_id: &OperationId,
1428 status: OperationStatus,
1429 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1430 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1431 let effects = Self::apply_stateless_classifier(
1432 mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1433 operation_id: operation_id_key.clone(),
1434 status: mm_dsl::OperationStatus::from(status),
1435 },
1436 "ClassifyOperationPublicResult",
1437 )?;
1438 let mut result = None;
1439 for effect in effects {
1440 match effect {
1441 mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1442 operation_id,
1443 result: classified,
1444 } if operation_id == operation_id_key => {
1445 result = Some(OperationPublicResultClass::from(classified));
1446 }
1447 other => {
1448 return Err(OpsLifecycleError::Internal(format!(
1449 "unexpected generated operation public-result effect: {other:?}"
1450 )));
1451 }
1452 }
1453 }
1454 result.ok_or_else(|| {
1455 OpsLifecycleError::Internal(format!(
1456 "generated operation public-result authority emitted no effect for {operation_id}"
1457 ))
1458 })
1459 }
1460
1461 fn operation_transition_rejection_is_idempotent(
1462 operation_id: &OperationId,
1463 action: OperationLifecycleAction,
1464 status: OperationStatus,
1465 ) -> Result<bool, OpsLifecycleError> {
1466 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1467 let action = mm_dsl::OpLifecycleActionKind::from(action);
1468 let status = mm_dsl::OperationStatus::from(status);
1469 let effects = Self::apply_stateless_classifier(
1470 mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1471 operation_id: operation_id_key.clone(),
1472 action,
1473 status,
1474 },
1475 "ClassifyOperationTransitionIdempotence",
1476 )?;
1477 let mut idempotent = None;
1478 for effect in effects {
1479 match effect {
1480 mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1481 operation_id,
1482 action: effect_action,
1483 status: effect_status,
1484 } if operation_id == operation_id_key
1485 && effect_action == action
1486 && effect_status == status =>
1487 {
1488 idempotent = Some(true);
1489 }
1490 mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1491 operation_id,
1492 action: effect_action,
1493 status: effect_status,
1494 } if operation_id == operation_id_key
1495 && effect_action == action
1496 && effect_status == status =>
1497 {
1498 idempotent = Some(false);
1499 }
1500 other => {
1501 return Err(OpsLifecycleError::Internal(format!(
1502 "unexpected generated operation transition-idempotence effect: {other:?}"
1503 )));
1504 }
1505 }
1506 }
1507 idempotent.ok_or_else(|| {
1508 OpsLifecycleError::Internal(format!(
1509 "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1510 ))
1511 })
1512 }
1513
1514 fn operation_completion_feed_class(
1515 operation_id: &OperationId,
1516 kind: OperationKind,
1517 ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1518 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1519 let kind = mm_dsl::OperationKind::from(kind);
1520 let effects = Self::apply_stateless_classifier(
1521 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1522 operation_id: operation_id_key.clone(),
1523 kind,
1524 },
1525 "ClassifyOperationCompletionFeed",
1526 )?;
1527 let mut class = None;
1528 for effect in effects {
1529 match effect {
1530 mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1531 operation_id,
1532 result,
1533 } if operation_id == operation_id_key => {
1534 class = Some(result);
1535 }
1536 other => {
1537 return Err(OpsLifecycleError::Internal(format!(
1538 "unexpected generated operation completion-feed effect: {other:?}"
1539 )));
1540 }
1541 }
1542 }
1543 class.ok_or_else(|| {
1544 OpsLifecycleError::Internal(format!(
1545 "generated operation completion-feed authority emitted no effect for {operation_id}"
1546 ))
1547 })
1548 }
1549
1550 fn operation_completion_wake_class(
1551 operation_id: &OperationId,
1552 kind: OperationKind,
1553 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1554 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1555 let kind = mm_dsl::OperationKind::from(kind);
1556 let effects = Self::apply_stateless_classifier(
1557 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1558 operation_id: operation_id_key.clone(),
1559 kind,
1560 },
1561 "ClassifyOperationCompletionWake",
1562 )?;
1563 let mut class = None;
1564 for effect in effects {
1565 match effect {
1566 mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1567 operation_id,
1568 result,
1569 } if operation_id == operation_id_key => {
1570 class = Some(OperationCompletionWakeClass::from(result));
1571 }
1572 other => {
1573 return Err(OpsLifecycleError::Internal(format!(
1574 "unexpected generated operation completion-wake effect: {other:?}"
1575 )));
1576 }
1577 }
1578 }
1579 class.ok_or_else(|| {
1580 OpsLifecycleError::Internal(format!(
1581 "generated operation completion-wake authority emitted no effect for {operation_id}"
1582 ))
1583 })
1584 }
1585
1586 fn operation_durability_class(
1587 operation_id: &OperationId,
1588 kind: OperationKind,
1589 ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1590 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1591 let kind = mm_dsl::OperationKind::from(kind);
1592 let effects = Self::apply_stateless_classifier(
1593 mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1594 operation_id: operation_id_key.clone(),
1595 kind,
1596 },
1597 "ClassifyOperationDurability",
1598 )?;
1599 let mut class = None;
1600 for effect in effects {
1601 match effect {
1602 mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1603 operation_id,
1604 result,
1605 } if operation_id == operation_id_key => {
1606 class = Some(result);
1607 }
1608 other => {
1609 return Err(OpsLifecycleError::Internal(format!(
1610 "unexpected generated operation durability effect: {other:?}"
1611 )));
1612 }
1613 }
1614 }
1615 class.ok_or_else(|| {
1616 OpsLifecycleError::Internal(format!(
1617 "generated operation durability authority emitted no effect for {operation_id}"
1618 ))
1619 })
1620 }
1621
1622 fn recovered_operation_record_disposition(
1623 operation_id: &OperationId,
1624 status: OperationStatus,
1625 kind: OperationKind,
1626 terminal_outcome_present: bool,
1627 terminal_payload_present: bool,
1628 completion_sequence_present: bool,
1629 ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1630 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1631 let effects = Self::apply_stateless_classifier(
1632 mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1633 operation_id: operation_id_key.clone(),
1634 status: mm_dsl::OperationStatus::from(status),
1635 kind: mm_dsl::OperationKind::from(kind),
1636 terminal_outcome_present,
1637 terminal_payload_present,
1638 completion_sequence_present,
1639 },
1640 "ClassifyRecoveredOperationRecord",
1641 )?;
1642 let mut disposition = None;
1643 for effect in effects {
1644 match effect {
1645 mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1646 if operation_id == operation_id_key =>
1647 {
1648 disposition = Some(RecoveredOperationRecordDisposition::Retain);
1649 }
1650 mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1651 if operation_id == operation_id_key =>
1652 {
1653 disposition = Some(RecoveredOperationRecordDisposition::Discard);
1654 }
1655 other => {
1656 return Err(OpsLifecycleError::Internal(format!(
1657 "unexpected generated recovered-operation classification effect: {other:?}"
1658 )));
1659 }
1660 }
1661 }
1662 disposition.ok_or_else(|| {
1663 OpsLifecycleError::Internal(format!(
1664 "generated recovered-operation classifier emitted no effect for {operation_id}"
1665 ))
1666 })
1667 }
1668
1669 fn apply_stateless_classifier(
1670 input: mm_dsl::MeerkatMachineInput,
1671 label: &'static str,
1672 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1673 let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1674 "ops stateless classifier Initialize must be accepted",
1675 );
1676 let transition =
1677 mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1678 OpsLifecycleError::Internal(format!(
1679 "DSL rejected ops transition ({label}): {err:?}"
1680 ))
1681 })?;
1682 Ok(transition.into_effects())
1683 }
1684
1685 fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1709 let satisfied = match self.try_satisfy_wait_all_authority() {
1710 Ok(Some(satisfied)) => satisfied,
1711 Ok(None) => return Ok(()),
1712 Err(err) => {
1713 if let Some(pending) = self.pending_wait.take() {
1721 drop(pending.sender);
1722 }
1723 self.wait_request_id = None;
1724 return Err(err);
1725 }
1726 };
1727 let shell_wait_id = self.wait_request_id.take();
1728 if shell_wait_id
1729 .as_ref()
1730 .is_some_and(|id| id != &satisfied.wait_request_id)
1731 {
1732 tracing::error!(
1733 shell_wait_request_id = ?shell_wait_id,
1734 authority_wait_request_id = %satisfied.wait_request_id,
1735 "generated wait_all authority satisfied a different wait request"
1736 );
1737 }
1738 if let Some(pending) = self.pending_wait.take() {
1739 if pending.wait_request_id == satisfied.wait_request_id {
1740 let _ = pending.sender.send(satisfied);
1741 } else if let Some(shell_wait_id) = shell_wait_id {
1742 tracing::error!(
1743 shell_wait_request_id = %shell_wait_id,
1744 pending_wait_request_id = %pending.wait_request_id,
1745 authority_wait_request_id = %satisfied.wait_request_id,
1746 "generated wait_all authority satisfied without a matching pending waiter"
1747 );
1748 }
1749 }
1750 Ok(())
1751 }
1752
1753 fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1759 let (tx, epoch_id, cursor_state) = match (
1760 &self.persist_tx,
1761 &self.persist_epoch_id,
1762 &self.persist_cursor_state,
1763 ) {
1764 (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1765 _ => return Ok(()),
1766 };
1767
1768 let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1769 let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1770 let request = OpsLifecyclePersistenceRequest {
1771 snapshot,
1772 result_tx,
1773 };
1774
1775 tx.send(request).map_err(|_| {
1776 OpsLifecycleError::Internal(
1777 "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1778 .into(),
1779 )
1780 })?;
1781 result_rx.recv().map_err(|_| {
1782 OpsLifecycleError::Internal(
1783 "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1784 .into(),
1785 )
1786 })?
1787 }
1788
1789 fn capture_snapshot(
1791 &self,
1792 epoch_id: meerkat_core::RuntimeEpochId,
1793 _cursor_state: &meerkat_core::EpochCursorState,
1794 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1795 let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1796 for op_id in self.operation_ids()? {
1797 let status = self.require_status(&op_id)?;
1798 let kind = self.require_kind(&op_id)?;
1799 if Self::operation_durability_class(&op_id, kind)?
1800 != mm_dsl::OperationDurabilityClass::Retain
1801 {
1802 continue;
1803 }
1804 let peer_ready = self.require_peer_ready(&op_id)?;
1805 let progress_count = self.require_progress_count(&op_id)?;
1806 let operation_source = self.operation_source(&op_id)?;
1807 let terminal_outcome = self.terminal_outcome(&op_id)?;
1808 let completion_sequence = self.completion_sequence(&op_id);
1809 if terminal_outcome.is_some() && completion_sequence.is_none() {
1810 return Err(OpsLifecycleError::Internal(format!(
1811 "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1812 )));
1813 }
1814 if terminal_outcome.is_none() && completion_sequence.is_some() {
1815 return Err(OpsLifecycleError::Internal(format!(
1816 "generated op terminal authority has completion sequence for non-terminal {op_id}"
1817 )));
1818 }
1819 let terminal_buffered = terminal_outcome.is_some();
1820 let watcher_count = self
1821 .records
1822 .get(&op_id)
1823 .map(|r| r.watchers.len() as u32)
1824 .unwrap_or(0);
1825 operations.insert(
1826 op_id,
1827 OperationCanonicalState {
1828 status,
1829 kind,
1830 operation_source,
1831 peer_ready,
1832 progress_count,
1833 watcher_count,
1834 terminal_outcome,
1835 completion_sequence,
1836 terminal_buffered,
1837 },
1838 );
1839 }
1840 let operation_specs: HashMap<OperationId, OperationSpec> = self
1841 .records
1842 .iter()
1843 .filter(|(id, _)| operations.contains_key(*id))
1844 .map(|(id, record)| {
1845 let mut spec = record.spec.clone();
1846 let operation_source = operations
1847 .get(id)
1848 .and_then(|state| state.operation_source.as_ref());
1849 Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1850 (id.clone(), spec)
1851 })
1852 .collect();
1853 let completed_order: VecDeque<OperationId> = self
1854 .completed_order
1855 .iter()
1856 .filter(|id| operations.contains_key(*id))
1857 .cloned()
1858 .collect();
1859 let active_count = operations
1860 .iter()
1861 .filter(|(id, state)| {
1862 matches!(
1863 Self::operation_status_is_terminal(id, state.status),
1864 Ok(false)
1865 )
1866 })
1867 .count();
1868 let authority_completion_entries = self.completion_feed_authority_entries()?;
1869 let published_completion_entries_by_id: HashMap<OperationId, CompletionEntry> = {
1870 let inner = self
1871 .feed_buffer
1872 .inner
1873 .read()
1874 .unwrap_or_else(std::sync::PoisonError::into_inner);
1875 let mut entries = HashMap::new();
1876 for entry in &inner.entries {
1877 if !authority_completion_entries.contains_key(&entry.operation_id) {
1878 return Err(OpsLifecycleError::Internal(format!(
1879 "public completion feed projection for {} has no generated authority",
1880 entry.operation_id
1881 )));
1882 }
1883 if entries
1884 .insert(entry.operation_id.clone(), entry.clone())
1885 .is_some()
1886 {
1887 return Err(OpsLifecycleError::Internal(format!(
1888 "public completion feed projection for {} appeared more than once",
1889 entry.operation_id
1890 )));
1891 }
1892 }
1893 entries
1894 };
1895 let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
1896 .iter()
1897 .map(|(operation_id, authority_entry)| {
1898 if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
1899 if projection.seq != authority_entry.seq
1900 || projection.kind != authority_entry.kind
1901 || projection.terminal_outcome != authority_entry.terminal_outcome
1902 {
1903 return Err(OpsLifecycleError::Internal(format!(
1904 "public completion feed projection for {operation_id} drifted from generated authority"
1905 )));
1906 }
1907 return Ok(projection.clone());
1908 }
1909
1910 let display_name = self
1911 .records
1912 .get(operation_id)
1913 .map(|record| record.spec.display_name.clone())
1914 .unwrap_or_default();
1915 let completed_at_ms = self.records.get(operation_id).and_then(|record| {
1916 record
1917 .completed_at
1918 .map(|completed_at| record.epoch_millis_for_instant(completed_at))
1919 });
1920
1921 Ok(CompletionEntry {
1922 seq: authority_entry.seq,
1923 operation_id: operation_id.clone(),
1924 kind: authority_entry.kind,
1925 display_name,
1926 terminal_outcome: authority_entry.terminal_outcome.clone(),
1927 completed_at_ms,
1928 })
1929 })
1930 .collect::<Result<_, _>>()?;
1931 completion_entries.sort_by_key(|entry| entry.seq);
1932
1933 let authority_state = RegistryCanonicalState {
1934 operations,
1935 completion_feed_entries: authority_completion_entries,
1936 completed_order,
1937 max_completed: self.max_completed,
1938 max_concurrent: self.max_concurrent,
1939 active_count,
1940 wait_request_id: self.wait_request_id.clone(),
1941 wait_operation_ids: self.wait_operation_ids()?,
1942 next_completion_seq: self.dsl.0.state().next_completion_seq,
1943 };
1944
1945 Ok(PersistedOpsSnapshot {
1946 epoch_id,
1947 authority_state,
1948 operation_specs,
1949 completion_entries,
1950 cursors: self.completion_cursor_snapshot(),
1951 })
1952 }
1953
1954 fn shell_record_mut(
1955 &mut self,
1956 id: &OperationId,
1957 ) -> Result<&mut ShellRecord, OpsLifecycleError> {
1958 self.records
1959 .get_mut(id)
1960 .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
1961 }
1962
1963 fn collect_wait_outcomes(
1964 &self,
1965 operation_ids: &[OperationId],
1966 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
1967 operation_ids
1968 .iter()
1969 .map(|operation_id| {
1970 let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
1971 OpsLifecycleError::Internal(format!(
1972 "wait_all completed without terminal outcome for {operation_id}"
1973 ))
1974 })?;
1975 Ok((operation_id.clone(), outcome))
1976 })
1977 .collect()
1978 }
1979}
1980
1981impl Default for ShellState {
1982 fn default() -> Self {
1983 Self::new(DEFAULT_MAX_COMPLETED, None)
1984 }
1985}
1986
1987#[derive(Debug, Clone)]
1993pub struct OpsLifecycleConfig {
1994 pub max_completed: usize,
1996 pub max_concurrent: Option<usize>,
1998}
1999
2000impl Default for OpsLifecycleConfig {
2001 fn default() -> Self {
2002 Self {
2003 max_completed: DEFAULT_MAX_COMPLETED,
2004 max_concurrent: None,
2005 }
2006 }
2007}
2008
2009#[derive(Debug)]
2016pub struct RuntimeOpsLifecycleRegistry {
2017 state: RwLock<ShellState>,
2018}
2019
2020#[derive(Debug, Clone)]
2021pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2022 pub operation_count: usize,
2023 pub active_count: usize,
2024 pub wait_request_id: Option<WaitRequestId>,
2025 pub pending_wait_present: bool,
2026 pub pending_wait_request_id: Option<WaitRequestId>,
2027 pub wait_operation_ids: Vec<OperationId>,
2028 pub operations: Vec<OperationLifecycleSnapshot>,
2029}
2030
2031impl Default for RuntimeOpsLifecycleRegistry {
2032 fn default() -> Self {
2033 Self {
2034 state: RwLock::new(ShellState::default()),
2035 }
2036 }
2037}
2038
2039impl RuntimeOpsLifecycleRegistry {
2040 pub fn new() -> Self {
2041 let dsl = new_ops_dsl_authority();
2042 let feed_capacity = DEFAULT_MAX_COMPLETED.saturating_mul(4).max(1024);
2043 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2044 Self {
2045 state: RwLock::new(ShellState {
2046 dsl,
2047 records: HashMap::new(),
2048 pending_wait: None,
2049 completed_order: VecDeque::new(),
2050 max_completed: DEFAULT_MAX_COMPLETED,
2051 max_concurrent: None,
2052 wait_request_id: None,
2053 feed_buffer,
2054 persist_tx: None,
2055 persistence_sealed: false,
2056 persist_epoch_id: None,
2057 persist_cursor_state: None,
2058 owner_retired: false,
2059 }),
2060 }
2061 }
2062
2063 pub fn with_config(config: OpsLifecycleConfig) -> Self {
2064 Self {
2065 state: RwLock::new(ShellState::new(config.max_completed, config.max_concurrent)),
2066 }
2067 }
2068
2069 fn recover_completion_feed_entry(
2070 shell: &mut ShellState,
2071 operation_id: &OperationId,
2072 entry: &CompletionFeedCanonicalState,
2073 ) -> Result<(), OpsLifecycleError> {
2074 let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2075 let terminal_outcome_kind =
2076 mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2077 let effects = shell.dsl_apply_with_effects(
2078 mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2079 operation_id: expected_operation_id.clone(),
2080 kind: mm_dsl::OperationKind::from(entry.kind),
2081 terminal_outcome: terminal_outcome_kind,
2082 terminal_payload: entry.terminal_outcome.clone(),
2083 completion_sequence: entry.seq,
2084 },
2085 "RecoverCompletionFeedEntry",
2086 )?;
2087 let recovered = effects.iter().find_map(|effect| match effect {
2088 mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2089 operation_id,
2090 seq,
2091 kind,
2092 terminal_outcome,
2093 terminal_payload,
2094 } => Some((
2095 operation_id,
2096 *seq,
2097 OperationKind::from(*kind),
2098 *terminal_outcome,
2099 terminal_payload,
2100 )),
2101 _ => None,
2102 });
2103 let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2104 return Err(OpsLifecycleError::Internal(
2105 "generated completion-feed recovery emitted no recovered entry".into(),
2106 ));
2107 };
2108 if operation_id != &expected_operation_id
2109 || seq != entry.seq
2110 || kind != entry.kind
2111 || terminal_outcome != terminal_outcome_kind
2112 || terminal_payload != &entry.terminal_outcome
2113 {
2114 return Err(OpsLifecycleError::Internal(format!(
2115 "generated completion-feed recovery drifted for {operation_id}"
2116 )));
2117 }
2118 Ok(())
2119 }
2120
2121 pub fn set_persistence_channel(
2127 &self,
2128 tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2129 epoch_id: meerkat_core::RuntimeEpochId,
2130 cursor_state: Arc<meerkat_core::EpochCursorState>,
2131 ) {
2132 if let Ok(mut state) = self.state.write() {
2133 if state.owner_retired || state.persistence_sealed {
2134 tracing::warn!(
2135 "ignored attempt to rewire a terminally sealed ops persistence channel"
2136 );
2137 return;
2138 }
2139 state.persist_tx = Some(tx);
2140 state.persist_epoch_id = Some(epoch_id);
2141 state.persist_cursor_state = Some(cursor_state);
2142 }
2143 }
2144
2145 pub(crate) fn retire_owner_for_unregister(
2154 &self,
2155 reason: String,
2156 ) -> Result<(), OpsLifecycleError> {
2157 let closed_sender = {
2158 let mut state = self.write_state()?;
2159 if state.owner_retired {
2160 return Ok(());
2161 }
2162 terminate_owner_locked(&mut state, &reason)?;
2163 state.owner_retired = true;
2164 state.persistence_sealed = true;
2165 state.persist_epoch_id = None;
2166 state.persist_cursor_state = None;
2167 state.persist_tx.take()
2168 };
2169 drop(closed_sender);
2170 Ok(())
2171 }
2172
2173 pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2180 let PersistedOpsSnapshot {
2181 authority_state,
2182 operation_specs,
2183 completion_entries,
2184 cursors,
2185 ..
2186 } = snapshot;
2187 let max_completed = authority_state.max_completed;
2188 let max_concurrent = authority_state.max_concurrent;
2189 let next_completion_seq = authority_state.next_completion_seq;
2190 let authority_completion_entries = authority_state.completion_feed_entries;
2191 let authority_operations = authority_state.operations;
2192 let mut shell = ShellState::new(max_completed, max_concurrent);
2193
2194 let mut retained_ids: HashSet<OperationId> = HashSet::new();
2199 for (op_id, op_state) in authority_operations {
2200 let terminal_outcome = op_state
2201 .terminal_outcome
2202 .as_ref()
2203 .map(mm_dsl::OperationTerminalOutcomeKind::from);
2204 let terminal_payload = op_state.terminal_outcome.clone();
2205 let disposition = ShellState::recovered_operation_record_disposition(
2206 &op_id,
2207 op_state.status,
2208 op_state.kind,
2209 terminal_outcome.is_some(),
2210 terminal_payload.is_some(),
2211 op_state.completion_sequence.is_some(),
2212 )?;
2213 if disposition == RecoveredOperationRecordDisposition::Discard {
2214 continue;
2215 }
2216 if let Some(spec_source) = operation_specs
2217 .get(&op_id)
2218 .and_then(|spec| spec.operation_source.as_ref())
2219 && op_state.operation_source.as_ref() != Some(spec_source)
2220 {
2221 return Err(OpsLifecycleError::Internal(format!(
2222 "persisted operation source mirror for {op_id} drifted from generated authority"
2223 )));
2224 }
2225 let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2226 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2227 status: mm_dsl::OperationStatus::from(op_state.status),
2228 kind: mm_dsl::OperationKind::from(op_state.kind),
2229 source: op_state
2230 .operation_source
2231 .as_ref()
2232 .map(mm_dsl::OperationSource::from_domain),
2233 peer_ready: op_state.peer_ready,
2234 progress_count: u64::from(op_state.progress_count),
2235 terminal_outcome,
2236 terminal_payload,
2237 completion_sequence: op_state.completion_sequence,
2238 };
2239 shell.dsl_apply(recovery, "RecoverOpRecord")?;
2240 let recovered_seq = shell.completion_sequence(&op_id);
2241 if op_state.completion_sequence != recovered_seq {
2242 return Err(OpsLifecycleError::Internal(format!(
2243 "generated op recovery completion sequence mismatch for {op_id}"
2244 )));
2245 }
2246 retained_ids.insert(op_id);
2247 }
2248 shell.dsl_apply(
2249 mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2250 next_completion_seq,
2251 },
2252 "RecoverOpsCompletionCursor",
2253 )?;
2254 shell.dsl_apply(
2255 mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2256 agent_applied_cursor: cursors.agent_applied_cursor,
2257 runtime_observed_cursor: cursors.runtime_observed_seq,
2258 runtime_injected_cursor: cursors.runtime_last_injected_seq,
2259 },
2260 "RecoverCompletionConsumerCursors",
2261 )?;
2262
2263 let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2266 .iter()
2267 .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2268 .collect();
2269 recovered_completed.sort_by_key(|(seq, _)| *seq);
2270 shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2271
2272 for (operation_id, entry) in &authority_completion_entries {
2276 if !retained_ids.contains(operation_id) {
2277 Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2278 }
2279 }
2280
2281 let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2282 for (operation_id, entry) in &authority_completion_entries {
2283 let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2284 return Err(OpsLifecycleError::Internal(format!(
2285 "persisted completion feed authority for {operation_id} was not recovered"
2286 )));
2287 };
2288 if recovered_entry != entry {
2289 return Err(OpsLifecycleError::Internal(format!(
2290 "persisted completion feed authority drifted from generated recovery for {operation_id}"
2291 )));
2292 }
2293 }
2294
2295 let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2299 for entry in completion_entries {
2300 let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2301 return Err(OpsLifecycleError::Internal(format!(
2302 "persisted completion feed projection for {} has no generated feed authority",
2303 entry.operation_id
2304 )));
2305 };
2306 if authority_entry.seq != entry.seq
2307 || authority_entry.kind != entry.kind
2308 || authority_entry.terminal_outcome != entry.terminal_outcome
2309 {
2310 return Err(OpsLifecycleError::Internal(format!(
2311 "persisted completion feed projection for {} drifted from generated feed authority",
2312 entry.operation_id
2313 )));
2314 }
2315 projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2316 }
2317
2318 let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2319 canonical_feed_entries.into_iter().collect();
2320 recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2321 for (operation_id, entry) in recovered_entries {
2322 let projection = projection_entries_by_id.get(&operation_id);
2323 let display_name = operation_specs
2324 .get(&operation_id)
2325 .map(|spec| spec.display_name.clone())
2326 .or_else(|| projection.map(|entry| entry.display_name.clone()))
2327 .unwrap_or_default();
2328 let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2329 shell.feed_buffer.push(CompletionEntry {
2330 seq: entry.seq,
2331 operation_id,
2332 kind: entry.kind,
2333 display_name,
2334 terminal_outcome: entry.terminal_outcome,
2335 completed_at_ms,
2336 });
2337 }
2338
2339 for (op_id, spec) in operation_specs {
2342 if retained_ids.contains(&op_id) {
2343 let mut spec = spec;
2344 let operation_source = shell.operation_source(&op_id)?;
2345 ShellState::align_spec_child_session_id_to_source(
2346 &mut spec,
2347 operation_source.as_ref(),
2348 );
2349 shell.records.insert(
2350 op_id,
2351 ShellRecord {
2352 spec,
2353 peer_handle: None,
2354 watchers: Vec::new(),
2355 created_at: Instant::now(),
2356 started_at: None,
2357 completed_at: None,
2358 created_at_wall: SystemTime::now(),
2359 },
2360 );
2361 }
2362 }
2363
2364 Ok(Self {
2365 state: RwLock::new(shell),
2366 })
2367 }
2368
2369 pub fn capture_persistence_snapshot(
2374 &self,
2375 epoch_id: meerkat_core::RuntimeEpochId,
2376 cursor_state: &meerkat_core::EpochCursorState,
2377 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2378 let state = self
2379 .state
2380 .read()
2381 .unwrap_or_else(std::sync::PoisonError::into_inner);
2382 state.capture_snapshot(epoch_id, cursor_state)
2383 }
2384
2385 pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2387 let state = self
2388 .state
2389 .read()
2390 .unwrap_or_else(std::sync::PoisonError::into_inner);
2391 state.completion_cursor_snapshot()
2392 }
2393
2394 pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2396 let state = self
2397 .state
2398 .read()
2399 .unwrap_or_else(std::sync::PoisonError::into_inner);
2400 Arc::new(RuntimeCompletionFeed {
2401 buffer: Arc::clone(&state.feed_buffer),
2402 })
2403 }
2404
2405 pub(crate) fn diagnostic_snapshot(
2407 &self,
2408 ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2409 let state = self
2410 .state
2411 .read()
2412 .unwrap_or_else(std::sync::PoisonError::into_inner);
2413 let mut operations = state
2414 .operation_ids()?
2415 .into_iter()
2416 .map(|id| state.snapshot(&id))
2417 .collect::<Result<Vec<_>, _>>()?
2418 .into_iter()
2419 .flatten()
2420 .collect::<Vec<_>>();
2421 operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2422 Ok(RuntimeOpsDiagnosticSnapshot {
2423 operation_count: state.operation_count(),
2424 active_count: state.active_count(),
2425 wait_request_id: state.wait_request_id.clone(),
2426 pending_wait_present: state.pending_wait.is_some(),
2427 pending_wait_request_id: state
2428 .pending_wait
2429 .as_ref()
2430 .map(|pending_wait| pending_wait.wait_request_id.clone()),
2431 wait_operation_ids: state.wait_operation_ids()?,
2432 operations,
2433 })
2434 }
2435
2436 fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2437 self.state
2438 .read()
2439 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2440 }
2441
2442 fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2443 self.state
2444 .write()
2445 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2446 }
2447
2448 fn cancel_wait_all_internal(
2449 &self,
2450 wait_request_id: &WaitRequestId,
2451 ) -> Result<(), OpsLifecycleError> {
2452 let mut state = self.write_state()?;
2453 match state.wait_request_id.as_ref() {
2454 Some(active) if active == wait_request_id => {
2455 state.dsl_apply(
2460 mm_dsl::MeerkatMachineInput::CancelWaitAll,
2461 "CancelWaitAll(cancel)",
2462 )?;
2463 state.wait_request_id = None;
2464 if state
2465 .pending_wait
2466 .as_ref()
2467 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2468 {
2469 state.pending_wait = None;
2470 }
2471 Ok(())
2472 }
2473 _ => {
2474 if state
2475 .pending_wait
2476 .as_ref()
2477 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2478 {
2479 state.pending_wait = None;
2480 }
2481 Ok(())
2482 }
2483 }
2484 }
2485}
2486
2487enum WaitAllFutureState {
2488 Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2489 Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2490 Done,
2491}
2492
2493struct WaitAllFuture<'a> {
2494 registry: &'a RuntimeOpsLifecycleRegistry,
2495 wait_request_id: WaitRequestId,
2496 state: WaitAllFutureState,
2497}
2498
2499impl Future for WaitAllFuture<'_> {
2500 type Output = Result<WaitAllResult, OpsLifecycleError>;
2501
2502 fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2503 match &mut self.state {
2504 WaitAllFutureState::Ready(result) => {
2505 let ready = result.take().unwrap_or_else(|| {
2506 Err(OpsLifecycleError::Internal(
2507 "wait_all future polled after completion".into(),
2508 ))
2509 });
2510 self.state = WaitAllFutureState::Done;
2511 Poll::Ready(ready)
2512 }
2513 WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2514 Poll::Pending => Poll::Pending,
2515 Poll::Ready(Ok(satisfied)) => {
2516 let outcomes = match self.registry.read_state() {
2517 Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2518 Err(err) => Err(err),
2519 };
2520 self.state = WaitAllFutureState::Done;
2521 Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2522 outcomes,
2523 satisfied,
2524 }))
2525 }
2526 Poll::Ready(Err(_)) => {
2527 self.state = WaitAllFutureState::Done;
2528 Poll::Ready(Err(OpsLifecycleError::Internal(
2529 "wait_all completion channel dropped".into(),
2530 )))
2531 }
2532 },
2533 WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2534 "wait_all future polled after completion".into(),
2535 ))),
2536 }
2537 }
2538}
2539
2540impl Drop for WaitAllFuture<'_> {
2541 fn drop(&mut self) {
2542 if matches!(self.state, WaitAllFutureState::Waiting(_))
2543 && let Err(err) = self
2544 .registry
2545 .cancel_wait_all_internal(&self.wait_request_id)
2546 {
2547 tracing::error!(
2548 wait_request_id = %self.wait_request_id,
2549 error = %err,
2550 "generated wait_all authority rejected cancellation during drop"
2551 );
2552 }
2553 }
2554}
2555
2556fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2561 match action {
2562 mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2563 mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2564 mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2565 mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2566 mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2567 mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2568 mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2569 mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2570 mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2571 mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2572 }
2573}
2574
2575fn op_lifecycle_rejection_error_from_effects(
2576 id: &OperationId,
2577 requested_action: mm_dsl::OpLifecycleActionKind,
2578 effects: &[mm_dsl::MeerkatMachineEffect],
2579) -> Result<OpsLifecycleError, OpsLifecycleError> {
2580 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2581 let mut rejection = None;
2582 for effect in effects {
2583 let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2584 operation_id,
2585 action,
2586 reason,
2587 status,
2588 } = effect
2589 else {
2590 continue;
2591 };
2592 if rejection.is_some() {
2593 return Err(OpsLifecycleError::Internal(
2594 "generated op lifecycle authority emitted multiple rejection results".into(),
2595 ));
2596 }
2597 if operation_id != &expected_id || *action != requested_action {
2598 return Err(OpsLifecycleError::Internal(format!(
2599 "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2600 )));
2601 }
2602 let error = match reason {
2603 mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2604 if status.is_some() {
2605 return Err(OpsLifecycleError::Internal(
2606 "generated op lifecycle authority emitted not-found with status".into(),
2607 ));
2608 }
2609 OpsLifecycleError::NotFound(id.clone())
2610 }
2611 mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2612 let status = status.ok_or_else(|| {
2613 OpsLifecycleError::Internal(
2614 "generated op lifecycle authority emitted invalid-transition without status"
2615 .into(),
2616 )
2617 })?;
2618 OpsLifecycleError::InvalidTransition {
2619 id: id.clone(),
2620 status: OperationStatus::from(status),
2621 action: op_lifecycle_action_label(requested_action),
2622 }
2623 }
2624 mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2625 if status.is_none() {
2626 return Err(OpsLifecycleError::Internal(
2627 "generated op lifecycle authority emitted peer-not-expected without status"
2628 .into(),
2629 ));
2630 }
2631 OpsLifecycleError::PeerNotExpected(id.clone())
2632 }
2633 mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2634 if status.is_none() {
2635 return Err(OpsLifecycleError::Internal(
2636 "generated op lifecycle authority emitted already-peer-ready without status"
2637 .into(),
2638 ));
2639 }
2640 OpsLifecycleError::AlreadyPeerReady(id.clone())
2641 }
2642 };
2643 rejection = Some(error);
2644 }
2645 rejection.ok_or_else(|| {
2646 OpsLifecycleError::Internal(
2647 "generated op lifecycle authority emitted no rejection result".into(),
2648 )
2649 })
2650}
2651
2652fn classify_generated_op_rejection(
2653 state: &mut ShellState,
2654 err: mm_dsl::MeerkatMachineTransitionError,
2655 id: &OperationId,
2656 action: mm_dsl::OpLifecycleActionKind,
2657) -> OpsLifecycleError {
2658 match err {
2659 mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2660 match state.dsl_apply_with_effects(
2661 mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2662 operation_id: mm_dsl::OperationId::from_domain(id).0,
2663 action,
2664 },
2665 "ResolveOpLifecycleTransitionRejection",
2666 ) {
2667 Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2668 .unwrap_or_else(|err| err),
2669 Err(err) => err,
2670 }
2671 }
2672 other => OpsLifecycleError::Internal(format!(
2673 "DSL rejected ops transition ({}): {other:?}",
2674 op_lifecycle_action_label(action)
2675 )),
2676 }
2677}
2678
2679fn apply_op_transition(
2680 state: &mut ShellState,
2681 id: &OperationId,
2682 input: mm_dsl::MeerkatMachineInput,
2683 action: mm_dsl::OpLifecycleActionKind,
2684) -> Result<(), OpsLifecycleError> {
2685 state.ensure_owner_active()?;
2686 state
2687 .dsl_apply_raw(input)
2688 .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2689}
2690
2691fn apply_terminal_op_transition_and_persist(
2692 state: &mut ShellState,
2693 id: &OperationId,
2694 input: mm_dsl::MeerkatMachineInput,
2695 action: mm_dsl::OpLifecycleActionKind,
2696) -> Result<(), OpsLifecycleError> {
2697 let previous_snapshot = state.dsl.0.snapshot();
2698 apply_op_transition(state, id, input, action)?;
2699 if let Err(err) = state.maybe_persist() {
2700 state.dsl.0.restore_snapshot(previous_snapshot);
2701 return Err(err);
2702 }
2703 Ok(())
2704}
2705
2706fn terminate_owner_locked(state: &mut ShellState, reason: &str) -> Result<(), OpsLifecycleError> {
2707 state.ensure_owner_active()?;
2708 let to_terminate = state.owner_termination_targets()?;
2709
2710 for (op_id, _status) in &to_terminate {
2711 let terminal_outcome = OperationTerminalOutcome::Terminated {
2712 reason: reason.to_owned(),
2713 };
2714 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2715
2716 apply_terminal_op_transition_and_persist(
2717 state,
2718 op_id,
2719 mm_dsl::MeerkatMachineInput::TerminateOp {
2720 operation_id: mm_dsl::OperationId::from_domain(op_id).0,
2721 outcome: outcome_kind,
2722 payload: terminal_outcome,
2723 },
2724 mm_dsl::OpLifecycleActionKind::Terminate,
2725 )?;
2726
2727 if let Some(entry) = state.finalize_terminal(op_id)? {
2728 state.publish_completion_entry(Some(entry));
2729 }
2730 }
2731 Ok(())
2732}
2733
2734fn op_registration_error_from_effects(
2735 id: &OperationId,
2736 effects: &[mm_dsl::MeerkatMachineEffect],
2737) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2738 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2739 let mut admission = None;
2740 for effect in effects {
2741 let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2742 operation_id,
2743 result,
2744 reject_reason,
2745 max_concurrent_limit,
2746 active_op_count,
2747 } = effect
2748 else {
2749 continue;
2750 };
2751 if admission.is_some() {
2752 return Err(OpsLifecycleError::Internal(
2753 "generated op registration authority emitted multiple admission results".into(),
2754 ));
2755 }
2756 if operation_id != &expected_id {
2757 return Err(OpsLifecycleError::Internal(format!(
2758 "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2759 )));
2760 }
2761 admission = Some(match result {
2762 mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2763 if reject_reason.is_some() {
2764 return Err(OpsLifecycleError::Internal(
2765 "generated op registration authority accepted with rejection reason".into(),
2766 ));
2767 }
2768 None
2769 }
2770 mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2771 let reason = reject_reason.ok_or_else(|| {
2772 OpsLifecycleError::Internal(
2773 "generated op registration authority rejected without reason".into(),
2774 )
2775 })?;
2776 let error = match reason {
2777 mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2778 OpsLifecycleError::AlreadyRegistered(id.clone())
2779 }
2780 mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2781 let limit = max_concurrent_limit.ok_or_else(|| {
2782 OpsLifecycleError::Internal(
2783 "generated op registration authority rejected capacity without limit"
2784 .into(),
2785 )
2786 })?;
2787 OpsLifecycleError::MaxConcurrentExceeded {
2788 limit: limit as usize,
2789 active: *active_op_count as usize,
2790 }
2791 }
2792 };
2793 Some(error)
2794 }
2795 });
2796 }
2797 admission.ok_or_else(|| {
2798 OpsLifecycleError::Internal(
2799 "generated op registration authority emitted no admission result".into(),
2800 )
2801 })
2802}
2803
2804impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2805 fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2806 self.register_operation_with_admission_limit(spec, None)
2807 }
2808
2809 fn register_operation_with_admission_limit(
2810 &self,
2811 mut spec: OperationSpec,
2812 max_concurrent: Option<usize>,
2813 ) -> Result<(), OpsLifecycleError> {
2814 let mut state = self.write_state()?;
2815 state.ensure_owner_active()?;
2816 let operation_id = spec.id.clone();
2817 let kind = spec.kind;
2818 let max_concurrent = max_concurrent
2819 .or(state.max_concurrent)
2820 .map(|limit| limit as u64);
2821
2822 let effects = state.dsl_apply_with_effects(
2823 mm_dsl::MeerkatMachineInput::RegisterOp {
2824 operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2825 kind: mm_dsl::OperationKind::from_domain(&kind),
2826 source: spec
2827 .operation_source
2828 .as_ref()
2829 .map(mm_dsl::OperationSource::from_domain),
2830 max_concurrent,
2831 },
2832 "RegisterOp",
2833 )?;
2834 if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
2835 return Err(error);
2836 }
2837
2838 let authority_operation_source = state.operation_source(&operation_id)?;
2839 ShellState::align_spec_child_session_id_to_source(
2840 &mut spec,
2841 authority_operation_source.as_ref(),
2842 );
2843
2844 state.records.insert(operation_id, ShellRecord::new(spec));
2846 Ok(())
2847 }
2848
2849 fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2850 let mut state = self.write_state()?;
2851
2852 apply_op_transition(
2853 &mut state,
2854 id,
2855 mm_dsl::MeerkatMachineInput::StartOp {
2856 operation_id: mm_dsl::OperationId::from_domain(id).0,
2857 },
2858 mm_dsl::OpLifecycleActionKind::Start,
2859 )?;
2860
2861 if let Some(shell) = state.records.get_mut(id) {
2863 shell.started_at = Some(Instant::now());
2864 }
2865 Ok(())
2866 }
2867
2868 fn provisioning_failed(
2869 &self,
2870 id: &OperationId,
2871 error: String,
2872 ) -> Result<(), OpsLifecycleError> {
2873 let mut state = self.write_state()?;
2874
2875 let terminal_outcome = OperationTerminalOutcome::Failed { error };
2876 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2877
2878 apply_terminal_op_transition_and_persist(
2879 &mut state,
2880 id,
2881 mm_dsl::MeerkatMachineInput::FailOp {
2882 operation_id: mm_dsl::OperationId::from_domain(id).0,
2883 outcome: outcome_kind,
2884 payload: terminal_outcome,
2885 },
2886 mm_dsl::OpLifecycleActionKind::Fail,
2887 )?;
2888
2889 let completion_entry = state.finalize_terminal(id)?;
2890 state.publish_completion_entry(completion_entry);
2891 Ok(())
2892 }
2893
2894 fn peer_ready(
2895 &self,
2896 id: &OperationId,
2897 peer: OperationPeerHandle,
2898 ) -> Result<(), OpsLifecycleError> {
2899 let mut state = self.write_state()?;
2900
2901 apply_op_transition(
2902 &mut state,
2903 id,
2904 mm_dsl::MeerkatMachineInput::PeerReadyOp {
2905 operation_id: mm_dsl::OperationId::from_domain(id).0,
2906 },
2907 mm_dsl::OpLifecycleActionKind::PeerReady,
2908 )?;
2909
2910 if let Some(shell) = state.records.get_mut(id) {
2912 shell.peer_handle = Some(peer);
2913 }
2914 Ok(())
2915 }
2916
2917 fn register_watcher(
2918 &self,
2919 id: &OperationId,
2920 ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
2921 let mut state = self.write_state()?;
2922
2923 if !state.contains(id) {
2924 return Err(OpsLifecycleError::NotFound(id.clone()));
2925 }
2926
2927 if let Some(outcome) = state.terminal_outcome(id)? {
2929 return Ok(resolved_operation_completion_watch(outcome));
2930 }
2931
2932 let shell = state.shell_record_mut(id)?;
2934 let (tx, rx) = tokio::sync::oneshot::channel();
2935 let watch = operation_completion_watch_from_receiver(rx);
2936 shell.watchers.push(OperationCompletionNotifier::new(tx));
2937 Ok(watch)
2938 }
2939
2940 fn report_progress(
2941 &self,
2942 id: &OperationId,
2943 _update: OperationProgressUpdate,
2944 ) -> Result<(), OpsLifecycleError> {
2945 let mut state = self.write_state()?;
2946
2947 apply_op_transition(
2948 &mut state,
2949 id,
2950 mm_dsl::MeerkatMachineInput::ProgressReportedOp {
2951 operation_id: mm_dsl::OperationId::from_domain(id).0,
2952 },
2953 mm_dsl::OpLifecycleActionKind::ProgressReported,
2954 )?;
2955 Ok(())
2956 }
2957
2958 fn complete_operation(
2959 &self,
2960 id: &OperationId,
2961 result: OperationResult,
2962 ) -> Result<(), OpsLifecycleError> {
2963 let mut state = self.write_state()?;
2964
2965 let terminal_outcome = OperationTerminalOutcome::Completed(result);
2966 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2967
2968 apply_terminal_op_transition_and_persist(
2969 &mut state,
2970 id,
2971 mm_dsl::MeerkatMachineInput::CompleteOp {
2972 operation_id: mm_dsl::OperationId::from_domain(id).0,
2973 outcome: outcome_kind,
2974 payload: terminal_outcome,
2975 },
2976 mm_dsl::OpLifecycleActionKind::Complete,
2977 )?;
2978
2979 let completion_entry = state.finalize_terminal(id)?;
2980 state.publish_completion_entry(completion_entry);
2981 Ok(())
2982 }
2983
2984 fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
2985 let mut state = self.write_state()?;
2986
2987 let terminal_outcome = OperationTerminalOutcome::Failed { error };
2988 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2989
2990 apply_terminal_op_transition_and_persist(
2991 &mut state,
2992 id,
2993 mm_dsl::MeerkatMachineInput::FailOp {
2994 operation_id: mm_dsl::OperationId::from_domain(id).0,
2995 outcome: outcome_kind,
2996 payload: terminal_outcome,
2997 },
2998 mm_dsl::OpLifecycleActionKind::Fail,
2999 )?;
3000
3001 let completion_entry = state.finalize_terminal(id)?;
3002 state.publish_completion_entry(completion_entry);
3003 Ok(())
3004 }
3005
3006 fn abort_provisioning(
3007 &self,
3008 id: &OperationId,
3009 reason: Option<String>,
3010 ) -> Result<(), OpsLifecycleError> {
3011 let mut state = self.write_state()?;
3012
3013 let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
3014 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3015
3016 apply_terminal_op_transition_and_persist(
3017 &mut state,
3018 id,
3019 mm_dsl::MeerkatMachineInput::AbortOp {
3020 operation_id: mm_dsl::OperationId::from_domain(id).0,
3021 outcome: outcome_kind,
3022 payload: terminal_outcome,
3023 },
3024 mm_dsl::OpLifecycleActionKind::Abort,
3025 )?;
3026
3027 let completion_entry = state.finalize_terminal(id)?;
3028 state.publish_completion_entry(completion_entry);
3029 Ok(())
3030 }
3031
3032 fn rollback_unreturned_operation(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3033 let mut state = self.write_state()?;
3034 state.ensure_owner_active()?;
3035 let expected_id = mm_dsl::OperationId::from_domain(id).0;
3036 let effects = state.dsl_apply_with_effects(
3037 mm_dsl::MeerkatMachineInput::RollbackUnreturnedOp {
3038 operation_id: expected_id.clone(),
3039 },
3040 "RollbackUnreturnedOp",
3041 )?;
3042 let mut acknowledgements = effects.iter().filter(|effect| {
3043 matches!(
3044 effect,
3045 mm_dsl::MeerkatMachineEffect::UnreturnedOpRolledBack { operation_id }
3046 if operation_id == &expected_id
3047 )
3048 });
3049 if acknowledgements.next().is_none() || acknowledgements.next().is_some() {
3050 return Err(OpsLifecycleError::Internal(format!(
3051 "generated rollback authority did not emit exactly one matching acknowledgement for {id}"
3052 )));
3053 }
3054 state.records.remove(id);
3055 state.completed_order.retain(|queued| queued != id);
3056 Ok(())
3057 }
3058
3059 fn cancel_operation(
3060 &self,
3061 id: &OperationId,
3062 reason: Option<String>,
3063 ) -> Result<(), OpsLifecycleError> {
3064 let mut state = self.write_state()?;
3065
3066 let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
3067 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3068
3069 apply_terminal_op_transition_and_persist(
3070 &mut state,
3071 id,
3072 mm_dsl::MeerkatMachineInput::CancelOp {
3073 operation_id: mm_dsl::OperationId::from_domain(id).0,
3074 outcome: outcome_kind,
3075 payload: terminal_outcome,
3076 },
3077 mm_dsl::OpLifecycleActionKind::Cancel,
3078 )?;
3079
3080 let completion_entry = state.finalize_terminal(id)?;
3081 state.publish_completion_entry(completion_entry);
3082 Ok(())
3083 }
3084
3085 fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3086 let mut state = self.write_state()?;
3087
3088 apply_op_transition(
3089 &mut state,
3090 id,
3091 mm_dsl::MeerkatMachineInput::RetireRequestedOp {
3092 operation_id: mm_dsl::OperationId::from_domain(id).0,
3093 },
3094 mm_dsl::OpLifecycleActionKind::RetireRequested,
3095 )?;
3096 Ok(())
3097 }
3098
3099 fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3100 let mut state = self.write_state()?;
3101
3102 let terminal_outcome = OperationTerminalOutcome::Retired;
3103 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3104
3105 apply_terminal_op_transition_and_persist(
3106 &mut state,
3107 id,
3108 mm_dsl::MeerkatMachineInput::RetireCompletedOp {
3109 operation_id: mm_dsl::OperationId::from_domain(id).0,
3110 outcome: outcome_kind,
3111 payload: terminal_outcome,
3112 },
3113 mm_dsl::OpLifecycleActionKind::RetireCompleted,
3114 )?;
3115
3116 let completion_entry = state.finalize_terminal(id)?;
3117 state.publish_completion_entry(completion_entry);
3118 Ok(())
3119 }
3120
3121 fn snapshot(
3122 &self,
3123 id: &OperationId,
3124 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3125 let state = self.read_state()?;
3126 state.snapshot(id)
3127 }
3128
3129 fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3130 let state = self.read_state()?;
3131 let mut snapshots = Vec::new();
3132 for id in state.operation_ids()? {
3133 let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3134 OpsLifecycleError::Internal(format!(
3135 "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3136 ))
3137 })?;
3138 snapshots.push(snapshot);
3139 }
3140 snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3141 Ok(snapshots)
3142 }
3143
3144 fn classify_operation_terminality(
3145 &self,
3146 id: &OperationId,
3147 status: OperationStatus,
3148 ) -> Result<bool, OpsLifecycleError> {
3149 ShellState::operation_status_is_terminal(id, status)
3150 }
3151
3152 fn classify_operation_public_result(
3153 &self,
3154 id: &OperationId,
3155 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3156 let state = self.read_state()?;
3157 let status = match state.status(id) {
3158 Some(status) => status,
3159 None if state.records.contains_key(id)
3160 || state.has_generated_operation_record_fact(id) =>
3161 {
3162 return Err(OpsLifecycleError::Internal(format!(
3163 "generated op lifecycle authority missing status for {id}"
3164 )));
3165 }
3166 None => OperationStatus::Absent,
3167 };
3168 ShellState::operation_public_result_class(id, status)
3169 }
3170
3171 fn classify_operation_completion_wake(
3172 &self,
3173 id: &OperationId,
3174 kind: OperationKind,
3175 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3176 ShellState::operation_completion_wake_class(id, kind)
3177 }
3178
3179 fn classify_operation_transition_idempotence(
3180 &self,
3181 id: &OperationId,
3182 action: OperationLifecycleAction,
3183 ) -> Result<bool, OpsLifecycleError> {
3184 let state = self.read_state()?;
3185 let status = match state.status(id) {
3186 Some(status) => status,
3187 None if state.records.contains_key(id)
3188 || state.has_generated_operation_record_fact(id) =>
3189 {
3190 return Err(OpsLifecycleError::Internal(format!(
3191 "generated op lifecycle authority missing status for {id}"
3192 )));
3193 }
3194 None => OperationStatus::Absent,
3195 };
3196 ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3197 }
3198
3199 fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3200 let mut state = self.write_state()?;
3201 if state.owner_retired {
3202 return Ok(());
3203 }
3204 terminate_owner_locked(&mut state, &reason)
3205 }
3206
3207 fn collect_completed(
3208 &self,
3209 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3210 let mut state = self.write_state()?;
3211 state.ensure_owner_active()?;
3212
3213 let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3214 let mut collected = Vec::with_capacity(ids.len());
3215 for id in ids {
3216 let outcome = state.terminal_outcome(&id)?;
3217 state.dsl_apply(
3218 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3219 operation_id: mm_dsl::OperationId::from_domain(&id).0,
3220 },
3221 "CollectCompletedOp",
3222 )?;
3223 state.completed_order.retain(|queued| queued != &id);
3224 state.records.remove(&id);
3225 if let Some(outcome) = outcome {
3226 collected.push((id, outcome));
3227 }
3228 }
3229 Ok(collected)
3230 }
3231
3232 fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3233 Some(self.completion_feed_handle())
3234 }
3235
3236 fn completion_cursor(
3237 &self,
3238 consumer: CompletionCursorConsumer,
3239 ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3240 let state = self.read_state()?;
3241 Ok(Some(state.completion_cursor(consumer)))
3242 }
3243
3244 fn advance_completion_cursor(
3245 &self,
3246 consumer: CompletionCursorConsumer,
3247 cursor: CompletionSeq,
3248 projection: Option<&meerkat_core::EpochCursorState>,
3249 ) -> Result<CompletionSeq, OpsLifecycleError> {
3250 let mut state = self.write_state()?;
3251 state.ensure_owner_active()?;
3252 let input = match consumer {
3253 CompletionCursorConsumer::AgentApplied => {
3254 mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3255 }
3256 CompletionCursorConsumer::RuntimeObserved => {
3257 mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3258 }
3259 CompletionCursorConsumer::RuntimeInjected => {
3260 mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3261 }
3262 };
3263 let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3264 let advanced = effects
3265 .iter()
3266 .find_map(|effect| match (consumer, effect) {
3267 (
3268 CompletionCursorConsumer::AgentApplied,
3269 mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3270 ) => Some(*cursor),
3271 (
3272 CompletionCursorConsumer::RuntimeObserved,
3273 mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3274 cursor,
3275 },
3276 ) => Some(*cursor),
3277 (
3278 CompletionCursorConsumer::RuntimeInjected,
3279 mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3280 cursor,
3281 },
3282 ) => Some(*cursor),
3283 _ => None,
3284 })
3285 .ok_or_else(|| {
3286 OpsLifecycleError::Internal(format!(
3287 "generated completion cursor transition emitted no feedback for {consumer:?}"
3288 ))
3289 })?;
3290 if let Some(projection) = projection {
3291 projection.project_authorized_completion_cursor(consumer, advanced);
3292 }
3293 Ok(advanced)
3294 }
3295
3296 fn wait_all(
3297 &self,
3298 run_id: &RunId,
3299 ids: &[OperationId],
3300 ) -> std::pin::Pin<
3301 Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3302 > {
3303 let wait_request_id = WaitRequestId::new();
3304 let owned_ids = ids.to_vec();
3305
3306 let state = match self.write_state() {
3307 Ok(mut state) => {
3308 if let Err(error) = state.ensure_owner_active() {
3309 return Box::pin(WaitAllFuture {
3310 registry: self,
3311 wait_request_id,
3312 state: WaitAllFutureState::Ready(Some(Err(error))),
3313 });
3314 }
3315 match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3316 Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3317 let outcomes =
3318 state
3319 .collect_wait_outcomes(&satisfied.operation_ids)
3320 .map(|outcomes| WaitAllResult {
3321 outcomes,
3322 satisfied,
3323 });
3324 WaitAllFutureState::Ready(Some(outcomes))
3325 }
3326 Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3327 if state.pending_wait.is_some() {
3328 let rollback = state.dsl_apply(
3334 mm_dsl::MeerkatMachineInput::CancelWaitAll,
3335 "CancelWaitAll(rollback)",
3336 );
3337 return Box::pin(WaitAllFuture {
3338 registry: self,
3339 wait_request_id,
3340 state: WaitAllFutureState::Ready(Some(Err(match rollback {
3341 Ok(()) => OpsLifecycleError::Internal(
3342 "wait_all started while a pending wait sender already existed"
3343 .into(),
3344 ),
3345 Err(err) => err,
3346 }))),
3347 });
3348 }
3349 state.wait_request_id = Some(wait_request_id.clone());
3350 let (sender, receiver) = tokio::sync::oneshot::channel();
3351 state.pending_wait = Some(PendingWaitState {
3352 wait_request_id: wait_request_id.clone(),
3353 sender,
3354 });
3355 WaitAllFutureState::Waiting(receiver)
3356 }
3357 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3358 }
3359 }
3360 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3361 };
3362
3363 Box::pin(WaitAllFuture {
3364 registry: self,
3365 wait_request_id,
3366 state,
3367 })
3368 }
3369}
3370
3371#[cfg(test)]
3372#[allow(clippy::unwrap_used, clippy::panic)]
3373mod tests {
3374 use super::*;
3375 use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3376 use meerkat_core::lifecycle::RunId;
3377 use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3378 use meerkat_core::types::SessionId;
3379 use std::sync::atomic::Ordering;
3380 use uuid::Uuid;
3381
3382 fn test_run_id() -> RunId {
3383 RunId(Uuid::from_u128(1))
3384 }
3385
3386 fn background_spec(name: &str) -> OperationSpec {
3387 OperationSpec {
3388 id: OperationId::new(),
3389 kind: OperationKind::BackgroundToolOp,
3390 owner_session_id: SessionId::new(),
3391 display_name: name.into(),
3392 source_label: "test".into(),
3393 operation_source: None,
3394 child_session_id: None,
3395 expect_peer_channel: false,
3396 }
3397 }
3398
3399 #[tokio::test]
3400 async fn late_watchers_resolve_immediately() {
3401 let registry = RuntimeOpsLifecycleRegistry::new();
3402 let spec = background_spec("late");
3403 let op_id = spec.id.clone();
3404 registry.register_operation(spec).unwrap();
3405 registry.provisioning_succeeded(&op_id).unwrap();
3406 registry
3407 .complete_operation(
3408 &op_id,
3409 OperationResult {
3410 id: op_id.clone(),
3411 content: "done".into(),
3412 is_error: false,
3413 duration_ms: 1,
3414 tokens_used: 0,
3415 },
3416 )
3417 .unwrap();
3418
3419 let watch = registry.register_watcher(&op_id).unwrap();
3420 match watch
3421 .await
3422 .expect("operation completion watch should resolve")
3423 {
3424 OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3425 other => panic!("expected completed outcome, got {other:?}"),
3426 }
3427 }
3428
3429 #[tokio::test]
3430 async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3431 let (tx, rx) = tokio::sync::oneshot::channel();
3432 let watch = operation_completion_watch_from_receiver(rx);
3433 drop(tx);
3434
3435 assert_eq!(
3436 watch.await,
3437 Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3438 );
3439 }
3440
3441 #[test]
3442 fn peer_ready_requires_peer_expectation() {
3443 let registry = RuntimeOpsLifecycleRegistry::new();
3444 let spec = background_spec("no-peer");
3445 let op_id = spec.id.clone();
3446 registry.register_operation(spec).unwrap();
3447 registry.provisioning_succeeded(&op_id).unwrap();
3448
3449 let result = registry.peer_ready(
3450 &op_id,
3451 OperationPeerHandle {
3452 peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3453 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3454 "peer",
3455 PeerId::new(),
3456 "inproc://peer",
3457 )
3458 .unwrap(),
3459 },
3460 );
3461 assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3462 }
3463
3464 #[test]
3470 fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3471 let op_id = OperationId::new();
3472 let outcomes = vec![
3473 (
3474 OperationTerminalOutcome::Completed(OperationResult {
3475 id: op_id.clone(),
3476 content: "done".into(),
3477 is_error: false,
3478 duration_ms: 7,
3479 tokens_used: 42,
3480 }),
3481 mm_dsl::OperationTerminalOutcomeKind::Completed,
3482 ),
3483 (
3484 OperationTerminalOutcome::Failed {
3485 error: "boom".into(),
3486 },
3487 mm_dsl::OperationTerminalOutcomeKind::Failed,
3488 ),
3489 (
3490 OperationTerminalOutcome::Aborted {
3491 reason: Some("user aborted".into()),
3492 },
3493 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3494 ),
3495 (
3496 OperationTerminalOutcome::Aborted { reason: None },
3497 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3498 ),
3499 (
3500 OperationTerminalOutcome::Cancelled {
3501 reason: Some("cancelled".into()),
3502 },
3503 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3504 ),
3505 (
3506 OperationTerminalOutcome::Cancelled { reason: None },
3507 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3508 ),
3509 (
3510 OperationTerminalOutcome::Retired,
3511 mm_dsl::OperationTerminalOutcomeKind::Retired,
3512 ),
3513 (
3514 OperationTerminalOutcome::Terminated {
3515 reason: "owner stopped".into(),
3516 },
3517 mm_dsl::OperationTerminalOutcomeKind::Terminated,
3518 ),
3519 ];
3520
3521 for (outcome, expected_kind) in &outcomes {
3522 assert_eq!(
3523 mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3524 *expected_kind,
3525 "typed payload {outcome:?} must classify to {expected_kind:?}"
3526 );
3527 let read = ShellState::checked_terminal_payload(
3528 *expected_kind,
3529 outcome,
3530 "test authority",
3531 "test-op",
3532 )
3533 .expect("matching discriminant must read back the exact payload");
3534 assert_eq!(&read, outcome);
3535 }
3536
3537 let err = ShellState::checked_terminal_payload(
3539 mm_dsl::OperationTerminalOutcomeKind::Completed,
3540 &OperationTerminalOutcome::Retired,
3541 "test authority",
3542 "test-op",
3543 )
3544 .expect_err("variant mismatch must be rejected");
3545 assert!(matches!(err, OpsLifecycleError::Internal(_)));
3546 }
3547
3548 #[test]
3554 fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3555 let registry = RuntimeOpsLifecycleRegistry::new();
3556 let spec = background_spec("variant-mismatch");
3557 let op_id = spec.id.clone();
3558 registry.register_operation(spec).unwrap();
3559 registry.provisioning_succeeded(&op_id).unwrap();
3560
3561 let mut state = registry.write_state().unwrap();
3562 let err = state
3563 .dsl_apply(
3564 mm_dsl::MeerkatMachineInput::CompleteOp {
3565 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3566 outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3567 payload: OperationTerminalOutcome::Retired,
3569 },
3570 "CompleteOp",
3571 )
3572 .expect_err("machine must reject payload variant mismatch");
3573 assert!(
3580 matches!(
3581 &err,
3582 OpsLifecycleError::Internal(message)
3583 if message.contains("GuardRejected") && message.contains("CompleteOp")
3584 ),
3585 "expected generated guard rejection for CompleteOp, got: {err:?}"
3586 );
3587 drop(state);
3588
3589 registry
3592 .complete_operation(
3593 &op_id,
3594 OperationResult {
3595 id: op_id.clone(),
3596 content: "done".into(),
3597 is_error: false,
3598 duration_ms: 1,
3599 tokens_used: 0,
3600 },
3601 )
3602 .expect("matching variant must complete");
3603 }
3604
3605 #[test]
3606 fn duplicate_registration_rejection_is_generated() {
3607 let registry = RuntimeOpsLifecycleRegistry::new();
3608 let spec = background_spec("duplicate");
3609 let op_id = spec.id.clone();
3610
3611 registry.register_operation(spec.clone()).unwrap();
3612 let result = registry.register_operation(spec);
3613
3614 assert!(matches!(
3615 result,
3616 Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3617 ));
3618 }
3619
3620 #[test]
3621 fn invalid_transition_rejection_is_generated() {
3622 let registry = RuntimeOpsLifecycleRegistry::new();
3623 let spec = background_spec("invalid-transition");
3624 let op_id = spec.id.clone();
3625 registry.register_operation(spec).unwrap();
3626
3627 let result = registry.complete_operation(
3628 &op_id,
3629 OperationResult {
3630 id: op_id.clone(),
3631 content: "too-early".into(),
3632 is_error: false,
3633 duration_ms: 1,
3634 tokens_used: 0,
3635 },
3636 );
3637
3638 assert!(matches!(
3639 result,
3640 Err(OpsLifecycleError::InvalidTransition {
3641 id,
3642 status: OperationStatus::Provisioning,
3643 action: "complete_operation",
3644 }) if id == op_id
3645 ));
3646 }
3647
3648 #[tokio::test]
3649 async fn multi_listener_completion() {
3650 let registry = RuntimeOpsLifecycleRegistry::new();
3651 let spec = background_spec("multi");
3652 let op_id = spec.id.clone();
3653 registry.register_operation(spec).unwrap();
3654 registry.provisioning_succeeded(&op_id).unwrap();
3655
3656 let watch1 = registry.register_watcher(&op_id).unwrap();
3657 let watch2 = registry.register_watcher(&op_id).unwrap();
3658 let watch3 = registry.register_watcher(&op_id).unwrap();
3659
3660 registry
3661 .complete_operation(
3662 &op_id,
3663 OperationResult {
3664 id: op_id.clone(),
3665 content: "multi-done".into(),
3666 is_error: false,
3667 duration_ms: 1,
3668 tokens_used: 0,
3669 },
3670 )
3671 .unwrap();
3672
3673 for watch in [watch1, watch2, watch3] {
3674 match watch
3675 .await
3676 .expect("operation completion watch should resolve")
3677 {
3678 OperationTerminalOutcome::Completed(result) => {
3679 assert_eq!(result.content, "multi-done");
3680 }
3681 other => panic!("expected completed, got {other:?}"),
3682 }
3683 }
3684 }
3685
3686 #[tokio::test]
3687 async fn wait_all_returns_all_outcomes() {
3688 let registry = RuntimeOpsLifecycleRegistry::new();
3689
3690 let spec_a = background_spec("a");
3691 let id_a = spec_a.id.clone();
3692 registry.register_operation(spec_a).unwrap();
3693 registry.provisioning_succeeded(&id_a).unwrap();
3694
3695 let spec_b = background_spec("b");
3696 let id_b = spec_b.id.clone();
3697 registry.register_operation(spec_b).unwrap();
3698 registry.provisioning_succeeded(&id_b).unwrap();
3699
3700 registry
3701 .complete_operation(
3702 &id_a,
3703 OperationResult {
3704 id: id_a.clone(),
3705 content: "a-done".into(),
3706 is_error: false,
3707 duration_ms: 1,
3708 tokens_used: 0,
3709 },
3710 )
3711 .unwrap();
3712 registry.fail_operation(&id_b, "b-error".into()).unwrap();
3713
3714 let wait_result = registry
3715 .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3716 .await
3717 .unwrap();
3718 assert_eq!(wait_result.outcomes.len(), 2);
3719 assert_eq!(wait_result.outcomes[0].0, id_a);
3720 assert!(matches!(
3721 wait_result.outcomes[0].1,
3722 OperationTerminalOutcome::Completed(_)
3723 ));
3724 assert_eq!(wait_result.outcomes[1].0, id_b);
3725 assert!(matches!(
3726 wait_result.outcomes[1].1,
3727 OperationTerminalOutcome::Failed { .. }
3728 ));
3729 assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3731 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3732 }
3733
3734 #[tokio::test]
3737 async fn wait_all_trait_path_submits_through_authority() {
3738 let registry = RuntimeOpsLifecycleRegistry::new();
3739 let spec = background_spec("trait-wait");
3740 let op_id = spec.id.clone();
3741 registry.register_operation(spec).unwrap();
3742 registry.provisioning_succeeded(&op_id).unwrap();
3743 registry
3744 .complete_operation(
3745 &op_id,
3746 OperationResult {
3747 id: op_id.clone(),
3748 content: "done".into(),
3749 is_error: false,
3750 duration_ms: 1,
3751 tokens_used: 0,
3752 },
3753 )
3754 .unwrap();
3755
3756 let trait_ref: &dyn OpsLifecycleRegistry = ®istry;
3758 let wait_result = trait_ref
3759 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3760 .await
3761 .unwrap();
3762 assert_eq!(wait_result.outcomes.len(), 1);
3763 assert!(matches!(
3764 wait_result.outcomes[0].1,
3765 OperationTerminalOutcome::Completed(_)
3766 ));
3767 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3769 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3770 let state = registry.read_state().unwrap();
3771 assert!(
3772 !state.wait_active(),
3773 "already-satisfied wait_all must be cleared by generated satisfaction authority"
3774 );
3775 assert!(state.wait_operation_ids().unwrap().is_empty());
3776 }
3777
3778 #[tokio::test]
3779 async fn wait_all_duplicate_rejection_is_generated() {
3780 let registry = RuntimeOpsLifecycleRegistry::new();
3781 let spec = background_spec("duplicate-wait");
3782 let op_id = spec.id.clone();
3783 registry.register_operation(spec).unwrap();
3784 registry.provisioning_succeeded(&op_id).unwrap();
3785
3786 let result = registry
3787 .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3788 .await;
3789
3790 assert!(matches!(
3791 result,
3792 Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3793 ));
3794 let state = registry.read_state().unwrap();
3795 assert!(
3796 !state.wait_active(),
3797 "duplicate wait rejection must not create a shell or machine barrier"
3798 );
3799 assert!(state.wait_operation_ids().unwrap().is_empty());
3800 }
3801
3802 #[tokio::test]
3803 async fn wait_all_active_rejection_is_generated() {
3804 let registry = RuntimeOpsLifecycleRegistry::new();
3805 let spec = background_spec("active-wait");
3806 let op_id = spec.id.clone();
3807 registry.register_operation(spec).unwrap();
3808 registry.provisioning_succeeded(&op_id).unwrap();
3809
3810 let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3811 let result = registry
3812 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3813 .await;
3814
3815 assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3816 drop(active_wait);
3817 let state = registry.read_state().unwrap();
3818 assert!(!state.wait_active());
3819 assert!(state.wait_operation_ids().unwrap().is_empty());
3820 }
3821
3822 #[tokio::test]
3823 async fn wait_all_unknown_operation_rejection_is_generated() {
3824 let registry = RuntimeOpsLifecycleRegistry::new();
3825 let op_id = OperationId::new();
3826
3827 let result = registry
3828 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3829 .await;
3830
3831 assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3832 let state = registry.read_state().unwrap();
3833 assert!(!state.wait_active());
3834 assert!(state.wait_operation_ids().unwrap().is_empty());
3835 }
3836
3837 #[tokio::test]
3838 async fn wait_all_resolves_from_authority_owned_wait_request() {
3839 let registry = RuntimeOpsLifecycleRegistry::new();
3840 let run_id = test_run_id();
3841
3842 let spec = background_spec("pending");
3843 let op_id = spec.id.clone();
3844 registry.register_operation(spec).unwrap();
3845 registry.provisioning_succeeded(&op_id).unwrap();
3846
3847 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3848 tokio::pin!(wait_fut);
3849 assert!(
3850 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3851 .await
3852 .is_err()
3853 );
3854
3855 let active_wait_request_id = {
3856 let state = registry.read_state().unwrap();
3857 let wait_request_id = match state.wait_request_id.clone() {
3858 Some(wait_request_id) => wait_request_id,
3859 None => panic!("wait request should be active"),
3860 };
3861 assert_eq!(
3862 state.wait_operation_ids().unwrap().as_slice(),
3863 std::slice::from_ref(&op_id)
3864 );
3865 wait_request_id
3866 };
3867
3868 registry
3869 .complete_operation(
3870 &op_id,
3871 OperationResult {
3872 id: op_id.clone(),
3873 content: "done".into(),
3874 is_error: false,
3875 duration_ms: 1,
3876 tokens_used: 0,
3877 },
3878 )
3879 .unwrap();
3880
3881 let wait_result = wait_fut.await.unwrap();
3882 assert_eq!(
3883 wait_result.satisfied.wait_request_id,
3884 active_wait_request_id
3885 );
3886 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
3887 assert!(matches!(
3888 wait_result.outcomes.as_slice(),
3889 [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
3890 ));
3891 assert!(registry.read_state().unwrap().wait_request_id.is_none());
3892 }
3893
3894 #[tokio::test]
3895 async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
3896 let registry = RuntimeOpsLifecycleRegistry::new();
3897 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3898 registry.set_persistence_channel(
3899 tx,
3900 meerkat_core::RuntimeEpochId::new(),
3901 Arc::new(meerkat_core::EpochCursorState::new()),
3902 );
3903 let worker = std::thread::spawn(move || {
3904 let request = rx.blocking_recv().expect("persistence request");
3905 let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
3906 "injected persistence failure".to_string(),
3907 )));
3908 });
3909
3910 let spec = background_spec("feed-after-persist");
3911 let op_id = spec.id.clone();
3912 registry.register_operation(spec).unwrap();
3913 registry.provisioning_succeeded(&op_id).unwrap();
3914 let watch = registry.register_watcher(&op_id).unwrap();
3915
3916 let err = registry
3917 .complete_operation(
3918 &op_id,
3919 OperationResult {
3920 id: op_id.clone(),
3921 content: "done".into(),
3922 is_error: false,
3923 duration_ms: 1,
3924 tokens_used: 0,
3925 },
3926 )
3927 .expect_err("persistence failure must fail the terminal transition");
3928 assert!(
3929 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
3930 "unexpected persistence error: {err:?}"
3931 );
3932 worker.join().expect("persistence worker");
3933
3934 let feed = registry.completion_feed_handle();
3935 let batch = feed.list_since(0);
3936 assert!(
3937 batch.entries.is_empty(),
3938 "completion feed must not publish entries before durable snapshot success"
3939 );
3940 assert_eq!(batch.watermark, 0);
3941 assert!(
3942 tokio::time::timeout(std::time::Duration::from_millis(10), watch)
3943 .await
3944 .is_err(),
3945 "watchers must not resolve before durable terminal snapshot success"
3946 );
3947 assert_eq!(
3948 registry.snapshot(&op_id).unwrap().unwrap().status,
3949 OperationStatus::Running,
3950 "failed persistence must roll the generated terminal transition back"
3951 );
3952
3953 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3954 registry.set_persistence_channel(
3955 tx,
3956 meerkat_core::RuntimeEpochId::new(),
3957 Arc::new(meerkat_core::EpochCursorState::new()),
3958 );
3959 let worker = std::thread::spawn(move || {
3960 let request = rx.blocking_recv().expect("second persistence request");
3961 let _ = request.result_tx.send(Ok(()));
3962 });
3963 registry
3964 .complete_operation(
3965 &op_id,
3966 OperationResult {
3967 id: op_id.clone(),
3968 content: "done".into(),
3969 is_error: false,
3970 duration_ms: 1,
3971 tokens_used: 0,
3972 },
3973 )
3974 .expect("rolled-back terminal transition should remain retryable");
3975 worker.join().expect("second persistence worker");
3976
3977 let batch = feed.list_since(0);
3978 assert_eq!(batch.entries.len(), 1);
3979 assert_eq!(batch.entries[0].operation_id, op_id);
3980 }
3981
3982 #[tokio::test]
3983 async fn dropping_wait_all_future_cancels_active_wait_request() {
3984 let registry = RuntimeOpsLifecycleRegistry::new();
3985 let run_id = test_run_id();
3986
3987 let spec = background_spec("cancelled-wait");
3988 let op_id = spec.id.clone();
3989 registry.register_operation(spec).unwrap();
3990 registry.provisioning_succeeded(&op_id).unwrap();
3991
3992 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3993 drop(wait_fut);
3994
3995 let state = registry.read_state().unwrap();
3996 assert!(state.wait_request_id.is_none());
3997 assert!(state.wait_operation_ids().unwrap().is_empty());
3998 assert!(!state.wait_active());
3999 }
4000
4001 #[tokio::test]
4007 async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
4008 let registry = RuntimeOpsLifecycleRegistry::new();
4009 let run_id = test_run_id();
4010
4011 let spec = background_spec("corrupt-barrier");
4012 let op_id = spec.id.clone();
4013 registry.register_operation(spec).unwrap();
4014 registry.provisioning_succeeded(&op_id).unwrap();
4015
4016 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
4018 tokio::pin!(wait_fut);
4019 assert!(
4020 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
4021 .await
4022 .is_err(),
4023 "barrier waiter must still be pending before corruption"
4024 );
4025 assert!(registry.read_state().unwrap().wait_request_id.is_some());
4026
4027 {
4031 let mut state = registry.write_state().unwrap();
4032 let mut machine_state = state.dsl.0.state().clone();
4033 machine_state.wait_run_id = None;
4034 state.dsl = DslAuthority(Box::new(
4035 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4036 ));
4037 }
4038
4039 let err = registry
4042 .complete_operation(
4043 &op_id,
4044 OperationResult {
4045 id: op_id.clone(),
4046 content: "done".into(),
4047 is_error: false,
4048 duration_ms: 1,
4049 tokens_used: 0,
4050 },
4051 )
4052 .expect_err("corrupt wait authority must fail the terminal transition");
4053 assert!(
4054 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
4055 "unexpected terminal error: {err:?}"
4056 );
4057
4058 let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
4060 .await
4061 .expect("waiter must resolve, not hang, after authority corruption");
4062 match waiter_result {
4063 Err(OpsLifecycleError::Internal(message)) => assert!(
4064 message.contains("wait_all completion channel dropped"),
4065 "unexpected waiter error message: {message}"
4066 ),
4067 other => panic!("expected dropped-channel Internal error, got {other:?}"),
4068 }
4069 }
4070
4071 #[test]
4075 fn completion_cursor_propagates_poison_not_none() {
4076 let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
4077
4078 let poison_registry = std::sync::Arc::clone(®istry);
4080 let join = std::thread::spawn(move || {
4081 let _guard = poison_registry.write_state().unwrap();
4082 panic!("intentional panic to poison ops lifecycle registry lock");
4083 });
4084 assert!(
4085 join.join().is_err(),
4086 "poisoning thread must have panicked while holding the write guard"
4087 );
4088
4089 let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
4090 let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
4091 match result {
4092 Err(OpsLifecycleError::Internal(message)) => assert!(
4093 message.contains("ops lifecycle registry poisoned"),
4094 "unexpected cursor error message: {message}"
4095 ),
4096 other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
4097 }
4098 }
4099
4100 #[test]
4101 fn terminate_owner_only_targets_non_terminal_operations() {
4102 let registry = RuntimeOpsLifecycleRegistry::new();
4103
4104 let running_spec = background_spec("running");
4105 let running_id = running_spec.id.clone();
4106 registry.register_operation(running_spec).unwrap();
4107 registry.provisioning_succeeded(&running_id).unwrap();
4108
4109 let completed_spec = background_spec("completed");
4110 let completed_id = completed_spec.id.clone();
4111 registry.register_operation(completed_spec).unwrap();
4112 registry.provisioning_succeeded(&completed_id).unwrap();
4113 registry
4114 .complete_operation(
4115 &completed_id,
4116 OperationResult {
4117 id: completed_id.clone(),
4118 content: "done".into(),
4119 is_error: false,
4120 duration_ms: 1,
4121 tokens_used: 0,
4122 },
4123 )
4124 .unwrap();
4125
4126 registry.terminate_owner("shutdown".into()).unwrap();
4127
4128 assert!(matches!(
4129 registry.snapshot(&running_id).unwrap().unwrap().status,
4130 OperationStatus::Terminated
4131 ));
4132 assert!(matches!(
4133 registry.snapshot(&completed_id).unwrap().unwrap().status,
4134 OperationStatus::Completed
4135 ));
4136 }
4137
4138 #[test]
4139 fn unregister_retirement_fences_detached_late_callbacks() {
4140 let registry = RuntimeOpsLifecycleRegistry::new();
4141 let running_spec = background_spec("detached-running");
4142 let running_id = running_spec.id.clone();
4143 registry.register_operation(running_spec).unwrap();
4144 registry.provisioning_succeeded(&running_id).unwrap();
4145
4146 registry
4147 .retire_owner_for_unregister("owner unregistered".into())
4148 .unwrap();
4149 assert!(matches!(
4150 registry.snapshot(&running_id).unwrap().unwrap().status,
4151 OperationStatus::Terminated
4152 ));
4153
4154 let late_result = OperationResult {
4155 id: running_id.clone(),
4156 content: "late detached completion".into(),
4157 is_error: false,
4158 duration_ms: 1,
4159 tokens_used: 0,
4160 };
4161 assert_eq!(
4162 registry.complete_operation(&running_id, late_result),
4163 Err(OpsLifecycleError::OwnerRetired)
4164 );
4165 assert_eq!(
4166 registry.register_operation(background_spec("late-new-op")),
4167 Err(OpsLifecycleError::OwnerRetired)
4168 );
4169 assert_eq!(
4170 registry.report_progress(
4171 &running_id,
4172 OperationProgressUpdate {
4173 message: "late progress".into(),
4174 percent: None,
4175 },
4176 ),
4177 Err(OpsLifecycleError::OwnerRetired)
4178 );
4179 }
4180
4181 #[test]
4182 fn collect_completed_drains_terminal_operations() {
4183 let registry = RuntimeOpsLifecycleRegistry::new();
4184
4185 let spec_a = background_spec("a");
4186 let id_a = spec_a.id.clone();
4187 registry.register_operation(spec_a).unwrap();
4188 registry.provisioning_succeeded(&id_a).unwrap();
4189 registry
4190 .complete_operation(
4191 &id_a,
4192 OperationResult {
4193 id: id_a.clone(),
4194 content: "done".into(),
4195 is_error: false,
4196 duration_ms: 1,
4197 tokens_used: 0,
4198 },
4199 )
4200 .unwrap();
4201
4202 let spec_b = background_spec("b");
4203 let id_b = spec_b.id.clone();
4204 registry.register_operation(spec_b).unwrap();
4205
4206 let collected = registry.collect_completed().unwrap();
4207 assert_eq!(collected.len(), 1);
4208 assert_eq!(collected[0].0, id_a);
4209
4210 assert!(registry.snapshot(&id_a).unwrap().is_none());
4211 assert!(registry.snapshot(&id_b).unwrap().is_some());
4212
4213 let collected2 = registry.collect_completed().unwrap();
4214 assert!(collected2.is_empty());
4215 }
4216
4217 #[test]
4218 fn bounded_completed_retention_evicts_oldest() {
4219 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4220 max_completed: 3,
4221 max_concurrent: None,
4222 });
4223
4224 let mut ids = Vec::new();
4225 for i in 0..5 {
4226 let spec = background_spec(&format!("op-{i}"));
4227 let id = spec.id.clone();
4228 registry.register_operation(spec).unwrap();
4229 registry.provisioning_succeeded(&id).unwrap();
4230 registry
4231 .complete_operation(
4232 &id,
4233 OperationResult {
4234 id: id.clone(),
4235 content: format!("done-{i}"),
4236 is_error: false,
4237 duration_ms: 1,
4238 tokens_used: 0,
4239 },
4240 )
4241 .unwrap();
4242 ids.push(id);
4243 }
4244
4245 assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4246 assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4247 assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4248 assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4249 assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4250 }
4251
4252 #[test]
4253 fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4254 let registry = RuntimeOpsLifecycleRegistry::new();
4255
4256 let completed_spec = background_spec("completed");
4257 let completed_id = completed_spec.id.clone();
4258 registry.register_operation(completed_spec).unwrap();
4259 registry.provisioning_succeeded(&completed_id).unwrap();
4260 registry
4261 .complete_operation(
4262 &completed_id,
4263 OperationResult {
4264 id: completed_id.clone(),
4265 content: "done".into(),
4266 is_error: false,
4267 duration_ms: 1,
4268 tokens_used: 0,
4269 },
4270 )
4271 .unwrap();
4272
4273 let running_spec = background_spec("running");
4274 let running_id = running_spec.id.clone();
4275 registry.register_operation(running_spec).unwrap();
4276 registry.provisioning_succeeded(&running_id).unwrap();
4277
4278 let cursor_state = meerkat_core::EpochCursorState::new();
4279 let snapshot = registry
4280 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4281 .unwrap();
4282 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4283
4284 assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4285 assert!(recovered.snapshot(&running_id).unwrap().is_none());
4286
4287 let collected = recovered.collect_completed().unwrap();
4288 assert_eq!(collected.len(), 1);
4289 assert_eq!(collected[0].0, completed_id);
4290 }
4291
4292 #[test]
4293 fn recovered_snapshot_retains_running_detached_job_wait_binding() {
4294 let registry = RuntimeOpsLifecycleRegistry::new();
4295 let owner_session_id = SessionId::new();
4296 let job_id = "job_recovery_wait";
4297 let operation_id = OperationId::for_detached_job_wait(&owner_session_id, "realm-a", job_id);
4298 let source = OperationSource::detached_job("realm-a", job_id);
4299 registry
4300 .register_operation(OperationSpec {
4301 id: operation_id.clone(),
4302 kind: OperationKind::DetachedJobWait,
4303 owner_session_id: owner_session_id.clone(),
4304 display_name: "await detached job".into(),
4305 source_label: "await_job".into(),
4306 operation_source: Some(source.clone()),
4307 child_session_id: None,
4308 expect_peer_channel: false,
4309 })
4310 .unwrap();
4311 registry.provisioning_succeeded(&operation_id).unwrap();
4312
4313 let cursor_state = meerkat_core::EpochCursorState::new();
4314 let snapshot = registry
4315 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4316 .unwrap();
4317 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4318 let wait = recovered
4319 .snapshot(&operation_id)
4320 .unwrap()
4321 .expect("running await binding survives recovery");
4322
4323 assert_eq!(wait.status, OperationStatus::Running);
4324 assert_eq!(wait.owner_session_id, owner_session_id);
4325 assert_eq!(wait.operation_source, Some(source));
4326 assert!(wait.terminal_outcome.is_none());
4327 assert!(
4328 recovered
4329 .completion_feed()
4330 .expect("completion feed")
4331 .list_since(0)
4332 .entries
4333 .is_empty(),
4334 "recovery of a wait binding must not mint completion"
4335 );
4336 }
4337
4338 #[test]
4339 fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4340 let registry = RuntimeOpsLifecycleRegistry::new();
4341
4342 let mut spec = background_spec("capacity");
4343 spec.kind = OperationKind::BackgroundToolCapacitySlot;
4344 let operation_id = spec.id.clone();
4345 registry.register_operation(spec).unwrap();
4346 registry.provisioning_succeeded(&operation_id).unwrap();
4347 registry.mark_retired(&operation_id).unwrap();
4348
4349 assert!(registry.snapshot(&operation_id).unwrap().is_none());
4350
4351 let cursor_state = meerkat_core::EpochCursorState::new();
4352 let snapshot = registry
4353 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4354 .unwrap();
4355 assert!(
4356 !snapshot
4357 .authority_state
4358 .operations
4359 .contains_key(&operation_id)
4360 );
4361 assert!(!snapshot.operation_specs.contains_key(&operation_id));
4362 assert!(snapshot.completion_entries.is_empty());
4363
4364 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4365 assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4366 }
4367
4368 #[test]
4369 fn rollback_unreturned_background_operation_restores_absence_without_completion() {
4370 let registry = RuntimeOpsLifecycleRegistry::new();
4371 let spec = background_spec("unreturned");
4372 let operation_id = spec.id.clone();
4373
4374 registry
4375 .register_operation(spec)
4376 .expect("register unreturned operation");
4377 registry
4378 .provisioning_succeeded(&operation_id)
4379 .expect("start unreturned operation");
4380 registry
4381 .rollback_unreturned_operation(&operation_id)
4382 .expect("generated rollback should restore absence");
4383
4384 assert!(
4385 registry
4386 .snapshot(&operation_id)
4387 .expect("snapshot after rollback")
4388 .is_none(),
4389 "never-returned operation must not retain public lifecycle state"
4390 );
4391 let feed = registry
4392 .completion_feed()
4393 .expect("runtime registry exposes completion feed");
4394 assert!(
4395 feed.list_since(0).entries.is_empty(),
4396 "never-returned operation must not mint a phantom completion"
4397 );
4398 }
4399
4400 #[test]
4401 fn recovered_snapshot_uses_authority_operation_source() {
4402 let registry = RuntimeOpsLifecycleRegistry::new();
4403 let child_session_id = SessionId::new();
4404 let operation_source = OperationSource::session_child(child_session_id.clone());
4405 let spec = OperationSpec {
4406 id: OperationId::new(),
4407 kind: OperationKind::MobMemberChild,
4408 owner_session_id: SessionId::new(),
4409 display_name: "source-recovery".into(),
4410 source_label: "test".into(),
4411 operation_source: Some(operation_source.clone()),
4412 child_session_id: Some(child_session_id),
4413 expect_peer_channel: true,
4414 };
4415 let operation_id = spec.id.clone();
4416
4417 registry.register_operation(spec).unwrap();
4418 registry.provisioning_succeeded(&operation_id).unwrap();
4419 registry.mark_retired(&operation_id).unwrap();
4420
4421 let cursor_state = meerkat_core::EpochCursorState::new();
4422 let mut snapshot = registry
4423 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4424 .unwrap();
4425 assert_eq!(
4426 snapshot
4427 .authority_state
4428 .operations
4429 .get(&operation_id)
4430 .and_then(|state| state.operation_source.as_ref()),
4431 Some(&operation_source)
4432 );
4433
4434 snapshot
4435 .operation_specs
4436 .get_mut(&operation_id)
4437 .expect("persisted spec")
4438 .operation_source = None;
4439 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4440 assert_eq!(
4441 recovered
4442 .snapshot(&operation_id)
4443 .unwrap()
4444 .unwrap()
4445 .operation_source,
4446 Some(operation_source)
4447 );
4448 }
4449
4450 #[test]
4451 fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4452 let registry = RuntimeOpsLifecycleRegistry::new();
4453 let child_session_id = SessionId::new();
4454 let operation_source = OperationSource::session_child(child_session_id.clone());
4455 let spec = OperationSpec {
4456 id: OperationId::new(),
4457 kind: OperationKind::MobMemberChild,
4458 owner_session_id: SessionId::new(),
4459 display_name: "source-drift".into(),
4460 source_label: "test".into(),
4461 operation_source: Some(operation_source),
4462 child_session_id: Some(child_session_id),
4463 expect_peer_channel: true,
4464 };
4465 let operation_id = spec.id.clone();
4466
4467 registry.register_operation(spec).unwrap();
4468 registry.provisioning_succeeded(&operation_id).unwrap();
4469 registry.mark_retired(&operation_id).unwrap();
4470
4471 let cursor_state = meerkat_core::EpochCursorState::new();
4472 let mut snapshot = registry
4473 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4474 .unwrap();
4475 snapshot
4476 .operation_specs
4477 .get_mut(&operation_id)
4478 .expect("persisted spec")
4479 .operation_source = Some(OperationSource::session_child(SessionId::new()));
4480
4481 let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4482 .expect_err("source mirror drift must fail recovery");
4483 assert!(
4484 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4485 "unexpected recovery error: {err:?}"
4486 );
4487 }
4488
4489 #[test]
4490 fn persisted_authority_state_serializes_explicit_no_operation_source() {
4491 let registry = RuntimeOpsLifecycleRegistry::new();
4492
4493 let spec = background_spec("explicit-no-source");
4494 let operation_id = spec.id.clone();
4495 registry.register_operation(spec).unwrap();
4496 registry.provisioning_succeeded(&operation_id).unwrap();
4497
4498 let cursor_state = meerkat_core::EpochCursorState::new();
4499 let snapshot = registry
4500 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4501 .unwrap();
4502 let value = serde_json::to_value(&snapshot).unwrap();
4503 let operations = value
4504 .get("authority_state")
4505 .and_then(|state| state.get("operations"))
4506 .and_then(serde_json::Value::as_object)
4507 .expect("serialized authority operations");
4508 let persisted_state = operations
4509 .values()
4510 .next()
4511 .and_then(serde_json::Value::as_object)
4512 .expect("serialized operation state");
4513
4514 assert!(
4515 persisted_state
4516 .get("operation_source")
4517 .is_some_and(serde_json::Value::is_null),
4518 "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4519 );
4520
4521 let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4522 assert_eq!(
4523 recovered_snapshot
4524 .authority_state
4525 .operations
4526 .get(&operation_id)
4527 .expect("round-tripped operation")
4528 .operation_source,
4529 None
4530 );
4531 }
4532
4533 #[test]
4534 fn persisted_authority_state_rejects_missing_operation_source_fact() {
4535 let registry = RuntimeOpsLifecycleRegistry::new();
4536
4537 let spec = background_spec("missing-source-fact");
4538 registry.register_operation(spec).unwrap();
4539
4540 let cursor_state = meerkat_core::EpochCursorState::new();
4541 let snapshot = registry
4542 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4543 .unwrap();
4544 let mut value = serde_json::to_value(&snapshot).unwrap();
4545 let operations = value
4546 .get_mut("authority_state")
4547 .and_then(|state| state.get_mut("operations"))
4548 .and_then(serde_json::Value::as_object_mut)
4549 .expect("serialized authority operations");
4550 let operation_state = operations
4551 .values_mut()
4552 .next()
4553 .and_then(serde_json::Value::as_object_mut)
4554 .expect("serialized operation state");
4555 assert!(operation_state.remove("operation_source").is_some());
4556
4557 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4558 .expect_err("missing generated source fact must fail recovery snapshot decoding");
4559 assert!(
4560 err.to_string().contains("operation_source"),
4561 "unexpected decode error: {err}"
4562 );
4563 }
4564
4565 #[test]
4566 fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4567 let registry = RuntimeOpsLifecycleRegistry::new();
4568
4569 let spec = background_spec("missing-feed-authority");
4570 let operation_id = spec.id.clone();
4571 registry.register_operation(spec).unwrap();
4572 registry.provisioning_succeeded(&operation_id).unwrap();
4573 registry
4574 .complete_operation(
4575 &operation_id,
4576 OperationResult {
4577 id: operation_id.clone(),
4578 content: "done".into(),
4579 is_error: false,
4580 duration_ms: 1,
4581 tokens_used: 0,
4582 },
4583 )
4584 .unwrap();
4585
4586 let cursor_state = meerkat_core::EpochCursorState::new();
4587 let snapshot = registry
4588 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4589 .unwrap();
4590 let mut value = serde_json::to_value(&snapshot).unwrap();
4591 let authority_state = value
4592 .get_mut("authority_state")
4593 .and_then(serde_json::Value::as_object_mut)
4594 .expect("serialized authority state");
4595 assert!(authority_state.remove("completion_feed_entries").is_some());
4596
4597 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4598 .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4599 assert!(
4600 err.to_string().contains("completion_feed_entries"),
4601 "unexpected decode error: {err}"
4602 );
4603 }
4604
4605 #[test]
4606 fn public_child_session_projection_uses_authority_operation_source() {
4607 let registry = RuntimeOpsLifecycleRegistry::new();
4608 let authority_child_session_id = SessionId::new();
4609 let stale_shell_child_session_id = SessionId::new();
4610 let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4611 let spec = OperationSpec {
4612 id: OperationId::new(),
4613 kind: OperationKind::MobMemberChild,
4614 owner_session_id: SessionId::new(),
4615 display_name: "child-projection".into(),
4616 source_label: "test".into(),
4617 operation_source: Some(operation_source),
4618 child_session_id: Some(stale_shell_child_session_id),
4619 expect_peer_channel: true,
4620 };
4621 let operation_id = spec.id.clone();
4622
4623 registry.register_operation(spec).unwrap();
4624
4625 assert_eq!(
4626 registry
4627 .snapshot(&operation_id)
4628 .unwrap()
4629 .unwrap()
4630 .child_session_id,
4631 Some(authority_child_session_id.clone())
4632 );
4633
4634 let cursor_state = meerkat_core::EpochCursorState::new();
4635 let snapshot = registry
4636 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4637 .unwrap();
4638 assert_eq!(
4639 snapshot
4640 .operation_specs
4641 .get(&operation_id)
4642 .expect("persisted spec")
4643 .child_session_id,
4644 Some(authority_child_session_id)
4645 );
4646 }
4647
4648 #[test]
4649 fn generated_terminal_payload_projection_fails_closed() {
4650 let registry = RuntimeOpsLifecycleRegistry::new();
4651
4652 let spec = background_spec("terminal-payload-drift");
4653 let operation_id = spec.id.clone();
4654 registry.register_operation(spec).unwrap();
4655 registry.provisioning_succeeded(&operation_id).unwrap();
4656 registry
4657 .complete_operation(
4658 &operation_id,
4659 OperationResult {
4660 id: operation_id.clone(),
4661 content: "done".into(),
4662 is_error: false,
4663 duration_ms: 1,
4664 tokens_used: 0,
4665 },
4666 )
4667 .unwrap();
4668
4669 {
4670 let mut state = registry.write_state().unwrap();
4671 let mut machine_state = state.dsl.0.state().clone();
4672 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4673 machine_state
4674 .op_terminal_payload
4675 .insert(operation_id_key, OperationTerminalOutcome::Retired);
4676 state.dsl = DslAuthority(Box::new(
4677 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4678 ));
4679 }
4680
4681 let err = match registry.register_watcher(&operation_id) {
4682 Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4683 Err(err) => err,
4684 };
4685 assert!(
4686 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4687 "unexpected watcher error: {err:?}"
4688 );
4689 let err = registry
4690 .snapshot(&operation_id)
4691 .expect_err("invalid generated terminal payload must reject public snapshot");
4692 assert!(
4693 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
4694 "unexpected public snapshot error: {err:?}"
4695 );
4696
4697 let cursor_state = meerkat_core::EpochCursorState::new();
4698 let err = match registry
4699 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4700 {
4701 Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
4702 Err(err) => err,
4703 };
4704 assert!(
4705 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4706 "unexpected snapshot error: {err:?}"
4707 );
4708
4709 let err = match registry.collect_completed() {
4710 Ok(_) => panic!("invalid generated terminal payload must reject collection"),
4711 Err(err) => err,
4712 };
4713 assert!(
4714 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4715 "unexpected collection error: {err:?}"
4716 );
4717 }
4718
4719 #[test]
4720 fn generated_terminal_payload_missing_projection_fails_closed() {
4721 let registry = RuntimeOpsLifecycleRegistry::new();
4722
4723 let spec = background_spec("terminal-payload-missing");
4724 let operation_id = spec.id.clone();
4725 registry.register_operation(spec).unwrap();
4726 registry.provisioning_succeeded(&operation_id).unwrap();
4727 registry
4728 .fail_operation(&operation_id, "boom".into())
4729 .unwrap();
4730
4731 {
4732 let mut state = registry.write_state().unwrap();
4733 let mut machine_state = state.dsl.0.state().clone();
4734 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4735 machine_state.op_terminal_payload.remove(&operation_id_key);
4736 state.dsl = DslAuthority(Box::new(
4737 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4738 ));
4739 }
4740
4741 let err = match registry.register_watcher(&operation_id) {
4742 Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
4743 Err(err) => err,
4744 };
4745 assert!(
4746 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4747 "unexpected watcher error: {err:?}"
4748 );
4749 let err = registry
4750 .snapshot(&operation_id)
4751 .expect_err("missing generated terminal payload must reject public snapshot");
4752 assert!(
4753 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4754 "unexpected public snapshot error: {err:?}"
4755 );
4756 }
4757
4758 #[test]
4759 fn generated_terminal_status_without_outcome_fails_closed() {
4760 let registry = RuntimeOpsLifecycleRegistry::new();
4761
4762 let spec = background_spec("terminal-outcome-missing");
4763 let operation_id = spec.id.clone();
4764 registry.register_operation(spec).unwrap();
4765 registry.provisioning_succeeded(&operation_id).unwrap();
4766 registry
4767 .fail_operation(&operation_id, "boom".into())
4768 .unwrap();
4769
4770 {
4771 let mut state = registry.write_state().unwrap();
4772 let mut machine_state = state.dsl.0.state().clone();
4773 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4774 machine_state.op_terminal_outcomes.remove(&operation_id_key);
4775 state.dsl = DslAuthority(Box::new(
4776 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4777 ));
4778 }
4779
4780 let err = registry
4781 .snapshot(&operation_id)
4782 .expect_err("terminal status without outcome must reject public snapshot");
4783 assert!(
4784 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4785 "unexpected public snapshot error: {err:?}"
4786 );
4787
4788 let err = match registry.collect_completed() {
4789 Ok(_) => panic!("terminal status without outcome must reject collection"),
4790 Err(err) => err,
4791 };
4792 assert!(
4793 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4794 "unexpected collection error: {err:?}"
4795 );
4796 }
4797
4798 #[test]
4799 fn generated_operation_source_projection_fails_closed() {
4800 let registry = RuntimeOpsLifecycleRegistry::new();
4801 let child_session_id = SessionId::new();
4802 let operation_source = OperationSource::session_child(child_session_id.clone());
4803 let spec = OperationSpec {
4804 id: OperationId::new(),
4805 kind: OperationKind::MobMemberChild,
4806 owner_session_id: SessionId::new(),
4807 display_name: "source-authority-drift".into(),
4808 source_label: "test".into(),
4809 operation_source: Some(operation_source),
4810 child_session_id: Some(child_session_id),
4811 expect_peer_channel: true,
4812 };
4813 let operation_id = spec.id.clone();
4814
4815 registry.register_operation(spec).unwrap();
4816
4817 {
4818 let mut state = registry.write_state().unwrap();
4819 let mut machine_state = state.dsl.0.state().clone();
4820 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4821 machine_state
4822 .op_sources
4823 .get_mut(&operation_id_key)
4824 .expect("generated operation source")
4825 .session_id = None;
4826 state.dsl = DslAuthority(Box::new(
4827 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4828 ));
4829 }
4830
4831 let err = registry
4832 .snapshot(&operation_id)
4833 .expect_err("invalid generated operation source must reject public snapshot");
4834 assert!(
4835 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4836 "unexpected public snapshot error: {err:?}"
4837 );
4838 let err = registry
4839 .list_operations()
4840 .expect_err("invalid generated operation source must reject public operation list");
4841 assert!(
4842 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4843 "unexpected operation list error: {err:?}"
4844 );
4845
4846 let cursor_state = meerkat_core::EpochCursorState::new();
4847 let err = match registry
4848 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4849 {
4850 Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
4851 Err(err) => err,
4852 };
4853 assert!(
4854 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4855 "unexpected snapshot error: {err:?}"
4856 );
4857 }
4858
4859 #[test]
4860 fn generated_operation_id_projection_fails_closed() {
4861 let registry = RuntimeOpsLifecycleRegistry::new();
4862
4863 {
4864 let mut state = registry.write_state().unwrap();
4865 let mut machine_state = state.dsl.0.state().clone();
4866 machine_state.op_statuses.insert(
4867 "not-json-operation-id".into(),
4868 mm_dsl::OperationStatus::Running,
4869 );
4870 state.dsl = DslAuthority(Box::new(
4871 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4872 ));
4873 }
4874
4875 let err = registry
4876 .list_operations()
4877 .expect_err("invalid generated operation id must reject public operation list");
4878 assert!(
4879 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4880 "unexpected operation list error: {err:?}"
4881 );
4882
4883 let cursor_state = meerkat_core::EpochCursorState::new();
4884 let err = match registry
4885 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4886 {
4887 Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
4888 Err(err) => err,
4889 };
4890 assert!(
4891 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4892 "unexpected persistence snapshot error: {err:?}"
4893 );
4894 }
4895
4896 #[test]
4897 fn generated_missing_kind_projection_fails_closed() {
4898 let registry = RuntimeOpsLifecycleRegistry::new();
4899 let spec = background_spec("missing-kind");
4900 let operation_id = spec.id.clone();
4901 registry.register_operation(spec).unwrap();
4902
4903 {
4904 let mut state = registry.write_state().unwrap();
4905 let mut machine_state = state.dsl.0.state().clone();
4906 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4907 machine_state.op_kinds.remove(&operation_id_key);
4908 state.dsl = DslAuthority(Box::new(
4909 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4910 ));
4911 }
4912
4913 let err = registry
4914 .snapshot(&operation_id)
4915 .expect_err("missing generated kind must reject public snapshot");
4916 assert!(
4917 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4918 "unexpected public snapshot error: {err:?}"
4919 );
4920 let err = registry
4921 .list_operations()
4922 .expect_err("missing generated kind must reject public list");
4923 assert!(
4924 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4925 "unexpected public list error: {err:?}"
4926 );
4927
4928 let cursor_state = meerkat_core::EpochCursorState::new();
4929 let err = registry
4930 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4931 .expect_err("missing generated kind must reject persistence snapshot");
4932 assert!(
4933 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4934 "unexpected persistence snapshot error: {err:?}"
4935 );
4936 }
4937
4938 #[test]
4939 fn generated_missing_status_projection_fails_closed() {
4940 let registry = RuntimeOpsLifecycleRegistry::new();
4941 let spec = background_spec("missing-status");
4942 let operation_id = spec.id.clone();
4943 registry.register_operation(spec).unwrap();
4944
4945 {
4946 let mut state = registry.write_state().unwrap();
4947 let mut machine_state = state.dsl.0.state().clone();
4948 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4949 machine_state.op_statuses.remove(&operation_id_key);
4950 state.dsl = DslAuthority(Box::new(
4951 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4952 ));
4953 }
4954
4955 let err = registry
4956 .snapshot(&operation_id)
4957 .expect_err("missing generated status must reject public snapshot");
4958 assert!(
4959 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4960 "unexpected public snapshot error: {err:?}"
4961 );
4962 let err = registry
4963 .list_operations()
4964 .expect_err("missing generated status must reject public list");
4965 assert!(
4966 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4967 "unexpected public list error: {err:?}"
4968 );
4969 let err = registry
4970 .classify_operation_public_result(&operation_id)
4971 .expect_err("missing generated status must reject public-result classification");
4972 assert!(
4973 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4974 "unexpected public-result error: {err:?}"
4975 );
4976 }
4977
4978 #[test]
4979 fn generated_retiring_public_result_remains_running_until_terminal() {
4980 let registry = RuntimeOpsLifecycleRegistry::new();
4981 let spec = background_spec("retiring-public-result");
4982 let operation_id = spec.id.clone();
4983 registry.register_operation(spec).unwrap();
4984 registry.provisioning_succeeded(&operation_id).unwrap();
4985 registry.request_retire(&operation_id).unwrap();
4986
4987 let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
4988 assert_eq!(snapshot.status, OperationStatus::Retiring);
4989 assert!(snapshot.terminal_outcome.is_none());
4990 assert!(!snapshot.terminal);
4991 assert_eq!(
4992 snapshot.public_result_class,
4993 OperationPublicResultClass::Running
4994 );
4995 assert_eq!(
4996 registry
4997 .classify_operation_public_result(&operation_id)
4998 .unwrap(),
4999 OperationPublicResultClass::Running
5000 );
5001 }
5002
5003 #[test]
5004 fn generated_missing_peer_ready_projection_fails_closed() {
5005 let registry = RuntimeOpsLifecycleRegistry::new();
5006 let spec = background_spec("missing-peer-ready");
5007 let operation_id = spec.id.clone();
5008 registry.register_operation(spec).unwrap();
5009
5010 {
5011 let mut state = registry.write_state().unwrap();
5012 let mut machine_state = state.dsl.0.state().clone();
5013 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5014 machine_state.op_peer_ready.remove(&operation_id_key);
5015 state.dsl = DslAuthority(Box::new(
5016 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5017 ));
5018 }
5019
5020 let err = registry
5021 .snapshot(&operation_id)
5022 .expect_err("missing generated peer-ready fact must reject public snapshot");
5023 assert!(
5024 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
5025 "unexpected public snapshot error: {err:?}"
5026 );
5027
5028 let cursor_state = meerkat_core::EpochCursorState::new();
5029 let err = registry
5030 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5031 .expect_err("missing generated peer-ready fact must reject persistence snapshot");
5032 assert!(
5033 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
5034 "unexpected persistence snapshot error: {err:?}"
5035 );
5036 }
5037
5038 #[test]
5039 fn generated_missing_progress_count_projection_fails_closed() {
5040 let registry = RuntimeOpsLifecycleRegistry::new();
5041 let spec = background_spec("missing-progress-count");
5042 let operation_id = spec.id.clone();
5043 registry.register_operation(spec).unwrap();
5044
5045 {
5046 let mut state = registry.write_state().unwrap();
5047 let mut machine_state = state.dsl.0.state().clone();
5048 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5049 machine_state.op_progress_counts.remove(&operation_id_key);
5050 state.dsl = DslAuthority(Box::new(
5051 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5052 ));
5053 }
5054
5055 let err = registry
5056 .snapshot(&operation_id)
5057 .expect_err("missing generated progress count must reject public snapshot");
5058 assert!(
5059 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
5060 "unexpected public snapshot error: {err:?}"
5061 );
5062
5063 let cursor_state = meerkat_core::EpochCursorState::new();
5064 let err = registry
5065 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5066 .expect_err("missing generated progress count must reject persistence snapshot");
5067 assert!(
5068 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
5069 "unexpected persistence snapshot error: {err:?}"
5070 );
5071 }
5072
5073 #[test]
5074 fn generated_terminal_sequence_missing_persistence_fails_closed() {
5075 let registry = RuntimeOpsLifecycleRegistry::new();
5076 let spec = background_spec("terminal-sequence-missing");
5077 let operation_id = spec.id.clone();
5078 registry.register_operation(spec).unwrap();
5079 registry.provisioning_succeeded(&operation_id).unwrap();
5080 registry
5081 .complete_operation(
5082 &operation_id,
5083 OperationResult {
5084 id: operation_id.clone(),
5085 content: "done".into(),
5086 is_error: false,
5087 duration_ms: 1,
5088 tokens_used: 0,
5089 },
5090 )
5091 .unwrap();
5092
5093 {
5094 let mut state = registry.write_state().unwrap();
5095 let mut machine_state = state.dsl.0.state().clone();
5096 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5097 machine_state.op_completion_seq.remove(&operation_id_key);
5098 state.dsl = DslAuthority(Box::new(
5099 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5100 ));
5101 }
5102
5103 let cursor_state = meerkat_core::EpochCursorState::new();
5104 let err = registry
5105 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5106 .expect_err("missing generated terminal sequence must reject persistence snapshot");
5107 assert!(
5108 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
5109 "unexpected persistence snapshot error: {err:?}"
5110 );
5111 }
5112
5113 #[test]
5114 fn generated_record_without_shell_projection_fails_closed() {
5115 let registry = RuntimeOpsLifecycleRegistry::new();
5116 let spec = background_spec("missing-shell-record");
5117 let operation_id = spec.id.clone();
5118 registry.register_operation(spec).unwrap();
5119
5120 {
5121 let mut state = registry.write_state().unwrap();
5122 state.records.remove(&operation_id);
5123 }
5124
5125 let err = registry
5126 .snapshot(&operation_id)
5127 .expect_err("generated operation without shell record must reject public snapshot");
5128 assert!(
5129 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5130 "unexpected public snapshot error: {err:?}"
5131 );
5132 let err = registry
5133 .list_operations()
5134 .expect_err("generated operation without shell record must reject public list");
5135 assert!(
5136 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5137 "unexpected public list error: {err:?}"
5138 );
5139 }
5140
5141 #[test]
5142 fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
5143 let registry = RuntimeOpsLifecycleRegistry::new();
5144 let child_session_id = SessionId::new();
5145 let operation_source = OperationSource::session_child(child_session_id.clone());
5146 let spec = OperationSpec {
5147 id: OperationId::new(),
5148 kind: OperationKind::MobMemberChild,
5149 owner_session_id: SessionId::new(),
5150 display_name: "child-drift".into(),
5151 source_label: "test".into(),
5152 operation_source: Some(operation_source),
5153 child_session_id: Some(child_session_id.clone()),
5154 expect_peer_channel: true,
5155 };
5156 let operation_id = spec.id.clone();
5157
5158 registry.register_operation(spec).unwrap();
5159 registry.provisioning_succeeded(&operation_id).unwrap();
5160 registry.mark_retired(&operation_id).unwrap();
5161
5162 let cursor_state = meerkat_core::EpochCursorState::new();
5163 let mut snapshot = registry
5164 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5165 .unwrap();
5166 snapshot
5167 .operation_specs
5168 .get_mut(&operation_id)
5169 .expect("persisted spec")
5170 .child_session_id = Some(SessionId::new());
5171
5172 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5173 assert_eq!(
5174 recovered
5175 .snapshot(&operation_id)
5176 .unwrap()
5177 .unwrap()
5178 .child_session_id,
5179 Some(child_session_id)
5180 );
5181 }
5182
5183 #[test]
5184 fn completion_wake_class_is_generated_by_operation_kind() {
5185 let registry = RuntimeOpsLifecycleRegistry::new();
5186 let operation_id = OperationId::new();
5187
5188 assert_eq!(
5189 registry
5190 .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
5191 .unwrap(),
5192 OperationCompletionWakeClass::Wake
5193 );
5194 assert_eq!(
5195 registry
5196 .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
5197 .unwrap(),
5198 OperationCompletionWakeClass::Ignore
5199 );
5200 assert_eq!(
5201 registry
5202 .classify_operation_completion_wake(
5203 &operation_id,
5204 OperationKind::BackgroundToolCapacitySlot,
5205 )
5206 .unwrap(),
5207 OperationCompletionWakeClass::Ignore
5208 );
5209 }
5210
5211 #[test]
5212 fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
5213 let registry = RuntimeOpsLifecycleRegistry::new();
5214
5215 let running_spec = background_spec("running");
5216 let running_id = running_spec.id.clone();
5217 registry.register_operation(running_spec.clone()).unwrap();
5218 registry.provisioning_succeeded(&running_id).unwrap();
5219
5220 let cursor_state = meerkat_core::EpochCursorState::new();
5221 let mut snapshot = registry
5222 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5223 .unwrap();
5224 snapshot.completion_entries.push(CompletionEntry {
5225 seq: 1,
5226 operation_id: running_id.clone(),
5227 kind: running_spec.kind,
5228 display_name: running_spec.display_name,
5229 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5230 id: running_id,
5231 content: "phantom".into(),
5232 is_error: false,
5233 duration_ms: 1,
5234 tokens_used: 0,
5235 }),
5236 completed_at_ms: None,
5237 });
5238
5239 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5240 Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5241 Err(err) => err,
5242 };
5243 assert!(
5244 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5245 "unexpected recovery error: {err:?}"
5246 );
5247 }
5248
5249 #[test]
5250 fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5251 let registry = RuntimeOpsLifecycleRegistry::new();
5252 let phantom_id = OperationId::new();
5253 {
5254 let state = registry.read_state().unwrap();
5255 state.feed_buffer.push(CompletionEntry {
5256 seq: 1,
5257 operation_id: phantom_id.clone(),
5258 kind: OperationKind::BackgroundToolOp,
5259 display_name: "phantom".into(),
5260 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5261 id: phantom_id,
5262 content: "phantom".into(),
5263 is_error: false,
5264 duration_ms: 1,
5265 tokens_used: 0,
5266 }),
5267 completed_at_ms: None,
5268 });
5269 }
5270
5271 let cursor_state = meerkat_core::EpochCursorState::new();
5272 let err = match registry
5273 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5274 {
5275 Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5276 Err(err) => err,
5277 };
5278 assert!(
5279 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5280 "unexpected capture error: {err:?}"
5281 );
5282 }
5283
5284 #[test]
5285 fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5286 let registry = RuntimeOpsLifecycleRegistry::new();
5287
5288 let spec = background_spec("terminal");
5289 let operation_id = spec.id.clone();
5290 registry.register_operation(spec).unwrap();
5291 registry.provisioning_succeeded(&operation_id).unwrap();
5292 registry
5293 .complete_operation(
5294 &operation_id,
5295 OperationResult {
5296 id: operation_id.clone(),
5297 content: "done".into(),
5298 is_error: false,
5299 duration_ms: 1,
5300 tokens_used: 0,
5301 },
5302 )
5303 .unwrap();
5304
5305 let cursor_state = meerkat_core::EpochCursorState::new();
5306 let mut snapshot = registry
5307 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5308 .unwrap();
5309 let phantom_id = OperationId::new();
5310 let phantom_result = OperationResult {
5311 id: phantom_id.clone(),
5312 content: "phantom".into(),
5313 is_error: false,
5314 duration_ms: 1,
5315 tokens_used: 0,
5316 };
5317 let phantom_entry = CompletionFeedCanonicalState {
5318 seq: snapshot.authority_state.next_completion_seq + 1,
5319 kind: OperationKind::BackgroundToolOp,
5320 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5321 };
5322 snapshot
5323 .authority_state
5324 .completion_feed_entries
5325 .insert(phantom_id.clone(), phantom_entry);
5326 snapshot.completion_entries.push(CompletionEntry {
5327 seq: snapshot.authority_state.next_completion_seq + 1,
5328 operation_id: phantom_id.clone(),
5329 kind: OperationKind::BackgroundToolOp,
5330 display_name: "phantom".into(),
5331 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5332 completed_at_ms: None,
5333 });
5334
5335 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5336 Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5337 Err(err) => err,
5338 };
5339 assert!(
5340 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5341 "unexpected recovery error: {err:?}"
5342 );
5343 }
5344
5345 #[test]
5346 fn recovered_completed_order_uses_generated_completion_sequences() {
5347 let registry = RuntimeOpsLifecycleRegistry::new();
5348
5349 let spec_a = background_spec("a");
5350 let id_a = spec_a.id.clone();
5351 registry.register_operation(spec_a).unwrap();
5352 registry.provisioning_succeeded(&id_a).unwrap();
5353 registry
5354 .complete_operation(
5355 &id_a,
5356 OperationResult {
5357 id: id_a.clone(),
5358 content: "a".into(),
5359 is_error: false,
5360 duration_ms: 1,
5361 tokens_used: 0,
5362 },
5363 )
5364 .unwrap();
5365
5366 let spec_b = background_spec("b");
5367 let id_b = spec_b.id.clone();
5368 registry.register_operation(spec_b).unwrap();
5369 registry.provisioning_succeeded(&id_b).unwrap();
5370 registry
5371 .complete_operation(
5372 &id_b,
5373 OperationResult {
5374 id: id_b.clone(),
5375 content: "b".into(),
5376 is_error: false,
5377 duration_ms: 1,
5378 tokens_used: 0,
5379 },
5380 )
5381 .unwrap();
5382
5383 let cursor_state = meerkat_core::EpochCursorState::new();
5384 let mut snapshot = registry
5385 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5386 .unwrap();
5387 snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5388
5389 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5390 let collected = recovered.collect_completed().unwrap();
5391
5392 assert_eq!(collected[0].0, id_a);
5393 assert_eq!(collected[1].0, id_b);
5394 }
5395
5396 #[test]
5397 fn max_concurrent_enforcement() {
5398 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5399 max_completed: DEFAULT_MAX_COMPLETED,
5400 max_concurrent: Some(2),
5401 });
5402
5403 let spec_a = background_spec("a");
5404 let id_a = spec_a.id.clone();
5405 registry.register_operation(spec_a).unwrap();
5406
5407 let spec_b = background_spec("b");
5408 registry.register_operation(spec_b).unwrap();
5409
5410 let spec_c = background_spec("c");
5411 let result = registry.register_operation(spec_c);
5412 assert!(matches!(
5413 result,
5414 Err(OpsLifecycleError::MaxConcurrentExceeded {
5415 limit: 2,
5416 active: 2,
5417 })
5418 ));
5419
5420 registry.provisioning_succeeded(&id_a).unwrap();
5421 registry
5422 .complete_operation(
5423 &id_a,
5424 OperationResult {
5425 id: id_a.clone(),
5426 content: "done".into(),
5427 is_error: false,
5428 duration_ms: 1,
5429 tokens_used: 0,
5430 },
5431 )
5432 .unwrap();
5433
5434 let spec_d = background_spec("d");
5435 assert!(registry.register_operation(spec_d).is_ok());
5436 }
5437
5438 #[test]
5439 fn snapshot_includes_timestamps() {
5440 let registry = RuntimeOpsLifecycleRegistry::new();
5441 let spec = background_spec("timed");
5442 let op_id = spec.id.clone();
5443 registry.register_operation(spec).unwrap();
5444
5445 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5446 assert!(snap1.created_at_ms > 0);
5447 assert!(snap1.started_at_ms.is_none());
5448 assert!(snap1.completed_at_ms.is_none());
5449 assert!(snap1.elapsed_ms.is_none());
5450
5451 registry.provisioning_succeeded(&op_id).unwrap();
5452 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5453 assert!(snap2.started_at_ms.is_some());
5454 assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5455
5456 registry
5457 .complete_operation(
5458 &op_id,
5459 OperationResult {
5460 id: op_id.clone(),
5461 content: "done".into(),
5462 is_error: false,
5463 duration_ms: 1,
5464 tokens_used: 0,
5465 },
5466 )
5467 .unwrap();
5468 let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5469 assert!(snap3.completed_at_ms.is_some());
5470 assert!(snap3.elapsed_ms.is_some());
5471 assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5472 }
5473
5474 #[test]
5475 fn snapshot_includes_peer_handle() {
5476 let registry = RuntimeOpsLifecycleRegistry::new();
5477 let child_session_id = SessionId::new();
5478 let spec = OperationSpec {
5479 id: OperationId::new(),
5480 kind: OperationKind::MobMemberChild,
5481 owner_session_id: SessionId::new(),
5482 display_name: "peer-test".into(),
5483 source_label: "test".into(),
5484 operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5485 child_session_id: Some(child_session_id),
5486 expect_peer_channel: true,
5487 };
5488 let op_id = spec.id.clone();
5489 registry.register_operation(spec).unwrap();
5490 registry.provisioning_succeeded(&op_id).unwrap();
5491
5492 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5493 assert!(snap1.peer_handle.is_none());
5494
5495 let handle = OperationPeerHandle {
5496 peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5497 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5498 "member-x",
5499 PeerId::new(),
5500 "inproc://x",
5501 )
5502 .unwrap(),
5503 };
5504 registry.peer_ready(&op_id, handle).unwrap();
5505
5506 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5507 assert_eq!(
5508 snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5509 "member-x"
5510 );
5511 }
5512}