1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24 CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31 CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32 OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33 OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34 OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35 OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36 WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43fn deserialize_required_operation_source<'de, D>(
54 deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57 D: serde::Deserializer<'de>,
58{
59 <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65 status: OperationStatus,
66 kind: OperationKind,
67 #[serde(deserialize_with = "deserialize_required_operation_source")]
68 operation_source: Option<OperationSource>,
69 peer_ready: bool,
70 progress_count: u32,
71 watcher_count: u32,
72 terminal_outcome: Option<OperationTerminalOutcome>,
73 #[serde(default, skip_serializing_if = "Option::is_none")]
74 completion_sequence: Option<CompletionSeq>,
75 terminal_buffered: bool,
76}
77
78#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81 seq: CompletionSeq,
82 kind: OperationKind,
83 terminal_outcome: OperationTerminalOutcome,
84}
85
86#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89 operations: HashMap<OperationId, OperationCanonicalState>,
90 completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91 completed_order: VecDeque<OperationId>,
92 max_completed: usize,
93 max_concurrent: Option<usize>,
94 active_count: usize,
95 wait_request_id: Option<WaitRequestId>,
96 wait_operation_ids: Vec<OperationId>,
97 next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101 pub fn max_completed(&self) -> usize {
103 self.max_completed
104 }
105
106 pub fn max_concurrent(&self) -> Option<usize> {
108 self.max_concurrent
109 }
110
111 pub fn operation_count(&self) -> usize {
113 self.operations.len()
114 }
115
116 pub fn completion_feed_count(&self) -> usize {
118 self.completion_feed_entries.len()
119 }
120}
121
122#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134 pub epoch_id: meerkat_core::RuntimeEpochId,
136 pub authority_state: RegistryCanonicalState,
138 pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140 pub completion_entries: Vec<CompletionEntry>,
143 pub cursors: meerkat_core::EpochCursorSnapshot,
145}
146
147#[derive(Debug)]
148pub struct OpsLifecyclePersistenceRequest {
149 snapshot: PersistedOpsSnapshot,
150 result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
151}
152
153impl OpsLifecyclePersistenceRequest {
154 pub fn snapshot(&self) -> &PersistedOpsSnapshot {
155 &self.snapshot
156 }
157
158 pub fn complete(self, result: Result<(), OpsLifecycleError>) {
159 let _ = self.result_tx.send(result);
160 }
161}
162
163#[derive(Debug)]
172struct FeedBufferInner {
173 entries: VecDeque<CompletionEntry>,
174 watermark: CompletionSeq,
175 max_retained: usize,
176}
177
178#[derive(Debug)]
183struct FeedBuffer {
184 inner: RwLock<FeedBufferInner>,
185 watermark_atomic: AtomicU64,
187 notify: tokio::sync::Notify,
189}
190
191impl FeedBuffer {
192 fn new(max_retained: usize) -> Self {
193 Self {
194 inner: RwLock::new(FeedBufferInner {
195 entries: VecDeque::new(),
196 watermark: 0,
197 max_retained,
198 }),
199 watermark_atomic: AtomicU64::new(0),
200 notify: tokio::sync::Notify::new(),
201 }
202 }
203
204 fn push(&self, entry: CompletionEntry) {
205 let mut inner = self
206 .inner
207 .write()
208 .unwrap_or_else(std::sync::PoisonError::into_inner);
209 let seq = entry.seq;
210 inner.entries.push_back(entry);
211 inner.watermark = seq;
212
213 while inner.entries.len() > inner.max_retained {
215 inner.entries.pop_front();
216 }
217
218 drop(inner);
219
220 self.watermark_atomic.store(seq, Ordering::Release);
221 self.notify.notify_waiters();
222 }
223}
224
225#[derive(Debug, Clone)]
230pub struct RuntimeCompletionFeed {
231 buffer: Arc<FeedBuffer>,
232}
233
234impl CompletionFeed for RuntimeCompletionFeed {
235 fn watermark(&self) -> CompletionSeq {
236 self.buffer.watermark_atomic.load(Ordering::Acquire)
237 }
238
239 fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
240 let inner = self
241 .buffer
242 .inner
243 .read()
244 .unwrap_or_else(std::sync::PoisonError::into_inner);
245 let entries: Vec<CompletionEntry> = inner
246 .entries
247 .iter()
248 .filter(|e| e.seq > after_seq)
249 .cloned()
250 .collect();
251 let watermark = inner.watermark;
252 CompletionBatch { entries, watermark }
253 }
254
255 fn wait_for_advance(
256 &self,
257 after_seq: CompletionSeq,
258 ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
259 Box::pin(async move {
260 loop {
261 let notified = self.buffer.notify.notified();
266 let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
267 if current > after_seq {
268 return current;
269 }
270 notified.await;
271 }
272 })
273 }
274}
275
276#[derive(Debug)]
281struct OperationCompletionNotifier {
282 tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
283}
284
285impl OperationCompletionNotifier {
286 fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
287 Self { tx }
288 }
289
290 fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
291 let _ = self.tx.send(outcome.clone());
292 }
293}
294
295fn operation_completion_watch_from_receiver(
296 rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
297) -> OperationCompletionWatch {
298 Box::pin(async move {
299 rx.await
300 .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
301 })
302}
303
304fn resolved_operation_completion_watch(
305 outcome: OperationTerminalOutcome,
306) -> OperationCompletionWatch {
307 Box::pin(async move { Ok(outcome) })
308}
309
310#[derive(Debug)]
314struct ShellRecord {
315 spec: OperationSpec,
316 peer_handle: Option<OperationPeerHandle>,
317 watchers: Vec<OperationCompletionNotifier>,
321 created_at: Instant,
323 started_at: Option<Instant>,
324 completed_at: Option<Instant>,
325 created_at_wall: SystemTime,
327}
328
329#[derive(Debug)]
330struct PendingWaitState {
331 wait_request_id: WaitRequestId,
332 sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
333}
334
335enum WaitAllAuthorityPlan {
336 AlreadySatisfied(WaitAllSatisfied),
337 ActivateBarrier,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq)]
341enum RecoveredOperationRecordDisposition {
342 Retain,
343 Discard,
344}
345
346impl ShellRecord {
347 fn new(spec: OperationSpec) -> Self {
348 Self {
349 spec,
350 peer_handle: None,
351 watchers: Vec::new(),
352 created_at: Instant::now(),
353 started_at: None,
354 completed_at: None,
355 created_at_wall: SystemTime::now(),
356 }
357 }
358
359 fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
360 wall_anchor
361 .duration_since(UNIX_EPOCH)
362 .map(|d| d.as_millis() as u64)
363 .unwrap_or(0)
364 }
365
366 fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
367 let offset = instant.saturating_duration_since(self.created_at);
370 let wall = self.created_at_wall + offset;
371 Self::epoch_millis(&wall)
372 }
373
374 fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
376 for watcher in std::mem::take(&mut self.watchers) {
377 watcher.notify_after_generated_terminal(outcome);
378 }
379 }
380
381 fn mark_completed(&mut self) {
383 self.completed_at = Some(Instant::now());
384 }
385}
386
387#[derive(Debug)]
392struct ShellState {
393 dsl: DslAuthority,
395 records: HashMap<OperationId, ShellRecord>,
397 pending_wait: Option<PendingWaitState>,
399 completed_order: VecDeque<OperationId>,
401 max_completed: usize,
403 max_concurrent: Option<usize>,
405 wait_request_id: Option<WaitRequestId>,
411 feed_buffer: Arc<FeedBuffer>,
413 persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
415 persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
417 persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
419 owner_retired: bool,
425}
426
427struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
433
434impl std::fmt::Debug for DslAuthority {
435 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
436 f.debug_struct("DslAuthority")
437 .field("state", self.0.state())
438 .finish()
439 }
440}
441
442fn new_ops_dsl_authority() -> DslAuthority {
446 DslAuthority(Box::new(
447 crate::meerkat_machine::dsl_authority::new_initialized_authority(
448 "ops lifecycle DSL Initialize must be accepted",
449 ),
450 ))
451}
452
453impl ShellState {
454 fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
455 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
456 let dsl = new_ops_dsl_authority();
457 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
458 let feed_capacity = max_completed.saturating_mul(4).max(1024);
459 tracing::info!(
460 feed_capacity,
461 "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
462 );
463 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
464 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
465 Self {
466 dsl,
467 records: HashMap::new(),
468 pending_wait: None,
469 completed_order: VecDeque::new(),
470 max_completed,
471 max_concurrent,
472 wait_request_id: None,
473 feed_buffer,
478 persist_tx: None,
479 persist_epoch_id: None,
480 persist_cursor_state: None,
481 owner_retired: false,
482 }
483 }
484
485 fn ensure_owner_active(&self) -> Result<(), OpsLifecycleError> {
486 if self.owner_retired {
487 Err(OpsLifecycleError::OwnerRetired)
488 } else {
489 Ok(())
490 }
491 }
492
493 fn dsl_apply(
500 &mut self,
501 input: mm_dsl::MeerkatMachineInput,
502 context: &str,
503 ) -> Result<(), OpsLifecycleError> {
504 self.dsl_apply_raw(input).map_err(|err| {
505 OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
506 })
507 }
508
509 fn dsl_apply_raw(
516 &mut self,
517 input: mm_dsl::MeerkatMachineInput,
518 ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
519 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
520 }
521
522 fn dsl_apply_with_effects(
523 &mut self,
524 input: mm_dsl::MeerkatMachineInput,
525 context: &str,
526 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
527 let transition =
528 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
529 OpsLifecycleError::Internal(format!(
530 "DSL rejected ops transition ({context}): {err:?}"
531 ))
532 })?;
533 Ok(transition.into_effects())
534 }
535
536 fn checked_terminal_payload(
541 kind: mm_dsl::OperationTerminalOutcomeKind,
542 payload: &OperationTerminalOutcome,
543 authority: &str,
544 operation_id: &str,
545 ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
546 if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
547 return Err(OpsLifecycleError::Internal(format!(
548 "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
549 )));
550 }
551 Ok(payload.clone())
552 }
553
554 fn status(&self, id: &OperationId) -> Option<OperationStatus> {
556 let id_key = mm_dsl::OperationId::from_domain(id).0;
557 self.dsl
558 .0
559 .state()
560 .op_statuses
561 .get(&id_key)
562 .copied()
563 .map(OperationStatus::from)
564 }
565
566 fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
567 self.status(id).ok_or_else(|| {
568 OpsLifecycleError::Internal(format!(
569 "generated op lifecycle authority missing status for {id}"
570 ))
571 })
572 }
573
574 fn kind(&self, id: &OperationId) -> Option<OperationKind> {
576 let id_key = mm_dsl::OperationId::from_domain(id).0;
577 self.dsl
578 .0
579 .state()
580 .op_kinds
581 .get(&id_key)
582 .copied()
583 .map(OperationKind::from)
584 }
585
586 fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
587 self.kind(id).ok_or_else(|| {
588 OpsLifecycleError::Internal(format!(
589 "generated op lifecycle authority missing kind for {id}"
590 ))
591 })
592 }
593
594 fn operation_source(
595 &self,
596 id: &OperationId,
597 ) -> Result<Option<OperationSource>, OpsLifecycleError> {
598 let id_key = mm_dsl::OperationId::from_domain(id).0;
599 self.dsl
600 .0
601 .state()
602 .op_sources
603 .get(&id_key)
604 .map(|source| {
605 source.to_domain().map_err(|error| {
606 OpsLifecycleError::Internal(format!(
607 "generated operation source authority has invalid source for {id}: {error}"
608 ))
609 })
610 })
611 .transpose()
612 }
613
614 fn child_session_id_from_operation_source(
615 operation_source: Option<&OperationSource>,
616 ) -> Option<SessionId> {
617 match operation_source {
618 Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
619 Some(OperationSource::BackendPeer { .. }) | None => None,
620 }
621 }
622
623 fn align_spec_child_session_id_to_source(
624 spec: &mut OperationSpec,
625 operation_source: Option<&OperationSource>,
626 ) {
627 spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
628 }
629
630 fn peer_ready(&self, id: &OperationId) -> Option<bool> {
632 let id_key = mm_dsl::OperationId::from_domain(id).0;
633 self.dsl.0.state().op_peer_ready.get(&id_key).copied()
634 }
635
636 fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
637 self.peer_ready(id).ok_or_else(|| {
638 OpsLifecycleError::Internal(format!(
639 "generated op peer wiring authority missing peer-ready fact for {id}"
640 ))
641 })
642 }
643
644 fn progress_count(&self, id: &OperationId) -> Option<u32> {
646 let id_key = mm_dsl::OperationId::from_domain(id).0;
647 self.dsl
648 .0
649 .state()
650 .op_progress_counts
651 .get(&id_key)
652 .map(|v| (*v).min(u32::MAX as u64) as u32)
653 }
654
655 fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
656 self.progress_count(id).ok_or_else(|| {
657 OpsLifecycleError::Internal(format!(
658 "generated op progress authority missing progress count for {id}"
659 ))
660 })
661 }
662
663 fn terminal_outcome(
667 &self,
668 id: &OperationId,
669 ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
670 let id_key = mm_dsl::OperationId::from_domain(id).0;
671 let state = self.dsl.0.state();
672 let status = self.status(id);
673 let terminal = match status {
674 Some(status) => Self::operation_status_is_terminal(id, status)?,
675 None => false,
676 };
677 let kind = state.op_terminal_outcomes.get(&id_key).copied();
678 let Some(kind) = kind else {
679 if terminal {
680 return Err(OpsLifecycleError::Internal(format!(
681 "generated op terminal authority missing terminal outcome for {id}"
682 )));
683 }
684 return Ok(None);
685 };
686 if !terminal {
687 return Err(OpsLifecycleError::Internal(format!(
688 "generated op terminal authority has terminal outcome for non-terminal {id}"
689 )));
690 }
691 let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
692 OpsLifecycleError::Internal(format!(
693 "generated op terminal authority missing terminal payload for {id}"
694 ))
695 })?;
696 Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
697 .map(Some)
698 }
699
700 fn contains(&self, id: &OperationId) -> bool {
702 let id_key = mm_dsl::OperationId::from_domain(id).0;
703 self.dsl.0.state().op_statuses.contains_key(&id_key)
704 }
705
706 fn active_count(&self) -> usize {
708 self.dsl.0.state().active_op_count as usize
709 }
710
711 fn operation_count(&self) -> usize {
713 self.dsl.0.state().op_statuses.len()
714 }
715
716 fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
718 let mut ids = BTreeSet::new();
719 let state = self.dsl.0.state();
720 Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
721 Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
722 Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
723 Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
724 Self::collect_operation_id_keys(
725 &mut ids,
726 "op_progress_counts",
727 state.op_progress_counts.keys(),
728 )?;
729 Self::collect_operation_id_keys(
730 &mut ids,
731 "op_terminal_outcomes",
732 state.op_terminal_outcomes.keys(),
733 )?;
734 Self::collect_operation_id_keys(
735 &mut ids,
736 "op_terminal_payload",
737 state.op_terminal_payload.keys(),
738 )?;
739 Self::collect_operation_id_keys(
740 &mut ids,
741 "op_completion_seq",
742 state.op_completion_seq.keys(),
743 )?;
744 ids.extend(self.records.keys().cloned());
745 Ok(ids.into_iter().collect())
746 }
747
748 fn collect_operation_id_keys<'a, I>(
749 ids: &mut BTreeSet<OperationId>,
750 field: &str,
751 keys: I,
752 ) -> Result<(), OpsLifecycleError>
753 where
754 I: IntoIterator<Item = &'a String>,
755 {
756 for key in keys {
757 let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
758 OpsLifecycleError::Internal(format!(
759 "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
760 ))
761 })?;
762 ids.insert(id);
763 }
764 Ok(())
765 }
766
767 fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
768 let id_key = mm_dsl::OperationId::from_domain(id).0;
769 let state = self.dsl.0.state();
770 state.op_statuses.contains_key(&id_key)
771 || state.op_kinds.contains_key(&id_key)
772 || state.op_sources.contains_key(&id_key)
773 || state.op_peer_ready.contains_key(&id_key)
774 || state.op_progress_counts.contains_key(&id_key)
775 || state.op_terminal_outcomes.contains_key(&id_key)
776 || state.op_terminal_payload.contains_key(&id_key)
777 || state.op_completion_seq.contains_key(&id_key)
778 }
779
780 fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
782 let id_key = mm_dsl::OperationId::from_domain(id).0;
783 self.dsl.0.state().op_completion_seq.get(&id_key).copied()
784 }
785
786 fn completion_feed_authority_entries(
787 &self,
788 ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
789 let state = self.dsl.0.state();
790 let sequence_keys: BTreeSet<String> =
791 state.completion_feed_sequences.keys().cloned().collect();
792 let companion_domains: [(&str, BTreeSet<String>); 3] = [
793 (
794 "completion_feed_kinds",
795 state.completion_feed_kinds.keys().cloned().collect(),
796 ),
797 (
798 "completion_feed_terminal_outcomes",
799 state
800 .completion_feed_terminal_outcomes
801 .keys()
802 .cloned()
803 .collect(),
804 ),
805 (
806 "completion_feed_terminal_payload",
807 state
808 .completion_feed_terminal_payload
809 .keys()
810 .cloned()
811 .collect(),
812 ),
813 ];
814 for (field, keys) in companion_domains {
815 if keys != sequence_keys {
816 return Err(OpsLifecycleError::Internal(format!(
817 "generated completion feed authority has mismatched {field} domain"
818 )));
819 }
820 }
821
822 let mut entries = HashMap::new();
823 for (id_key, seq) in &state.completion_feed_sequences {
824 if !state.completion_sequence_claims.contains(seq) {
825 return Err(OpsLifecycleError::Internal(format!(
826 "generated completion feed authority sequence {seq} for {id_key} is not claimed"
827 )));
828 }
829 let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
830 OpsLifecycleError::Internal(format!(
831 "generated completion feed authority used invalid operation id key {id_key}: {error}"
832 ))
833 })?;
834 let kind = state
835 .completion_feed_kinds
836 .get(id_key)
837 .copied()
838 .map(OperationKind::from)
839 .ok_or_else(|| {
840 OpsLifecycleError::Internal(format!(
841 "generated completion feed authority missing kind for {id_key}"
842 ))
843 })?;
844 let outcome_kind = state
845 .completion_feed_terminal_outcomes
846 .get(id_key)
847 .copied()
848 .ok_or_else(|| {
849 OpsLifecycleError::Internal(format!(
850 "generated completion feed authority missing terminal outcome for {id_key}"
851 ))
852 })?;
853 let payload = state
854 .completion_feed_terminal_payload
855 .get(id_key)
856 .ok_or_else(|| {
857 OpsLifecycleError::Internal(format!(
858 "generated completion feed authority missing terminal payload for {id_key}"
859 ))
860 })?;
861 let terminal_outcome = Self::checked_terminal_payload(
862 outcome_kind,
863 payload,
864 "generated completion feed authority",
865 id_key,
866 )?;
867 entries.insert(
868 operation_id,
869 CompletionFeedCanonicalState {
870 seq: *seq,
871 kind,
872 terminal_outcome,
873 },
874 );
875 }
876 Ok(entries)
877 }
878
879 fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
880 let state = self.dsl.0.state();
881 match consumer {
882 CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
883 CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
884 CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
885 }
886 }
887
888 fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
889 meerkat_core::EpochCursorSnapshot {
890 agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
891 runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
892 runtime_last_injected_seq: self
893 .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
894 }
895 }
896
897 fn snapshot(
899 &self,
900 id: &OperationId,
901 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
902 let Some(shell) = self.records.get(id) else {
903 if self.has_generated_operation_record_fact(id) {
904 return Err(OpsLifecycleError::Internal(format!(
905 "generated op lifecycle authority has operation facts without shell projection record for {id}"
906 )));
907 }
908 return Ok(None);
909 };
910 let kind = self.require_kind(id)?;
911 let status = self.require_status(id)?;
912 let terminal = Self::operation_status_is_terminal(id, status)?;
913 let public_result_class = Self::operation_public_result_class(id, status)?;
914 let peer_ready = self.require_peer_ready(id)?;
915 let progress_count = self.require_progress_count(id)?;
916 let operation_source = self.operation_source(id)?;
917 let terminal_outcome = self.terminal_outcome(id)?;
918
919 let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
920 let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
921 let completed_at_ms = shell
922 .completed_at
923 .map(|i| shell.epoch_millis_for_instant(i));
924 let elapsed_ms = shell.completed_at.map(|completed| {
925 completed
926 .saturating_duration_since(shell.created_at)
927 .as_millis() as u64
928 });
929
930 Ok(Some(OperationLifecycleSnapshot {
931 id: shell.spec.id.clone(),
932 kind,
933 display_name: shell.spec.display_name.clone(),
934 child_session_id: Self::child_session_id_from_operation_source(
935 operation_source.as_ref(),
936 ),
937 operation_source,
938 status,
939 terminal,
940 public_result_class,
941 peer_ready,
942 progress_count,
943 watcher_count: shell.watchers.len() as u32,
944 terminal_outcome,
945 peer_handle: shell.peer_handle.clone(),
946 created_at_ms,
947 started_at_ms,
948 completed_at_ms,
949 elapsed_ms,
950 }))
951 }
952
953 fn finalize_terminal(
958 &mut self,
959 id: &OperationId,
960 ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
961 let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
962 OpsLifecycleError::Internal(format!(
963 "generated op terminal transition did not mint terminal outcome for {id}"
964 ))
965 })?;
966 let kind = self.require_kind(id)?;
967
968 if let Some(shell) = self.records.get_mut(id) {
970 shell.notify_watchers(&outcome);
971 shell.mark_completed();
972 }
973
974 if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
975 {
976 self.dsl_apply(
977 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
978 operation_id: mm_dsl::OperationId::from_domain(id).0,
979 },
980 "CollectCompletedOp",
981 )?;
982 self.records.remove(id);
983 self.completed_order.retain(|queued| queued != id);
984 return Ok(None);
985 }
986
987 let mut completion_entry = None;
988 if Self::operation_completion_feed_class(id, kind)?
989 == mm_dsl::OperationCompletionFeedClass::Emit
990 {
991 let feed_authority = self
992 .completion_feed_authority_entries()?
993 .remove(id)
994 .ok_or_else(|| {
995 OpsLifecycleError::Internal(format!(
996 "generated op terminal transition did not mint completion feed authority for {id}"
997 ))
998 })?;
999 if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
1000 return Err(OpsLifecycleError::Internal(format!(
1001 "generated completion feed authority drifted from terminal op authority for {id}"
1002 )));
1003 }
1004 let seq = self.completion_sequence(id).ok_or_else(|| {
1005 OpsLifecycleError::Internal(format!(
1006 "generated op terminal transition did not mint completion sequence for {id}"
1007 ))
1008 })?;
1009 if feed_authority.seq != seq {
1010 return Err(OpsLifecycleError::Internal(format!(
1011 "generated completion feed authority sequence drifted for {id}"
1012 )));
1013 }
1014 let display_name = self
1015 .records
1016 .get(id)
1017 .map(|r| r.spec.display_name.clone())
1018 .unwrap_or_default();
1019 let completed_at_ms = self
1020 .records
1021 .get(id)
1022 .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1023 completion_entry = Some(CompletionEntry {
1024 seq: feed_authority.seq,
1025 operation_id: id.clone(),
1026 kind: feed_authority.kind,
1027 display_name,
1028 terminal_outcome: feed_authority.terminal_outcome,
1029 completed_at_ms,
1030 });
1031 }
1032
1033 self.completed_order.push_back(id.clone());
1035 while self.completed_order.len() > self.max_completed {
1036 if let Some(evicted) = self.completed_order.pop_front() {
1037 self.dsl_apply(
1038 mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1039 operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1040 },
1041 "EvictCompletedOp",
1042 )?;
1043 self.records.remove(&evicted);
1044 }
1045 }
1046
1047 self.maybe_satisfy_wait()?;
1052 Ok(completion_entry)
1053 }
1054
1055 fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1056 if let Some(entry) = entry {
1057 self.feed_buffer.push(entry);
1058 }
1059 }
1060
1061 fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1063 self.dsl
1064 .0
1065 .state()
1066 .wait_operation_ids
1067 .iter()
1068 .map(|key| {
1069 serde_json::from_str::<OperationId>(key).map_err(|error| {
1070 OpsLifecycleError::Internal(format!(
1071 "generated wait operation identity authority used invalid operation id key {key}: {error}"
1072 ))
1073 })
1074 })
1075 .collect()
1076 }
1077
1078 #[cfg(test)]
1080 fn wait_active(&self) -> bool {
1081 self.dsl.0.state().wait_active
1082 }
1083
1084 fn wait_all_satisfied_from_effects(
1085 effects: &[mm_dsl::MeerkatMachineEffect],
1086 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1087 let mut satisfied = None;
1088 for effect in effects {
1089 let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1090 wait_request_id,
1091 run_id,
1092 operation_ids,
1093 } = effect
1094 else {
1095 continue;
1096 };
1097 if satisfied.is_some() {
1098 return Err(OpsLifecycleError::Internal(
1099 "generated wait_all authority emitted multiple satisfaction effects".into(),
1100 ));
1101 }
1102 let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1103 OpsLifecycleError::Internal(format!(
1104 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1105 wait_request_id.0
1106 ))
1107 })?;
1108 let mut ids = Vec::with_capacity(operation_ids.len());
1109 for operation_id in operation_ids {
1110 ids.push(
1111 serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1112 OpsLifecycleError::Internal(format!(
1113 "generated wait_all authority emitted invalid operation id '{}': {err}",
1114 operation_id.0
1115 ))
1116 })?,
1117 );
1118 }
1119 satisfied = Some(WaitAllSatisfied {
1120 wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1121 run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1122 OpsLifecycleError::Internal(format!(
1123 "generated wait_all authority emitted invalid run id '{}': {err}",
1124 run_id.0
1125 ))
1126 })?),
1127 operation_ids: ids,
1128 });
1129 }
1130 Ok(satisfied)
1131 }
1132
1133 fn parse_wait_all_operation_id(
1134 raw: &str,
1135 context: &str,
1136 ) -> Result<OperationId, OpsLifecycleError> {
1137 serde_json::from_str::<OperationId>(raw).map_err(|err| {
1138 OpsLifecycleError::Internal(format!(
1139 "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1140 ))
1141 })
1142 }
1143
1144 fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1145 let mut seen = HashSet::new();
1146 operation_ids
1147 .iter()
1148 .find(|operation_id| !seen.insert((*operation_id).clone()))
1149 .cloned()
1150 }
1151
1152 fn wait_all_admission_error_from_effects(
1153 wait_request_id: &WaitRequestId,
1154 effects: &[mm_dsl::MeerkatMachineEffect],
1155 ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1156 let mut admission = None;
1157 for effect in effects {
1158 let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1159 wait_request_id: resolved_wait_request_id,
1160 result,
1161 reject_reason,
1162 rejected_operation_id,
1163 } = effect
1164 else {
1165 continue;
1166 };
1167 if admission.is_some() {
1168 return Err(OpsLifecycleError::Internal(
1169 "generated wait_all authority emitted multiple admission results".into(),
1170 ));
1171 }
1172 let resolved_uuid =
1173 uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1174 OpsLifecycleError::Internal(format!(
1175 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1176 resolved_wait_request_id.0
1177 ))
1178 })?;
1179 let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1180 if &resolved_wait_request_id != wait_request_id {
1181 return Err(OpsLifecycleError::Internal(format!(
1182 "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1183 )));
1184 }
1185 admission = Some(match result {
1186 mm_dsl::WaitAllAdmissionResultKind::Accept => {
1187 if reject_reason.is_some() || rejected_operation_id.is_some() {
1188 return Err(OpsLifecycleError::Internal(
1189 "generated wait_all authority accepted with rejection payload".into(),
1190 ));
1191 }
1192 None
1193 }
1194 mm_dsl::WaitAllAdmissionResultKind::Reject => {
1195 let reason = reject_reason.ok_or_else(|| {
1196 OpsLifecycleError::Internal(
1197 "generated wait_all authority rejected without reason".into(),
1198 )
1199 })?;
1200 let error = match reason {
1201 mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1202 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1203 OpsLifecycleError::Internal(
1204 "generated wait_all authority rejected duplicate without operation id"
1205 .into(),
1206 )
1207 })?;
1208 OpsLifecycleError::DuplicateWaitOperation(
1209 Self::parse_wait_all_operation_id(raw, "duplicate")?,
1210 )
1211 }
1212 mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1213 if rejected_operation_id.is_some() {
1214 return Err(OpsLifecycleError::Internal(
1215 "generated wait_all authority rejected active wait with operation id"
1216 .into(),
1217 ));
1218 }
1219 OpsLifecycleError::WaitAlreadyActive
1220 }
1221 mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1222 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1223 OpsLifecycleError::Internal(
1224 "generated wait_all authority rejected missing operation without operation id"
1225 .into(),
1226 )
1227 })?;
1228 OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1229 raw, "missing",
1230 )?)
1231 }
1232 };
1233 Some(error)
1234 }
1235 });
1236 }
1237 admission.ok_or_else(|| {
1238 OpsLifecycleError::Internal(
1239 "generated wait_all authority emitted no admission result".into(),
1240 )
1241 })
1242 }
1243
1244 fn resolve_wait_all_admission(
1245 &mut self,
1246 wait_request_id: &WaitRequestId,
1247 operation_ids: &[OperationId],
1248 dsl_ids: &BTreeSet<String>,
1249 dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1250 operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1251 operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1252 ) -> Result<(), OpsLifecycleError> {
1253 let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1254 .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1255 let not_found = operation_ids
1256 .iter()
1257 .find(|operation_id| !self.contains(operation_id))
1258 .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1259 let dsl_id_sequence: Vec<String> = operation_ids
1260 .iter()
1261 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1262 .collect();
1263 let effects = self.dsl_apply_with_effects(
1264 mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1265 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1266 operation_id_sequence: dsl_id_sequence,
1267 operation_ids: dsl_ids.clone(),
1268 operation_id_tokens: dsl_id_tokens.clone(),
1269 operation_token_by_id: operation_token_by_id.clone(),
1270 operation_id_by_token: operation_id_by_token.clone(),
1271 duplicate_operation_id: duplicate,
1272 not_found_operation_id: not_found,
1273 },
1274 "ResolveWaitAllAdmission",
1275 )?;
1276 if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1277 {
1278 return Err(error);
1279 }
1280 Ok(())
1281 }
1282
1283 fn try_satisfy_wait_all_authority(
1284 &mut self,
1285 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1286 let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1287 return Ok(None);
1288 };
1289 let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1290 return Err(OpsLifecycleError::Internal(
1291 "generated wait_all authority has active wait without run id".into(),
1292 ));
1293 };
1294 let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1295 let transition = match mm_dsl::MeerkatMachineMutator::apply(
1296 &mut *self.dsl.0,
1297 mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1298 wait_request_id: dsl_wait_request_id,
1299 run_id: dsl_run_id,
1300 operation_id_tokens: dsl_operation_id_tokens,
1301 },
1302 ) {
1303 Ok(transition) => transition,
1304 Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1305 Err(err) => {
1306 return Err(OpsLifecycleError::Internal(format!(
1307 "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1308 )));
1309 }
1310 };
1311 Self::wait_all_satisfied_from_effects(transition.effects())?
1312 .ok_or_else(|| {
1313 OpsLifecycleError::Internal(
1314 "generated wait_all authority accepted satisfaction without effect".into(),
1315 )
1316 })
1317 .map(Some)
1318 }
1319
1320 fn begin_wait_all_authority(
1321 &mut self,
1322 run_id: &RunId,
1323 wait_request_id: &WaitRequestId,
1324 operation_ids: &[OperationId],
1325 ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1326 let mut dsl_ids = BTreeSet::new();
1327 let mut dsl_id_tokens = BTreeSet::new();
1328 let mut operation_token_by_id = BTreeMap::new();
1329 let mut operation_id_by_token = BTreeMap::new();
1330 for id in operation_ids {
1331 let token = mm_dsl::OperationId::from_domain(id);
1332 let raw_id = token.0.clone();
1333 dsl_ids.insert(raw_id.clone());
1334 dsl_id_tokens.insert(token.clone());
1335 operation_token_by_id.insert(raw_id.clone(), token.clone());
1336 operation_id_by_token.insert(token, raw_id);
1337 }
1338 self.resolve_wait_all_admission(
1339 wait_request_id,
1340 operation_ids,
1341 &dsl_ids,
1342 &dsl_id_tokens,
1343 &operation_token_by_id,
1344 &operation_id_by_token,
1345 )?;
1346 self.dsl_apply(
1347 mm_dsl::MeerkatMachineInput::RequestWaitAll {
1348 run_id: mm_dsl::RunId::from_domain(run_id),
1349 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1350 operation_id_sequence: operation_ids
1351 .iter()
1352 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1353 .collect(),
1354 operation_ids: dsl_ids,
1355 operation_id_tokens: dsl_id_tokens,
1356 operation_token_by_id,
1357 operation_id_by_token,
1358 },
1359 "RequestWaitAll",
1360 )?;
1361 if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1362 return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1363 }
1364 Ok(WaitAllAuthorityPlan::ActivateBarrier)
1365 }
1366
1367 fn owner_termination_targets(
1368 &self,
1369 ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1370 let mut targets = Vec::new();
1371 for id in self.operation_ids()? {
1372 let status = self.require_status(&id)?;
1373 if !Self::operation_status_is_terminal(&id, status)? {
1374 targets.push((id, status));
1375 }
1376 }
1377 Ok(targets)
1378 }
1379
1380 fn operation_status_is_terminal(
1381 operation_id: &OperationId,
1382 status: OperationStatus,
1383 ) -> Result<bool, OpsLifecycleError> {
1384 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1385 let effects = Self::apply_stateless_classifier(
1386 mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1387 operation_id: operation_id_key.clone(),
1388 status: mm_dsl::OperationStatus::from(status),
1389 },
1390 "ClassifyOperationTerminality",
1391 )?;
1392 let mut terminal = None;
1393 for effect in effects {
1394 match effect {
1395 mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1396 if operation_id == operation_id_key =>
1397 {
1398 terminal = Some(true);
1399 }
1400 mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1401 if operation_id == operation_id_key =>
1402 {
1403 terminal = Some(false);
1404 }
1405 other => {
1406 return Err(OpsLifecycleError::Internal(format!(
1407 "unexpected generated operation terminality effect: {other:?}"
1408 )));
1409 }
1410 }
1411 }
1412 terminal.ok_or_else(|| {
1413 OpsLifecycleError::Internal(format!(
1414 "generated operation terminality authority emitted no effect for {operation_id}"
1415 ))
1416 })
1417 }
1418
1419 fn operation_public_result_class(
1420 operation_id: &OperationId,
1421 status: OperationStatus,
1422 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1423 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1424 let effects = Self::apply_stateless_classifier(
1425 mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1426 operation_id: operation_id_key.clone(),
1427 status: mm_dsl::OperationStatus::from(status),
1428 },
1429 "ClassifyOperationPublicResult",
1430 )?;
1431 let mut result = None;
1432 for effect in effects {
1433 match effect {
1434 mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1435 operation_id,
1436 result: classified,
1437 } if operation_id == operation_id_key => {
1438 result = Some(OperationPublicResultClass::from(classified));
1439 }
1440 other => {
1441 return Err(OpsLifecycleError::Internal(format!(
1442 "unexpected generated operation public-result effect: {other:?}"
1443 )));
1444 }
1445 }
1446 }
1447 result.ok_or_else(|| {
1448 OpsLifecycleError::Internal(format!(
1449 "generated operation public-result authority emitted no effect for {operation_id}"
1450 ))
1451 })
1452 }
1453
1454 fn operation_transition_rejection_is_idempotent(
1455 operation_id: &OperationId,
1456 action: OperationLifecycleAction,
1457 status: OperationStatus,
1458 ) -> Result<bool, OpsLifecycleError> {
1459 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1460 let action = mm_dsl::OpLifecycleActionKind::from(action);
1461 let status = mm_dsl::OperationStatus::from(status);
1462 let effects = Self::apply_stateless_classifier(
1463 mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1464 operation_id: operation_id_key.clone(),
1465 action,
1466 status,
1467 },
1468 "ClassifyOperationTransitionIdempotence",
1469 )?;
1470 let mut idempotent = None;
1471 for effect in effects {
1472 match effect {
1473 mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1474 operation_id,
1475 action: effect_action,
1476 status: effect_status,
1477 } if operation_id == operation_id_key
1478 && effect_action == action
1479 && effect_status == status =>
1480 {
1481 idempotent = Some(true);
1482 }
1483 mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1484 operation_id,
1485 action: effect_action,
1486 status: effect_status,
1487 } if operation_id == operation_id_key
1488 && effect_action == action
1489 && effect_status == status =>
1490 {
1491 idempotent = Some(false);
1492 }
1493 other => {
1494 return Err(OpsLifecycleError::Internal(format!(
1495 "unexpected generated operation transition-idempotence effect: {other:?}"
1496 )));
1497 }
1498 }
1499 }
1500 idempotent.ok_or_else(|| {
1501 OpsLifecycleError::Internal(format!(
1502 "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1503 ))
1504 })
1505 }
1506
1507 fn operation_completion_feed_class(
1508 operation_id: &OperationId,
1509 kind: OperationKind,
1510 ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1511 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1512 let kind = mm_dsl::OperationKind::from(kind);
1513 let effects = Self::apply_stateless_classifier(
1514 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1515 operation_id: operation_id_key.clone(),
1516 kind,
1517 },
1518 "ClassifyOperationCompletionFeed",
1519 )?;
1520 let mut class = None;
1521 for effect in effects {
1522 match effect {
1523 mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1524 operation_id,
1525 result,
1526 } if operation_id == operation_id_key => {
1527 class = Some(result);
1528 }
1529 other => {
1530 return Err(OpsLifecycleError::Internal(format!(
1531 "unexpected generated operation completion-feed effect: {other:?}"
1532 )));
1533 }
1534 }
1535 }
1536 class.ok_or_else(|| {
1537 OpsLifecycleError::Internal(format!(
1538 "generated operation completion-feed authority emitted no effect for {operation_id}"
1539 ))
1540 })
1541 }
1542
1543 fn operation_completion_wake_class(
1544 operation_id: &OperationId,
1545 kind: OperationKind,
1546 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1547 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1548 let kind = mm_dsl::OperationKind::from(kind);
1549 let effects = Self::apply_stateless_classifier(
1550 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1551 operation_id: operation_id_key.clone(),
1552 kind,
1553 },
1554 "ClassifyOperationCompletionWake",
1555 )?;
1556 let mut class = None;
1557 for effect in effects {
1558 match effect {
1559 mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1560 operation_id,
1561 result,
1562 } if operation_id == operation_id_key => {
1563 class = Some(OperationCompletionWakeClass::from(result));
1564 }
1565 other => {
1566 return Err(OpsLifecycleError::Internal(format!(
1567 "unexpected generated operation completion-wake effect: {other:?}"
1568 )));
1569 }
1570 }
1571 }
1572 class.ok_or_else(|| {
1573 OpsLifecycleError::Internal(format!(
1574 "generated operation completion-wake authority emitted no effect for {operation_id}"
1575 ))
1576 })
1577 }
1578
1579 fn operation_durability_class(
1580 operation_id: &OperationId,
1581 kind: OperationKind,
1582 ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1583 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1584 let kind = mm_dsl::OperationKind::from(kind);
1585 let effects = Self::apply_stateless_classifier(
1586 mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1587 operation_id: operation_id_key.clone(),
1588 kind,
1589 },
1590 "ClassifyOperationDurability",
1591 )?;
1592 let mut class = None;
1593 for effect in effects {
1594 match effect {
1595 mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1596 operation_id,
1597 result,
1598 } if operation_id == operation_id_key => {
1599 class = Some(result);
1600 }
1601 other => {
1602 return Err(OpsLifecycleError::Internal(format!(
1603 "unexpected generated operation durability effect: {other:?}"
1604 )));
1605 }
1606 }
1607 }
1608 class.ok_or_else(|| {
1609 OpsLifecycleError::Internal(format!(
1610 "generated operation durability authority emitted no effect for {operation_id}"
1611 ))
1612 })
1613 }
1614
1615 fn recovered_operation_record_disposition(
1616 operation_id: &OperationId,
1617 status: OperationStatus,
1618 kind: OperationKind,
1619 terminal_outcome_present: bool,
1620 terminal_payload_present: bool,
1621 completion_sequence_present: bool,
1622 ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1623 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1624 let effects = Self::apply_stateless_classifier(
1625 mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1626 operation_id: operation_id_key.clone(),
1627 status: mm_dsl::OperationStatus::from(status),
1628 kind: mm_dsl::OperationKind::from(kind),
1629 terminal_outcome_present,
1630 terminal_payload_present,
1631 completion_sequence_present,
1632 },
1633 "ClassifyRecoveredOperationRecord",
1634 )?;
1635 let mut disposition = None;
1636 for effect in effects {
1637 match effect {
1638 mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1639 if operation_id == operation_id_key =>
1640 {
1641 disposition = Some(RecoveredOperationRecordDisposition::Retain);
1642 }
1643 mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1644 if operation_id == operation_id_key =>
1645 {
1646 disposition = Some(RecoveredOperationRecordDisposition::Discard);
1647 }
1648 other => {
1649 return Err(OpsLifecycleError::Internal(format!(
1650 "unexpected generated recovered-operation classification effect: {other:?}"
1651 )));
1652 }
1653 }
1654 }
1655 disposition.ok_or_else(|| {
1656 OpsLifecycleError::Internal(format!(
1657 "generated recovered-operation classifier emitted no effect for {operation_id}"
1658 ))
1659 })
1660 }
1661
1662 fn apply_stateless_classifier(
1663 input: mm_dsl::MeerkatMachineInput,
1664 label: &'static str,
1665 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1666 let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1667 "ops stateless classifier Initialize must be accepted",
1668 );
1669 let transition =
1670 mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1671 OpsLifecycleError::Internal(format!(
1672 "DSL rejected ops transition ({label}): {err:?}"
1673 ))
1674 })?;
1675 Ok(transition.into_effects())
1676 }
1677
1678 fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1702 let satisfied = match self.try_satisfy_wait_all_authority() {
1703 Ok(Some(satisfied)) => satisfied,
1704 Ok(None) => return Ok(()),
1705 Err(err) => {
1706 if let Some(pending) = self.pending_wait.take() {
1714 drop(pending.sender);
1715 }
1716 self.wait_request_id = None;
1717 return Err(err);
1718 }
1719 };
1720 let shell_wait_id = self.wait_request_id.take();
1721 if shell_wait_id
1722 .as_ref()
1723 .is_some_and(|id| id != &satisfied.wait_request_id)
1724 {
1725 tracing::error!(
1726 shell_wait_request_id = ?shell_wait_id,
1727 authority_wait_request_id = %satisfied.wait_request_id,
1728 "generated wait_all authority satisfied a different wait request"
1729 );
1730 }
1731 if let Some(pending) = self.pending_wait.take() {
1732 if pending.wait_request_id == satisfied.wait_request_id {
1733 let _ = pending.sender.send(satisfied);
1734 } else if let Some(shell_wait_id) = shell_wait_id {
1735 tracing::error!(
1736 shell_wait_request_id = %shell_wait_id,
1737 pending_wait_request_id = %pending.wait_request_id,
1738 authority_wait_request_id = %satisfied.wait_request_id,
1739 "generated wait_all authority satisfied without a matching pending waiter"
1740 );
1741 }
1742 }
1743 Ok(())
1744 }
1745
1746 fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1752 let (tx, epoch_id, cursor_state) = match (
1753 &self.persist_tx,
1754 &self.persist_epoch_id,
1755 &self.persist_cursor_state,
1756 ) {
1757 (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1758 _ => return Ok(()),
1759 };
1760
1761 let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1762 let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1763 let request = OpsLifecyclePersistenceRequest {
1764 snapshot,
1765 result_tx,
1766 };
1767
1768 tx.send(request).map_err(|_| {
1769 OpsLifecycleError::Internal(
1770 "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1771 .into(),
1772 )
1773 })?;
1774 result_rx.recv().map_err(|_| {
1775 OpsLifecycleError::Internal(
1776 "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1777 .into(),
1778 )
1779 })?
1780 }
1781
1782 fn capture_snapshot(
1784 &self,
1785 epoch_id: meerkat_core::RuntimeEpochId,
1786 _cursor_state: &meerkat_core::EpochCursorState,
1787 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1788 let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1789 for op_id in self.operation_ids()? {
1790 let status = self.require_status(&op_id)?;
1791 let kind = self.require_kind(&op_id)?;
1792 if Self::operation_durability_class(&op_id, kind)?
1793 != mm_dsl::OperationDurabilityClass::Retain
1794 {
1795 continue;
1796 }
1797 let peer_ready = self.require_peer_ready(&op_id)?;
1798 let progress_count = self.require_progress_count(&op_id)?;
1799 let operation_source = self.operation_source(&op_id)?;
1800 let terminal_outcome = self.terminal_outcome(&op_id)?;
1801 let completion_sequence = self.completion_sequence(&op_id);
1802 if terminal_outcome.is_some() && completion_sequence.is_none() {
1803 return Err(OpsLifecycleError::Internal(format!(
1804 "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1805 )));
1806 }
1807 if terminal_outcome.is_none() && completion_sequence.is_some() {
1808 return Err(OpsLifecycleError::Internal(format!(
1809 "generated op terminal authority has completion sequence for non-terminal {op_id}"
1810 )));
1811 }
1812 let terminal_buffered = terminal_outcome.is_some();
1813 let watcher_count = self
1814 .records
1815 .get(&op_id)
1816 .map(|r| r.watchers.len() as u32)
1817 .unwrap_or(0);
1818 operations.insert(
1819 op_id,
1820 OperationCanonicalState {
1821 status,
1822 kind,
1823 operation_source,
1824 peer_ready,
1825 progress_count,
1826 watcher_count,
1827 terminal_outcome,
1828 completion_sequence,
1829 terminal_buffered,
1830 },
1831 );
1832 }
1833 let operation_specs: HashMap<OperationId, OperationSpec> = self
1834 .records
1835 .iter()
1836 .filter(|(id, _)| operations.contains_key(*id))
1837 .map(|(id, record)| {
1838 let mut spec = record.spec.clone();
1839 let operation_source = operations
1840 .get(id)
1841 .and_then(|state| state.operation_source.as_ref());
1842 Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1843 (id.clone(), spec)
1844 })
1845 .collect();
1846 let completed_order: VecDeque<OperationId> = self
1847 .completed_order
1848 .iter()
1849 .filter(|id| operations.contains_key(*id))
1850 .cloned()
1851 .collect();
1852 let active_count = operations
1853 .iter()
1854 .filter(|(id, state)| {
1855 matches!(
1856 Self::operation_status_is_terminal(id, state.status),
1857 Ok(false)
1858 )
1859 })
1860 .count();
1861 let authority_completion_entries = self.completion_feed_authority_entries()?;
1862 let published_completion_entries_by_id: HashMap<OperationId, CompletionEntry> = {
1863 let inner = self
1864 .feed_buffer
1865 .inner
1866 .read()
1867 .unwrap_or_else(std::sync::PoisonError::into_inner);
1868 let mut entries = HashMap::new();
1869 for entry in &inner.entries {
1870 if !authority_completion_entries.contains_key(&entry.operation_id) {
1871 return Err(OpsLifecycleError::Internal(format!(
1872 "public completion feed projection for {} has no generated authority",
1873 entry.operation_id
1874 )));
1875 }
1876 if entries
1877 .insert(entry.operation_id.clone(), entry.clone())
1878 .is_some()
1879 {
1880 return Err(OpsLifecycleError::Internal(format!(
1881 "public completion feed projection for {} appeared more than once",
1882 entry.operation_id
1883 )));
1884 }
1885 }
1886 entries
1887 };
1888 let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
1889 .iter()
1890 .map(|(operation_id, authority_entry)| {
1891 if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
1892 if projection.seq != authority_entry.seq
1893 || projection.kind != authority_entry.kind
1894 || projection.terminal_outcome != authority_entry.terminal_outcome
1895 {
1896 return Err(OpsLifecycleError::Internal(format!(
1897 "public completion feed projection for {operation_id} drifted from generated authority"
1898 )));
1899 }
1900 return Ok(projection.clone());
1901 }
1902
1903 let display_name = self
1904 .records
1905 .get(operation_id)
1906 .map(|record| record.spec.display_name.clone())
1907 .unwrap_or_default();
1908 let completed_at_ms = self.records.get(operation_id).and_then(|record| {
1909 record
1910 .completed_at
1911 .map(|completed_at| record.epoch_millis_for_instant(completed_at))
1912 });
1913
1914 Ok(CompletionEntry {
1915 seq: authority_entry.seq,
1916 operation_id: operation_id.clone(),
1917 kind: authority_entry.kind,
1918 display_name,
1919 terminal_outcome: authority_entry.terminal_outcome.clone(),
1920 completed_at_ms,
1921 })
1922 })
1923 .collect::<Result<_, _>>()?;
1924 completion_entries.sort_by_key(|entry| entry.seq);
1925
1926 let authority_state = RegistryCanonicalState {
1927 operations,
1928 completion_feed_entries: authority_completion_entries,
1929 completed_order,
1930 max_completed: self.max_completed,
1931 max_concurrent: self.max_concurrent,
1932 active_count,
1933 wait_request_id: self.wait_request_id.clone(),
1934 wait_operation_ids: self.wait_operation_ids()?,
1935 next_completion_seq: self.dsl.0.state().next_completion_seq,
1936 };
1937
1938 Ok(PersistedOpsSnapshot {
1939 epoch_id,
1940 authority_state,
1941 operation_specs,
1942 completion_entries,
1943 cursors: self.completion_cursor_snapshot(),
1944 })
1945 }
1946
1947 fn shell_record_mut(
1948 &mut self,
1949 id: &OperationId,
1950 ) -> Result<&mut ShellRecord, OpsLifecycleError> {
1951 self.records
1952 .get_mut(id)
1953 .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
1954 }
1955
1956 fn collect_wait_outcomes(
1957 &self,
1958 operation_ids: &[OperationId],
1959 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
1960 operation_ids
1961 .iter()
1962 .map(|operation_id| {
1963 let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
1964 OpsLifecycleError::Internal(format!(
1965 "wait_all completed without terminal outcome for {operation_id}"
1966 ))
1967 })?;
1968 Ok((operation_id.clone(), outcome))
1969 })
1970 .collect()
1971 }
1972}
1973
1974impl Default for ShellState {
1975 fn default() -> Self {
1976 Self::new(DEFAULT_MAX_COMPLETED, None)
1977 }
1978}
1979
1980#[derive(Debug, Clone)]
1986pub struct OpsLifecycleConfig {
1987 pub max_completed: usize,
1989 pub max_concurrent: Option<usize>,
1991}
1992
1993impl Default for OpsLifecycleConfig {
1994 fn default() -> Self {
1995 Self {
1996 max_completed: DEFAULT_MAX_COMPLETED,
1997 max_concurrent: None,
1998 }
1999 }
2000}
2001
2002#[derive(Debug)]
2009pub struct RuntimeOpsLifecycleRegistry {
2010 state: RwLock<ShellState>,
2011}
2012
2013#[derive(Debug, Clone)]
2014pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2015 pub operation_count: usize,
2016 pub active_count: usize,
2017 pub wait_request_id: Option<WaitRequestId>,
2018 pub pending_wait_present: bool,
2019 pub pending_wait_request_id: Option<WaitRequestId>,
2020 pub wait_operation_ids: Vec<OperationId>,
2021 pub operations: Vec<OperationLifecycleSnapshot>,
2022}
2023
2024impl Default for RuntimeOpsLifecycleRegistry {
2025 fn default() -> Self {
2026 Self {
2027 state: RwLock::new(ShellState::default()),
2028 }
2029 }
2030}
2031
2032impl RuntimeOpsLifecycleRegistry {
2033 pub fn new() -> Self {
2034 let dsl = new_ops_dsl_authority();
2035 let feed_capacity = DEFAULT_MAX_COMPLETED.saturating_mul(4).max(1024);
2036 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2037 Self {
2038 state: RwLock::new(ShellState {
2039 dsl,
2040 records: HashMap::new(),
2041 pending_wait: None,
2042 completed_order: VecDeque::new(),
2043 max_completed: DEFAULT_MAX_COMPLETED,
2044 max_concurrent: None,
2045 wait_request_id: None,
2046 feed_buffer,
2047 persist_tx: None,
2048 persist_epoch_id: None,
2049 persist_cursor_state: None,
2050 owner_retired: false,
2051 }),
2052 }
2053 }
2054
2055 pub fn with_config(config: OpsLifecycleConfig) -> Self {
2056 Self {
2057 state: RwLock::new(ShellState::new(config.max_completed, config.max_concurrent)),
2058 }
2059 }
2060
2061 fn recover_completion_feed_entry(
2062 shell: &mut ShellState,
2063 operation_id: &OperationId,
2064 entry: &CompletionFeedCanonicalState,
2065 ) -> Result<(), OpsLifecycleError> {
2066 let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2067 let terminal_outcome_kind =
2068 mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2069 let effects = shell.dsl_apply_with_effects(
2070 mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2071 operation_id: expected_operation_id.clone(),
2072 kind: mm_dsl::OperationKind::from(entry.kind),
2073 terminal_outcome: terminal_outcome_kind,
2074 terminal_payload: entry.terminal_outcome.clone(),
2075 completion_sequence: entry.seq,
2076 },
2077 "RecoverCompletionFeedEntry",
2078 )?;
2079 let recovered = effects.iter().find_map(|effect| match effect {
2080 mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2081 operation_id,
2082 seq,
2083 kind,
2084 terminal_outcome,
2085 terminal_payload,
2086 } => Some((
2087 operation_id,
2088 *seq,
2089 OperationKind::from(*kind),
2090 *terminal_outcome,
2091 terminal_payload,
2092 )),
2093 _ => None,
2094 });
2095 let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2096 return Err(OpsLifecycleError::Internal(
2097 "generated completion-feed recovery emitted no recovered entry".into(),
2098 ));
2099 };
2100 if operation_id != &expected_operation_id
2101 || seq != entry.seq
2102 || kind != entry.kind
2103 || terminal_outcome != terminal_outcome_kind
2104 || terminal_payload != &entry.terminal_outcome
2105 {
2106 return Err(OpsLifecycleError::Internal(format!(
2107 "generated completion-feed recovery drifted for {operation_id}"
2108 )));
2109 }
2110 Ok(())
2111 }
2112
2113 pub fn set_persistence_channel(
2119 &self,
2120 tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2121 epoch_id: meerkat_core::RuntimeEpochId,
2122 cursor_state: Arc<meerkat_core::EpochCursorState>,
2123 ) {
2124 if let Ok(mut state) = self.state.write() {
2125 if state.owner_retired {
2126 return;
2127 }
2128 state.persist_tx = Some(tx);
2129 state.persist_epoch_id = Some(epoch_id);
2130 state.persist_cursor_state = Some(cursor_state);
2131 }
2132 }
2133
2134 pub(crate) fn retire_owner_for_unregister(
2143 &self,
2144 reason: String,
2145 ) -> Result<(), OpsLifecycleError> {
2146 let closed_sender = {
2147 let mut state = self.write_state()?;
2148 if state.owner_retired {
2149 return Ok(());
2150 }
2151 terminate_owner_locked(&mut state, &reason)?;
2152 state.owner_retired = true;
2153 state.persist_epoch_id = None;
2154 state.persist_cursor_state = None;
2155 state.persist_tx.take()
2156 };
2157 drop(closed_sender);
2158 Ok(())
2159 }
2160
2161 pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2168 let PersistedOpsSnapshot {
2169 authority_state,
2170 operation_specs,
2171 completion_entries,
2172 cursors,
2173 ..
2174 } = snapshot;
2175 let max_completed = authority_state.max_completed;
2176 let max_concurrent = authority_state.max_concurrent;
2177 let next_completion_seq = authority_state.next_completion_seq;
2178 let authority_completion_entries = authority_state.completion_feed_entries;
2179 let authority_operations = authority_state.operations;
2180 let mut shell = ShellState::new(max_completed, max_concurrent);
2181
2182 let mut retained_ids: HashSet<OperationId> = HashSet::new();
2188 for (op_id, op_state) in authority_operations {
2189 let terminal_outcome = op_state
2190 .terminal_outcome
2191 .as_ref()
2192 .map(mm_dsl::OperationTerminalOutcomeKind::from);
2193 let terminal_payload = op_state.terminal_outcome.clone();
2194 let disposition = ShellState::recovered_operation_record_disposition(
2195 &op_id,
2196 op_state.status,
2197 op_state.kind,
2198 terminal_outcome.is_some(),
2199 terminal_payload.is_some(),
2200 op_state.completion_sequence.is_some(),
2201 )?;
2202 if disposition == RecoveredOperationRecordDisposition::Discard {
2203 continue;
2204 }
2205 if let Some(spec_source) = operation_specs
2206 .get(&op_id)
2207 .and_then(|spec| spec.operation_source.as_ref())
2208 && op_state.operation_source.as_ref() != Some(spec_source)
2209 {
2210 return Err(OpsLifecycleError::Internal(format!(
2211 "persisted operation source mirror for {op_id} drifted from generated authority"
2212 )));
2213 }
2214 let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2215 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2216 status: mm_dsl::OperationStatus::from(op_state.status),
2217 kind: mm_dsl::OperationKind::from(op_state.kind),
2218 source: op_state
2219 .operation_source
2220 .as_ref()
2221 .map(mm_dsl::OperationSource::from_domain),
2222 peer_ready: op_state.peer_ready,
2223 progress_count: u64::from(op_state.progress_count),
2224 terminal_outcome,
2225 terminal_payload,
2226 completion_sequence: op_state.completion_sequence,
2227 };
2228 shell.dsl_apply(recovery, "RecoverOpRecord")?;
2229 let recovered_seq = shell.completion_sequence(&op_id).ok_or_else(|| {
2230 OpsLifecycleError::Internal(format!(
2231 "generated op recovery accepted {op_id} without completion sequence"
2232 ))
2233 })?;
2234 if op_state.completion_sequence != Some(recovered_seq) {
2235 return Err(OpsLifecycleError::Internal(format!(
2236 "generated op recovery completion sequence mismatch for {op_id}"
2237 )));
2238 }
2239 retained_ids.insert(op_id);
2240 }
2241 shell.dsl_apply(
2242 mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2243 next_completion_seq,
2244 },
2245 "RecoverOpsCompletionCursor",
2246 )?;
2247 shell.dsl_apply(
2248 mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2249 agent_applied_cursor: cursors.agent_applied_cursor,
2250 runtime_observed_cursor: cursors.runtime_observed_seq,
2251 runtime_injected_cursor: cursors.runtime_last_injected_seq,
2252 },
2253 "RecoverCompletionConsumerCursors",
2254 )?;
2255
2256 let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2259 .iter()
2260 .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2261 .collect();
2262 recovered_completed.sort_by_key(|(seq, _)| *seq);
2263 shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2264
2265 for (operation_id, entry) in &authority_completion_entries {
2269 if !retained_ids.contains(operation_id) {
2270 Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2271 }
2272 }
2273
2274 let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2275 for (operation_id, entry) in &authority_completion_entries {
2276 let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2277 return Err(OpsLifecycleError::Internal(format!(
2278 "persisted completion feed authority for {operation_id} was not recovered"
2279 )));
2280 };
2281 if recovered_entry != entry {
2282 return Err(OpsLifecycleError::Internal(format!(
2283 "persisted completion feed authority drifted from generated recovery for {operation_id}"
2284 )));
2285 }
2286 }
2287
2288 let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2292 for entry in completion_entries {
2293 let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2294 return Err(OpsLifecycleError::Internal(format!(
2295 "persisted completion feed projection for {} has no generated feed authority",
2296 entry.operation_id
2297 )));
2298 };
2299 if authority_entry.seq != entry.seq
2300 || authority_entry.kind != entry.kind
2301 || authority_entry.terminal_outcome != entry.terminal_outcome
2302 {
2303 return Err(OpsLifecycleError::Internal(format!(
2304 "persisted completion feed projection for {} drifted from generated feed authority",
2305 entry.operation_id
2306 )));
2307 }
2308 projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2309 }
2310
2311 let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2312 canonical_feed_entries.into_iter().collect();
2313 recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2314 for (operation_id, entry) in recovered_entries {
2315 let projection = projection_entries_by_id.get(&operation_id);
2316 let display_name = operation_specs
2317 .get(&operation_id)
2318 .map(|spec| spec.display_name.clone())
2319 .or_else(|| projection.map(|entry| entry.display_name.clone()))
2320 .unwrap_or_default();
2321 let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2322 shell.feed_buffer.push(CompletionEntry {
2323 seq: entry.seq,
2324 operation_id,
2325 kind: entry.kind,
2326 display_name,
2327 terminal_outcome: entry.terminal_outcome,
2328 completed_at_ms,
2329 });
2330 }
2331
2332 for (op_id, spec) in operation_specs {
2335 if retained_ids.contains(&op_id) {
2336 let mut spec = spec;
2337 let operation_source = shell.operation_source(&op_id)?;
2338 ShellState::align_spec_child_session_id_to_source(
2339 &mut spec,
2340 operation_source.as_ref(),
2341 );
2342 shell.records.insert(
2343 op_id,
2344 ShellRecord {
2345 spec,
2346 peer_handle: None,
2347 watchers: Vec::new(),
2348 created_at: Instant::now(),
2349 started_at: None,
2350 completed_at: None,
2351 created_at_wall: SystemTime::now(),
2352 },
2353 );
2354 }
2355 }
2356
2357 Ok(Self {
2358 state: RwLock::new(shell),
2359 })
2360 }
2361
2362 pub fn capture_persistence_snapshot(
2367 &self,
2368 epoch_id: meerkat_core::RuntimeEpochId,
2369 cursor_state: &meerkat_core::EpochCursorState,
2370 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2371 let state = self
2372 .state
2373 .read()
2374 .unwrap_or_else(std::sync::PoisonError::into_inner);
2375 state.capture_snapshot(epoch_id, cursor_state)
2376 }
2377
2378 pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2380 let state = self
2381 .state
2382 .read()
2383 .unwrap_or_else(std::sync::PoisonError::into_inner);
2384 state.completion_cursor_snapshot()
2385 }
2386
2387 pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2389 let state = self
2390 .state
2391 .read()
2392 .unwrap_or_else(std::sync::PoisonError::into_inner);
2393 Arc::new(RuntimeCompletionFeed {
2394 buffer: Arc::clone(&state.feed_buffer),
2395 })
2396 }
2397
2398 pub(crate) fn diagnostic_snapshot(
2400 &self,
2401 ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2402 let state = self
2403 .state
2404 .read()
2405 .unwrap_or_else(std::sync::PoisonError::into_inner);
2406 let mut operations = state
2407 .operation_ids()?
2408 .into_iter()
2409 .map(|id| state.snapshot(&id))
2410 .collect::<Result<Vec<_>, _>>()?
2411 .into_iter()
2412 .flatten()
2413 .collect::<Vec<_>>();
2414 operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2415 Ok(RuntimeOpsDiagnosticSnapshot {
2416 operation_count: state.operation_count(),
2417 active_count: state.active_count(),
2418 wait_request_id: state.wait_request_id.clone(),
2419 pending_wait_present: state.pending_wait.is_some(),
2420 pending_wait_request_id: state
2421 .pending_wait
2422 .as_ref()
2423 .map(|pending_wait| pending_wait.wait_request_id.clone()),
2424 wait_operation_ids: state.wait_operation_ids()?,
2425 operations,
2426 })
2427 }
2428
2429 fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2430 self.state
2431 .read()
2432 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2433 }
2434
2435 fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2436 self.state
2437 .write()
2438 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2439 }
2440
2441 fn cancel_wait_all_internal(
2442 &self,
2443 wait_request_id: &WaitRequestId,
2444 ) -> Result<(), OpsLifecycleError> {
2445 let mut state = self.write_state()?;
2446 match state.wait_request_id.as_ref() {
2447 Some(active) if active == wait_request_id => {
2448 state.dsl_apply(
2453 mm_dsl::MeerkatMachineInput::CancelWaitAll,
2454 "CancelWaitAll(cancel)",
2455 )?;
2456 state.wait_request_id = None;
2457 if state
2458 .pending_wait
2459 .as_ref()
2460 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2461 {
2462 state.pending_wait = None;
2463 }
2464 Ok(())
2465 }
2466 _ => {
2467 if state
2468 .pending_wait
2469 .as_ref()
2470 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2471 {
2472 state.pending_wait = None;
2473 }
2474 Ok(())
2475 }
2476 }
2477 }
2478}
2479
2480enum WaitAllFutureState {
2481 Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2482 Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2483 Done,
2484}
2485
2486struct WaitAllFuture<'a> {
2487 registry: &'a RuntimeOpsLifecycleRegistry,
2488 wait_request_id: WaitRequestId,
2489 state: WaitAllFutureState,
2490}
2491
2492impl Future for WaitAllFuture<'_> {
2493 type Output = Result<WaitAllResult, OpsLifecycleError>;
2494
2495 fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2496 match &mut self.state {
2497 WaitAllFutureState::Ready(result) => {
2498 let ready = result.take().unwrap_or_else(|| {
2499 Err(OpsLifecycleError::Internal(
2500 "wait_all future polled after completion".into(),
2501 ))
2502 });
2503 self.state = WaitAllFutureState::Done;
2504 Poll::Ready(ready)
2505 }
2506 WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2507 Poll::Pending => Poll::Pending,
2508 Poll::Ready(Ok(satisfied)) => {
2509 let outcomes = match self.registry.read_state() {
2510 Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2511 Err(err) => Err(err),
2512 };
2513 self.state = WaitAllFutureState::Done;
2514 Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2515 outcomes,
2516 satisfied,
2517 }))
2518 }
2519 Poll::Ready(Err(_)) => {
2520 self.state = WaitAllFutureState::Done;
2521 Poll::Ready(Err(OpsLifecycleError::Internal(
2522 "wait_all completion channel dropped".into(),
2523 )))
2524 }
2525 },
2526 WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2527 "wait_all future polled after completion".into(),
2528 ))),
2529 }
2530 }
2531}
2532
2533impl Drop for WaitAllFuture<'_> {
2534 fn drop(&mut self) {
2535 if matches!(self.state, WaitAllFutureState::Waiting(_))
2536 && let Err(err) = self
2537 .registry
2538 .cancel_wait_all_internal(&self.wait_request_id)
2539 {
2540 tracing::error!(
2541 wait_request_id = %self.wait_request_id,
2542 error = %err,
2543 "generated wait_all authority rejected cancellation during drop"
2544 );
2545 }
2546 }
2547}
2548
2549fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2554 match action {
2555 mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2556 mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2557 mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2558 mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2559 mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2560 mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2561 mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2562 mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2563 mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2564 mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2565 }
2566}
2567
2568fn op_lifecycle_rejection_error_from_effects(
2569 id: &OperationId,
2570 requested_action: mm_dsl::OpLifecycleActionKind,
2571 effects: &[mm_dsl::MeerkatMachineEffect],
2572) -> Result<OpsLifecycleError, OpsLifecycleError> {
2573 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2574 let mut rejection = None;
2575 for effect in effects {
2576 let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2577 operation_id,
2578 action,
2579 reason,
2580 status,
2581 } = effect
2582 else {
2583 continue;
2584 };
2585 if rejection.is_some() {
2586 return Err(OpsLifecycleError::Internal(
2587 "generated op lifecycle authority emitted multiple rejection results".into(),
2588 ));
2589 }
2590 if operation_id != &expected_id || *action != requested_action {
2591 return Err(OpsLifecycleError::Internal(format!(
2592 "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2593 )));
2594 }
2595 let error = match reason {
2596 mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2597 if status.is_some() {
2598 return Err(OpsLifecycleError::Internal(
2599 "generated op lifecycle authority emitted not-found with status".into(),
2600 ));
2601 }
2602 OpsLifecycleError::NotFound(id.clone())
2603 }
2604 mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2605 let status = status.ok_or_else(|| {
2606 OpsLifecycleError::Internal(
2607 "generated op lifecycle authority emitted invalid-transition without status"
2608 .into(),
2609 )
2610 })?;
2611 OpsLifecycleError::InvalidTransition {
2612 id: id.clone(),
2613 status: OperationStatus::from(status),
2614 action: op_lifecycle_action_label(requested_action),
2615 }
2616 }
2617 mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2618 if status.is_none() {
2619 return Err(OpsLifecycleError::Internal(
2620 "generated op lifecycle authority emitted peer-not-expected without status"
2621 .into(),
2622 ));
2623 }
2624 OpsLifecycleError::PeerNotExpected(id.clone())
2625 }
2626 mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2627 if status.is_none() {
2628 return Err(OpsLifecycleError::Internal(
2629 "generated op lifecycle authority emitted already-peer-ready without status"
2630 .into(),
2631 ));
2632 }
2633 OpsLifecycleError::AlreadyPeerReady(id.clone())
2634 }
2635 };
2636 rejection = Some(error);
2637 }
2638 rejection.ok_or_else(|| {
2639 OpsLifecycleError::Internal(
2640 "generated op lifecycle authority emitted no rejection result".into(),
2641 )
2642 })
2643}
2644
2645fn classify_generated_op_rejection(
2646 state: &mut ShellState,
2647 err: mm_dsl::MeerkatMachineTransitionError,
2648 id: &OperationId,
2649 action: mm_dsl::OpLifecycleActionKind,
2650) -> OpsLifecycleError {
2651 match err {
2652 mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2653 match state.dsl_apply_with_effects(
2654 mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2655 operation_id: mm_dsl::OperationId::from_domain(id).0,
2656 action,
2657 },
2658 "ResolveOpLifecycleTransitionRejection",
2659 ) {
2660 Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2661 .unwrap_or_else(|err| err),
2662 Err(err) => err,
2663 }
2664 }
2665 other => OpsLifecycleError::Internal(format!(
2666 "DSL rejected ops transition ({}): {other:?}",
2667 op_lifecycle_action_label(action)
2668 )),
2669 }
2670}
2671
2672fn apply_op_transition(
2673 state: &mut ShellState,
2674 id: &OperationId,
2675 input: mm_dsl::MeerkatMachineInput,
2676 action: mm_dsl::OpLifecycleActionKind,
2677) -> Result<(), OpsLifecycleError> {
2678 state.ensure_owner_active()?;
2679 state
2680 .dsl_apply_raw(input)
2681 .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2682}
2683
2684fn apply_terminal_op_transition_and_persist(
2685 state: &mut ShellState,
2686 id: &OperationId,
2687 input: mm_dsl::MeerkatMachineInput,
2688 action: mm_dsl::OpLifecycleActionKind,
2689) -> Result<(), OpsLifecycleError> {
2690 let previous_snapshot = state.dsl.0.snapshot();
2691 apply_op_transition(state, id, input, action)?;
2692 if let Err(err) = state.maybe_persist() {
2693 state.dsl.0.restore_snapshot(previous_snapshot);
2694 return Err(err);
2695 }
2696 Ok(())
2697}
2698
2699fn terminate_owner_locked(state: &mut ShellState, reason: &str) -> Result<(), OpsLifecycleError> {
2700 state.ensure_owner_active()?;
2701 let to_terminate = state.owner_termination_targets()?;
2702
2703 for (op_id, _status) in &to_terminate {
2704 let terminal_outcome = OperationTerminalOutcome::Terminated {
2705 reason: reason.to_owned(),
2706 };
2707 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2708
2709 apply_terminal_op_transition_and_persist(
2710 state,
2711 op_id,
2712 mm_dsl::MeerkatMachineInput::TerminateOp {
2713 operation_id: mm_dsl::OperationId::from_domain(op_id).0,
2714 outcome: outcome_kind,
2715 payload: terminal_outcome,
2716 },
2717 mm_dsl::OpLifecycleActionKind::Terminate,
2718 )?;
2719
2720 if let Some(entry) = state.finalize_terminal(op_id)? {
2721 state.publish_completion_entry(Some(entry));
2722 }
2723 }
2724 Ok(())
2725}
2726
2727fn op_registration_error_from_effects(
2728 id: &OperationId,
2729 effects: &[mm_dsl::MeerkatMachineEffect],
2730) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2731 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2732 let mut admission = None;
2733 for effect in effects {
2734 let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2735 operation_id,
2736 result,
2737 reject_reason,
2738 max_concurrent_limit,
2739 active_op_count,
2740 } = effect
2741 else {
2742 continue;
2743 };
2744 if admission.is_some() {
2745 return Err(OpsLifecycleError::Internal(
2746 "generated op registration authority emitted multiple admission results".into(),
2747 ));
2748 }
2749 if operation_id != &expected_id {
2750 return Err(OpsLifecycleError::Internal(format!(
2751 "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2752 )));
2753 }
2754 admission = Some(match result {
2755 mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2756 if reject_reason.is_some() {
2757 return Err(OpsLifecycleError::Internal(
2758 "generated op registration authority accepted with rejection reason".into(),
2759 ));
2760 }
2761 None
2762 }
2763 mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2764 let reason = reject_reason.ok_or_else(|| {
2765 OpsLifecycleError::Internal(
2766 "generated op registration authority rejected without reason".into(),
2767 )
2768 })?;
2769 let error = match reason {
2770 mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2771 OpsLifecycleError::AlreadyRegistered(id.clone())
2772 }
2773 mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2774 let limit = max_concurrent_limit.ok_or_else(|| {
2775 OpsLifecycleError::Internal(
2776 "generated op registration authority rejected capacity without limit"
2777 .into(),
2778 )
2779 })?;
2780 OpsLifecycleError::MaxConcurrentExceeded {
2781 limit: limit as usize,
2782 active: *active_op_count as usize,
2783 }
2784 }
2785 };
2786 Some(error)
2787 }
2788 });
2789 }
2790 admission.ok_or_else(|| {
2791 OpsLifecycleError::Internal(
2792 "generated op registration authority emitted no admission result".into(),
2793 )
2794 })
2795}
2796
2797impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2798 fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2799 self.register_operation_with_admission_limit(spec, None)
2800 }
2801
2802 fn register_operation_with_admission_limit(
2803 &self,
2804 mut spec: OperationSpec,
2805 max_concurrent: Option<usize>,
2806 ) -> Result<(), OpsLifecycleError> {
2807 let mut state = self.write_state()?;
2808 state.ensure_owner_active()?;
2809 let operation_id = spec.id.clone();
2810 let kind = spec.kind;
2811 let max_concurrent = max_concurrent
2812 .or(state.max_concurrent)
2813 .map(|limit| limit as u64);
2814
2815 let effects = state.dsl_apply_with_effects(
2816 mm_dsl::MeerkatMachineInput::RegisterOp {
2817 operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2818 kind: mm_dsl::OperationKind::from_domain(&kind),
2819 source: spec
2820 .operation_source
2821 .as_ref()
2822 .map(mm_dsl::OperationSource::from_domain),
2823 max_concurrent,
2824 },
2825 "RegisterOp",
2826 )?;
2827 if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
2828 return Err(error);
2829 }
2830
2831 let authority_operation_source = state.operation_source(&operation_id)?;
2832 ShellState::align_spec_child_session_id_to_source(
2833 &mut spec,
2834 authority_operation_source.as_ref(),
2835 );
2836
2837 state.records.insert(operation_id, ShellRecord::new(spec));
2839 Ok(())
2840 }
2841
2842 fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2843 let mut state = self.write_state()?;
2844
2845 apply_op_transition(
2846 &mut state,
2847 id,
2848 mm_dsl::MeerkatMachineInput::StartOp {
2849 operation_id: mm_dsl::OperationId::from_domain(id).0,
2850 },
2851 mm_dsl::OpLifecycleActionKind::Start,
2852 )?;
2853
2854 if let Some(shell) = state.records.get_mut(id) {
2856 shell.started_at = Some(Instant::now());
2857 }
2858 Ok(())
2859 }
2860
2861 fn provisioning_failed(
2862 &self,
2863 id: &OperationId,
2864 error: String,
2865 ) -> Result<(), OpsLifecycleError> {
2866 let mut state = self.write_state()?;
2867
2868 let terminal_outcome = OperationTerminalOutcome::Failed { error };
2869 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2870
2871 apply_terminal_op_transition_and_persist(
2872 &mut state,
2873 id,
2874 mm_dsl::MeerkatMachineInput::FailOp {
2875 operation_id: mm_dsl::OperationId::from_domain(id).0,
2876 outcome: outcome_kind,
2877 payload: terminal_outcome,
2878 },
2879 mm_dsl::OpLifecycleActionKind::Fail,
2880 )?;
2881
2882 let completion_entry = state.finalize_terminal(id)?;
2883 state.publish_completion_entry(completion_entry);
2884 Ok(())
2885 }
2886
2887 fn peer_ready(
2888 &self,
2889 id: &OperationId,
2890 peer: OperationPeerHandle,
2891 ) -> Result<(), OpsLifecycleError> {
2892 let mut state = self.write_state()?;
2893
2894 apply_op_transition(
2895 &mut state,
2896 id,
2897 mm_dsl::MeerkatMachineInput::PeerReadyOp {
2898 operation_id: mm_dsl::OperationId::from_domain(id).0,
2899 },
2900 mm_dsl::OpLifecycleActionKind::PeerReady,
2901 )?;
2902
2903 if let Some(shell) = state.records.get_mut(id) {
2905 shell.peer_handle = Some(peer);
2906 }
2907 Ok(())
2908 }
2909
2910 fn register_watcher(
2911 &self,
2912 id: &OperationId,
2913 ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
2914 let mut state = self.write_state()?;
2915
2916 if !state.contains(id) {
2917 return Err(OpsLifecycleError::NotFound(id.clone()));
2918 }
2919
2920 if let Some(outcome) = state.terminal_outcome(id)? {
2922 return Ok(resolved_operation_completion_watch(outcome));
2923 }
2924
2925 let shell = state.shell_record_mut(id)?;
2927 let (tx, rx) = tokio::sync::oneshot::channel();
2928 let watch = operation_completion_watch_from_receiver(rx);
2929 shell.watchers.push(OperationCompletionNotifier::new(tx));
2930 Ok(watch)
2931 }
2932
2933 fn report_progress(
2934 &self,
2935 id: &OperationId,
2936 _update: OperationProgressUpdate,
2937 ) -> Result<(), OpsLifecycleError> {
2938 let mut state = self.write_state()?;
2939
2940 apply_op_transition(
2941 &mut state,
2942 id,
2943 mm_dsl::MeerkatMachineInput::ProgressReportedOp {
2944 operation_id: mm_dsl::OperationId::from_domain(id).0,
2945 },
2946 mm_dsl::OpLifecycleActionKind::ProgressReported,
2947 )?;
2948 Ok(())
2949 }
2950
2951 fn complete_operation(
2952 &self,
2953 id: &OperationId,
2954 result: OperationResult,
2955 ) -> Result<(), OpsLifecycleError> {
2956 let mut state = self.write_state()?;
2957
2958 let terminal_outcome = OperationTerminalOutcome::Completed(result);
2959 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2960
2961 apply_terminal_op_transition_and_persist(
2962 &mut state,
2963 id,
2964 mm_dsl::MeerkatMachineInput::CompleteOp {
2965 operation_id: mm_dsl::OperationId::from_domain(id).0,
2966 outcome: outcome_kind,
2967 payload: terminal_outcome,
2968 },
2969 mm_dsl::OpLifecycleActionKind::Complete,
2970 )?;
2971
2972 let completion_entry = state.finalize_terminal(id)?;
2973 state.publish_completion_entry(completion_entry);
2974 Ok(())
2975 }
2976
2977 fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
2978 let mut state = self.write_state()?;
2979
2980 let terminal_outcome = OperationTerminalOutcome::Failed { error };
2981 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2982
2983 apply_terminal_op_transition_and_persist(
2984 &mut state,
2985 id,
2986 mm_dsl::MeerkatMachineInput::FailOp {
2987 operation_id: mm_dsl::OperationId::from_domain(id).0,
2988 outcome: outcome_kind,
2989 payload: terminal_outcome,
2990 },
2991 mm_dsl::OpLifecycleActionKind::Fail,
2992 )?;
2993
2994 let completion_entry = state.finalize_terminal(id)?;
2995 state.publish_completion_entry(completion_entry);
2996 Ok(())
2997 }
2998
2999 fn abort_provisioning(
3000 &self,
3001 id: &OperationId,
3002 reason: Option<String>,
3003 ) -> Result<(), OpsLifecycleError> {
3004 let mut state = self.write_state()?;
3005
3006 let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
3007 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3008
3009 apply_terminal_op_transition_and_persist(
3010 &mut state,
3011 id,
3012 mm_dsl::MeerkatMachineInput::AbortOp {
3013 operation_id: mm_dsl::OperationId::from_domain(id).0,
3014 outcome: outcome_kind,
3015 payload: terminal_outcome,
3016 },
3017 mm_dsl::OpLifecycleActionKind::Abort,
3018 )?;
3019
3020 let completion_entry = state.finalize_terminal(id)?;
3021 state.publish_completion_entry(completion_entry);
3022 Ok(())
3023 }
3024
3025 fn cancel_operation(
3026 &self,
3027 id: &OperationId,
3028 reason: Option<String>,
3029 ) -> Result<(), OpsLifecycleError> {
3030 let mut state = self.write_state()?;
3031
3032 let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
3033 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3034
3035 apply_terminal_op_transition_and_persist(
3036 &mut state,
3037 id,
3038 mm_dsl::MeerkatMachineInput::CancelOp {
3039 operation_id: mm_dsl::OperationId::from_domain(id).0,
3040 outcome: outcome_kind,
3041 payload: terminal_outcome,
3042 },
3043 mm_dsl::OpLifecycleActionKind::Cancel,
3044 )?;
3045
3046 let completion_entry = state.finalize_terminal(id)?;
3047 state.publish_completion_entry(completion_entry);
3048 Ok(())
3049 }
3050
3051 fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3052 let mut state = self.write_state()?;
3053
3054 apply_op_transition(
3055 &mut state,
3056 id,
3057 mm_dsl::MeerkatMachineInput::RetireRequestedOp {
3058 operation_id: mm_dsl::OperationId::from_domain(id).0,
3059 },
3060 mm_dsl::OpLifecycleActionKind::RetireRequested,
3061 )?;
3062 Ok(())
3063 }
3064
3065 fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3066 let mut state = self.write_state()?;
3067
3068 let terminal_outcome = OperationTerminalOutcome::Retired;
3069 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3070
3071 apply_terminal_op_transition_and_persist(
3072 &mut state,
3073 id,
3074 mm_dsl::MeerkatMachineInput::RetireCompletedOp {
3075 operation_id: mm_dsl::OperationId::from_domain(id).0,
3076 outcome: outcome_kind,
3077 payload: terminal_outcome,
3078 },
3079 mm_dsl::OpLifecycleActionKind::RetireCompleted,
3080 )?;
3081
3082 let completion_entry = state.finalize_terminal(id)?;
3083 state.publish_completion_entry(completion_entry);
3084 Ok(())
3085 }
3086
3087 fn snapshot(
3088 &self,
3089 id: &OperationId,
3090 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3091 let state = self.read_state()?;
3092 state.snapshot(id)
3093 }
3094
3095 fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3096 let state = self.read_state()?;
3097 let mut snapshots = Vec::new();
3098 for id in state.operation_ids()? {
3099 let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3100 OpsLifecycleError::Internal(format!(
3101 "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3102 ))
3103 })?;
3104 snapshots.push(snapshot);
3105 }
3106 snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3107 Ok(snapshots)
3108 }
3109
3110 fn classify_operation_terminality(
3111 &self,
3112 id: &OperationId,
3113 status: OperationStatus,
3114 ) -> Result<bool, OpsLifecycleError> {
3115 ShellState::operation_status_is_terminal(id, status)
3116 }
3117
3118 fn classify_operation_public_result(
3119 &self,
3120 id: &OperationId,
3121 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3122 let state = self.read_state()?;
3123 let status = match state.status(id) {
3124 Some(status) => status,
3125 None if state.records.contains_key(id)
3126 || state.has_generated_operation_record_fact(id) =>
3127 {
3128 return Err(OpsLifecycleError::Internal(format!(
3129 "generated op lifecycle authority missing status for {id}"
3130 )));
3131 }
3132 None => OperationStatus::Absent,
3133 };
3134 ShellState::operation_public_result_class(id, status)
3135 }
3136
3137 fn classify_operation_completion_wake(
3138 &self,
3139 id: &OperationId,
3140 kind: OperationKind,
3141 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3142 ShellState::operation_completion_wake_class(id, kind)
3143 }
3144
3145 fn classify_operation_transition_idempotence(
3146 &self,
3147 id: &OperationId,
3148 action: OperationLifecycleAction,
3149 ) -> Result<bool, OpsLifecycleError> {
3150 let state = self.read_state()?;
3151 let status = match state.status(id) {
3152 Some(status) => status,
3153 None if state.records.contains_key(id)
3154 || state.has_generated_operation_record_fact(id) =>
3155 {
3156 return Err(OpsLifecycleError::Internal(format!(
3157 "generated op lifecycle authority missing status for {id}"
3158 )));
3159 }
3160 None => OperationStatus::Absent,
3161 };
3162 ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3163 }
3164
3165 fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3166 let mut state = self.write_state()?;
3167 if state.owner_retired {
3168 return Ok(());
3169 }
3170 terminate_owner_locked(&mut state, &reason)
3171 }
3172
3173 fn collect_completed(
3174 &self,
3175 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3176 let mut state = self.write_state()?;
3177 state.ensure_owner_active()?;
3178
3179 let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3180 let mut collected = Vec::with_capacity(ids.len());
3181 for id in ids {
3182 let outcome = state.terminal_outcome(&id)?;
3183 state.dsl_apply(
3184 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3185 operation_id: mm_dsl::OperationId::from_domain(&id).0,
3186 },
3187 "CollectCompletedOp",
3188 )?;
3189 state.completed_order.retain(|queued| queued != &id);
3190 state.records.remove(&id);
3191 if let Some(outcome) = outcome {
3192 collected.push((id, outcome));
3193 }
3194 }
3195 Ok(collected)
3196 }
3197
3198 fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3199 Some(self.completion_feed_handle())
3200 }
3201
3202 fn completion_cursor(
3203 &self,
3204 consumer: CompletionCursorConsumer,
3205 ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3206 let state = self.read_state()?;
3207 Ok(Some(state.completion_cursor(consumer)))
3208 }
3209
3210 fn advance_completion_cursor(
3211 &self,
3212 consumer: CompletionCursorConsumer,
3213 cursor: CompletionSeq,
3214 projection: Option<&meerkat_core::EpochCursorState>,
3215 ) -> Result<CompletionSeq, OpsLifecycleError> {
3216 let mut state = self.write_state()?;
3217 state.ensure_owner_active()?;
3218 let input = match consumer {
3219 CompletionCursorConsumer::AgentApplied => {
3220 mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3221 }
3222 CompletionCursorConsumer::RuntimeObserved => {
3223 mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3224 }
3225 CompletionCursorConsumer::RuntimeInjected => {
3226 mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3227 }
3228 };
3229 let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3230 let advanced = effects
3231 .iter()
3232 .find_map(|effect| match (consumer, effect) {
3233 (
3234 CompletionCursorConsumer::AgentApplied,
3235 mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3236 ) => Some(*cursor),
3237 (
3238 CompletionCursorConsumer::RuntimeObserved,
3239 mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3240 cursor,
3241 },
3242 ) => Some(*cursor),
3243 (
3244 CompletionCursorConsumer::RuntimeInjected,
3245 mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3246 cursor,
3247 },
3248 ) => Some(*cursor),
3249 _ => None,
3250 })
3251 .ok_or_else(|| {
3252 OpsLifecycleError::Internal(format!(
3253 "generated completion cursor transition emitted no feedback for {consumer:?}"
3254 ))
3255 })?;
3256 if let Some(projection) = projection {
3257 projection.project_authorized_completion_cursor(consumer, advanced);
3258 }
3259 Ok(advanced)
3260 }
3261
3262 fn wait_all(
3263 &self,
3264 run_id: &RunId,
3265 ids: &[OperationId],
3266 ) -> std::pin::Pin<
3267 Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3268 > {
3269 let wait_request_id = WaitRequestId::new();
3270 let owned_ids = ids.to_vec();
3271
3272 let state = match self.write_state() {
3273 Ok(mut state) => {
3274 if let Err(error) = state.ensure_owner_active() {
3275 return Box::pin(WaitAllFuture {
3276 registry: self,
3277 wait_request_id,
3278 state: WaitAllFutureState::Ready(Some(Err(error))),
3279 });
3280 }
3281 match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3282 Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3283 let outcomes =
3284 state
3285 .collect_wait_outcomes(&satisfied.operation_ids)
3286 .map(|outcomes| WaitAllResult {
3287 outcomes,
3288 satisfied,
3289 });
3290 WaitAllFutureState::Ready(Some(outcomes))
3291 }
3292 Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3293 if state.pending_wait.is_some() {
3294 let rollback = state.dsl_apply(
3300 mm_dsl::MeerkatMachineInput::CancelWaitAll,
3301 "CancelWaitAll(rollback)",
3302 );
3303 return Box::pin(WaitAllFuture {
3304 registry: self,
3305 wait_request_id,
3306 state: WaitAllFutureState::Ready(Some(Err(match rollback {
3307 Ok(()) => OpsLifecycleError::Internal(
3308 "wait_all started while a pending wait sender already existed"
3309 .into(),
3310 ),
3311 Err(err) => err,
3312 }))),
3313 });
3314 }
3315 state.wait_request_id = Some(wait_request_id.clone());
3316 let (sender, receiver) = tokio::sync::oneshot::channel();
3317 state.pending_wait = Some(PendingWaitState {
3318 wait_request_id: wait_request_id.clone(),
3319 sender,
3320 });
3321 WaitAllFutureState::Waiting(receiver)
3322 }
3323 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3324 }
3325 }
3326 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3327 };
3328
3329 Box::pin(WaitAllFuture {
3330 registry: self,
3331 wait_request_id,
3332 state,
3333 })
3334 }
3335}
3336
3337#[cfg(test)]
3338#[allow(clippy::unwrap_used, clippy::panic)]
3339mod tests {
3340 use super::*;
3341 use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3342 use meerkat_core::lifecycle::RunId;
3343 use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3344 use meerkat_core::types::SessionId;
3345 use std::sync::atomic::Ordering;
3346 use uuid::Uuid;
3347
3348 fn test_run_id() -> RunId {
3349 RunId(Uuid::from_u128(1))
3350 }
3351
3352 fn background_spec(name: &str) -> OperationSpec {
3353 OperationSpec {
3354 id: OperationId::new(),
3355 kind: OperationKind::BackgroundToolOp,
3356 owner_session_id: SessionId::new(),
3357 display_name: name.into(),
3358 source_label: "test".into(),
3359 operation_source: None,
3360 child_session_id: None,
3361 expect_peer_channel: false,
3362 }
3363 }
3364
3365 #[tokio::test]
3366 async fn late_watchers_resolve_immediately() {
3367 let registry = RuntimeOpsLifecycleRegistry::new();
3368 let spec = background_spec("late");
3369 let op_id = spec.id.clone();
3370 registry.register_operation(spec).unwrap();
3371 registry.provisioning_succeeded(&op_id).unwrap();
3372 registry
3373 .complete_operation(
3374 &op_id,
3375 OperationResult {
3376 id: op_id.clone(),
3377 content: "done".into(),
3378 is_error: false,
3379 duration_ms: 1,
3380 tokens_used: 0,
3381 },
3382 )
3383 .unwrap();
3384
3385 let watch = registry.register_watcher(&op_id).unwrap();
3386 match watch
3387 .await
3388 .expect("operation completion watch should resolve")
3389 {
3390 OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3391 other => panic!("expected completed outcome, got {other:?}"),
3392 }
3393 }
3394
3395 #[tokio::test]
3396 async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3397 let (tx, rx) = tokio::sync::oneshot::channel();
3398 let watch = operation_completion_watch_from_receiver(rx);
3399 drop(tx);
3400
3401 assert_eq!(
3402 watch.await,
3403 Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3404 );
3405 }
3406
3407 #[test]
3408 fn peer_ready_requires_peer_expectation() {
3409 let registry = RuntimeOpsLifecycleRegistry::new();
3410 let spec = background_spec("no-peer");
3411 let op_id = spec.id.clone();
3412 registry.register_operation(spec).unwrap();
3413 registry.provisioning_succeeded(&op_id).unwrap();
3414
3415 let result = registry.peer_ready(
3416 &op_id,
3417 OperationPeerHandle {
3418 peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3419 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3420 "peer",
3421 PeerId::new(),
3422 "inproc://peer",
3423 )
3424 .unwrap(),
3425 },
3426 );
3427 assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3428 }
3429
3430 #[test]
3436 fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3437 let op_id = OperationId::new();
3438 let outcomes = vec![
3439 (
3440 OperationTerminalOutcome::Completed(OperationResult {
3441 id: op_id.clone(),
3442 content: "done".into(),
3443 is_error: false,
3444 duration_ms: 7,
3445 tokens_used: 42,
3446 }),
3447 mm_dsl::OperationTerminalOutcomeKind::Completed,
3448 ),
3449 (
3450 OperationTerminalOutcome::Failed {
3451 error: "boom".into(),
3452 },
3453 mm_dsl::OperationTerminalOutcomeKind::Failed,
3454 ),
3455 (
3456 OperationTerminalOutcome::Aborted {
3457 reason: Some("user aborted".into()),
3458 },
3459 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3460 ),
3461 (
3462 OperationTerminalOutcome::Aborted { reason: None },
3463 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3464 ),
3465 (
3466 OperationTerminalOutcome::Cancelled {
3467 reason: Some("cancelled".into()),
3468 },
3469 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3470 ),
3471 (
3472 OperationTerminalOutcome::Cancelled { reason: None },
3473 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3474 ),
3475 (
3476 OperationTerminalOutcome::Retired,
3477 mm_dsl::OperationTerminalOutcomeKind::Retired,
3478 ),
3479 (
3480 OperationTerminalOutcome::Terminated {
3481 reason: "owner stopped".into(),
3482 },
3483 mm_dsl::OperationTerminalOutcomeKind::Terminated,
3484 ),
3485 ];
3486
3487 for (outcome, expected_kind) in &outcomes {
3488 assert_eq!(
3489 mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3490 *expected_kind,
3491 "typed payload {outcome:?} must classify to {expected_kind:?}"
3492 );
3493 let read = ShellState::checked_terminal_payload(
3494 *expected_kind,
3495 outcome,
3496 "test authority",
3497 "test-op",
3498 )
3499 .expect("matching discriminant must read back the exact payload");
3500 assert_eq!(&read, outcome);
3501 }
3502
3503 let err = ShellState::checked_terminal_payload(
3505 mm_dsl::OperationTerminalOutcomeKind::Completed,
3506 &OperationTerminalOutcome::Retired,
3507 "test authority",
3508 "test-op",
3509 )
3510 .expect_err("variant mismatch must be rejected");
3511 assert!(matches!(err, OpsLifecycleError::Internal(_)));
3512 }
3513
3514 #[test]
3520 fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3521 let registry = RuntimeOpsLifecycleRegistry::new();
3522 let spec = background_spec("variant-mismatch");
3523 let op_id = spec.id.clone();
3524 registry.register_operation(spec).unwrap();
3525 registry.provisioning_succeeded(&op_id).unwrap();
3526
3527 let mut state = registry.write_state().unwrap();
3528 let err = state
3529 .dsl_apply(
3530 mm_dsl::MeerkatMachineInput::CompleteOp {
3531 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3532 outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3533 payload: OperationTerminalOutcome::Retired,
3535 },
3536 "CompleteOp",
3537 )
3538 .expect_err("machine must reject payload variant mismatch");
3539 assert!(
3546 matches!(
3547 &err,
3548 OpsLifecycleError::Internal(message)
3549 if message.contains("GuardRejected") && message.contains("CompleteOp")
3550 ),
3551 "expected generated guard rejection for CompleteOp, got: {err:?}"
3552 );
3553 drop(state);
3554
3555 registry
3558 .complete_operation(
3559 &op_id,
3560 OperationResult {
3561 id: op_id.clone(),
3562 content: "done".into(),
3563 is_error: false,
3564 duration_ms: 1,
3565 tokens_used: 0,
3566 },
3567 )
3568 .expect("matching variant must complete");
3569 }
3570
3571 #[test]
3572 fn duplicate_registration_rejection_is_generated() {
3573 let registry = RuntimeOpsLifecycleRegistry::new();
3574 let spec = background_spec("duplicate");
3575 let op_id = spec.id.clone();
3576
3577 registry.register_operation(spec.clone()).unwrap();
3578 let result = registry.register_operation(spec);
3579
3580 assert!(matches!(
3581 result,
3582 Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3583 ));
3584 }
3585
3586 #[test]
3587 fn invalid_transition_rejection_is_generated() {
3588 let registry = RuntimeOpsLifecycleRegistry::new();
3589 let spec = background_spec("invalid-transition");
3590 let op_id = spec.id.clone();
3591 registry.register_operation(spec).unwrap();
3592
3593 let result = registry.complete_operation(
3594 &op_id,
3595 OperationResult {
3596 id: op_id.clone(),
3597 content: "too-early".into(),
3598 is_error: false,
3599 duration_ms: 1,
3600 tokens_used: 0,
3601 },
3602 );
3603
3604 assert!(matches!(
3605 result,
3606 Err(OpsLifecycleError::InvalidTransition {
3607 id,
3608 status: OperationStatus::Provisioning,
3609 action: "complete_operation",
3610 }) if id == op_id
3611 ));
3612 }
3613
3614 #[tokio::test]
3615 async fn multi_listener_completion() {
3616 let registry = RuntimeOpsLifecycleRegistry::new();
3617 let spec = background_spec("multi");
3618 let op_id = spec.id.clone();
3619 registry.register_operation(spec).unwrap();
3620 registry.provisioning_succeeded(&op_id).unwrap();
3621
3622 let watch1 = registry.register_watcher(&op_id).unwrap();
3623 let watch2 = registry.register_watcher(&op_id).unwrap();
3624 let watch3 = registry.register_watcher(&op_id).unwrap();
3625
3626 registry
3627 .complete_operation(
3628 &op_id,
3629 OperationResult {
3630 id: op_id.clone(),
3631 content: "multi-done".into(),
3632 is_error: false,
3633 duration_ms: 1,
3634 tokens_used: 0,
3635 },
3636 )
3637 .unwrap();
3638
3639 for watch in [watch1, watch2, watch3] {
3640 match watch
3641 .await
3642 .expect("operation completion watch should resolve")
3643 {
3644 OperationTerminalOutcome::Completed(result) => {
3645 assert_eq!(result.content, "multi-done");
3646 }
3647 other => panic!("expected completed, got {other:?}"),
3648 }
3649 }
3650 }
3651
3652 #[tokio::test]
3653 async fn wait_all_returns_all_outcomes() {
3654 let registry = RuntimeOpsLifecycleRegistry::new();
3655
3656 let spec_a = background_spec("a");
3657 let id_a = spec_a.id.clone();
3658 registry.register_operation(spec_a).unwrap();
3659 registry.provisioning_succeeded(&id_a).unwrap();
3660
3661 let spec_b = background_spec("b");
3662 let id_b = spec_b.id.clone();
3663 registry.register_operation(spec_b).unwrap();
3664 registry.provisioning_succeeded(&id_b).unwrap();
3665
3666 registry
3667 .complete_operation(
3668 &id_a,
3669 OperationResult {
3670 id: id_a.clone(),
3671 content: "a-done".into(),
3672 is_error: false,
3673 duration_ms: 1,
3674 tokens_used: 0,
3675 },
3676 )
3677 .unwrap();
3678 registry.fail_operation(&id_b, "b-error".into()).unwrap();
3679
3680 let wait_result = registry
3681 .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3682 .await
3683 .unwrap();
3684 assert_eq!(wait_result.outcomes.len(), 2);
3685 assert_eq!(wait_result.outcomes[0].0, id_a);
3686 assert!(matches!(
3687 wait_result.outcomes[0].1,
3688 OperationTerminalOutcome::Completed(_)
3689 ));
3690 assert_eq!(wait_result.outcomes[1].0, id_b);
3691 assert!(matches!(
3692 wait_result.outcomes[1].1,
3693 OperationTerminalOutcome::Failed { .. }
3694 ));
3695 assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3697 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3698 }
3699
3700 #[tokio::test]
3703 async fn wait_all_trait_path_submits_through_authority() {
3704 let registry = RuntimeOpsLifecycleRegistry::new();
3705 let spec = background_spec("trait-wait");
3706 let op_id = spec.id.clone();
3707 registry.register_operation(spec).unwrap();
3708 registry.provisioning_succeeded(&op_id).unwrap();
3709 registry
3710 .complete_operation(
3711 &op_id,
3712 OperationResult {
3713 id: op_id.clone(),
3714 content: "done".into(),
3715 is_error: false,
3716 duration_ms: 1,
3717 tokens_used: 0,
3718 },
3719 )
3720 .unwrap();
3721
3722 let trait_ref: &dyn OpsLifecycleRegistry = ®istry;
3724 let wait_result = trait_ref
3725 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3726 .await
3727 .unwrap();
3728 assert_eq!(wait_result.outcomes.len(), 1);
3729 assert!(matches!(
3730 wait_result.outcomes[0].1,
3731 OperationTerminalOutcome::Completed(_)
3732 ));
3733 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3735 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3736 let state = registry.read_state().unwrap();
3737 assert!(
3738 !state.wait_active(),
3739 "already-satisfied wait_all must be cleared by generated satisfaction authority"
3740 );
3741 assert!(state.wait_operation_ids().unwrap().is_empty());
3742 }
3743
3744 #[tokio::test]
3745 async fn wait_all_duplicate_rejection_is_generated() {
3746 let registry = RuntimeOpsLifecycleRegistry::new();
3747 let spec = background_spec("duplicate-wait");
3748 let op_id = spec.id.clone();
3749 registry.register_operation(spec).unwrap();
3750 registry.provisioning_succeeded(&op_id).unwrap();
3751
3752 let result = registry
3753 .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3754 .await;
3755
3756 assert!(matches!(
3757 result,
3758 Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3759 ));
3760 let state = registry.read_state().unwrap();
3761 assert!(
3762 !state.wait_active(),
3763 "duplicate wait rejection must not create a shell or machine barrier"
3764 );
3765 assert!(state.wait_operation_ids().unwrap().is_empty());
3766 }
3767
3768 #[tokio::test]
3769 async fn wait_all_active_rejection_is_generated() {
3770 let registry = RuntimeOpsLifecycleRegistry::new();
3771 let spec = background_spec("active-wait");
3772 let op_id = spec.id.clone();
3773 registry.register_operation(spec).unwrap();
3774 registry.provisioning_succeeded(&op_id).unwrap();
3775
3776 let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3777 let result = registry
3778 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3779 .await;
3780
3781 assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3782 drop(active_wait);
3783 let state = registry.read_state().unwrap();
3784 assert!(!state.wait_active());
3785 assert!(state.wait_operation_ids().unwrap().is_empty());
3786 }
3787
3788 #[tokio::test]
3789 async fn wait_all_unknown_operation_rejection_is_generated() {
3790 let registry = RuntimeOpsLifecycleRegistry::new();
3791 let op_id = OperationId::new();
3792
3793 let result = registry
3794 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3795 .await;
3796
3797 assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3798 let state = registry.read_state().unwrap();
3799 assert!(!state.wait_active());
3800 assert!(state.wait_operation_ids().unwrap().is_empty());
3801 }
3802
3803 #[tokio::test]
3804 async fn wait_all_resolves_from_authority_owned_wait_request() {
3805 let registry = RuntimeOpsLifecycleRegistry::new();
3806 let run_id = test_run_id();
3807
3808 let spec = background_spec("pending");
3809 let op_id = spec.id.clone();
3810 registry.register_operation(spec).unwrap();
3811 registry.provisioning_succeeded(&op_id).unwrap();
3812
3813 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3814 tokio::pin!(wait_fut);
3815 assert!(
3816 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3817 .await
3818 .is_err()
3819 );
3820
3821 let active_wait_request_id = {
3822 let state = registry.read_state().unwrap();
3823 let wait_request_id = match state.wait_request_id.clone() {
3824 Some(wait_request_id) => wait_request_id,
3825 None => panic!("wait request should be active"),
3826 };
3827 assert_eq!(
3828 state.wait_operation_ids().unwrap().as_slice(),
3829 std::slice::from_ref(&op_id)
3830 );
3831 wait_request_id
3832 };
3833
3834 registry
3835 .complete_operation(
3836 &op_id,
3837 OperationResult {
3838 id: op_id.clone(),
3839 content: "done".into(),
3840 is_error: false,
3841 duration_ms: 1,
3842 tokens_used: 0,
3843 },
3844 )
3845 .unwrap();
3846
3847 let wait_result = wait_fut.await.unwrap();
3848 assert_eq!(
3849 wait_result.satisfied.wait_request_id,
3850 active_wait_request_id
3851 );
3852 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
3853 assert!(matches!(
3854 wait_result.outcomes.as_slice(),
3855 [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
3856 ));
3857 assert!(registry.read_state().unwrap().wait_request_id.is_none());
3858 }
3859
3860 #[tokio::test]
3861 async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
3862 let registry = RuntimeOpsLifecycleRegistry::new();
3863 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3864 registry.set_persistence_channel(
3865 tx,
3866 meerkat_core::RuntimeEpochId::new(),
3867 Arc::new(meerkat_core::EpochCursorState::new()),
3868 );
3869 let worker = std::thread::spawn(move || {
3870 let request = rx.blocking_recv().expect("persistence request");
3871 let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
3872 "injected persistence failure".to_string(),
3873 )));
3874 });
3875
3876 let spec = background_spec("feed-after-persist");
3877 let op_id = spec.id.clone();
3878 registry.register_operation(spec).unwrap();
3879 registry.provisioning_succeeded(&op_id).unwrap();
3880 let watch = registry.register_watcher(&op_id).unwrap();
3881
3882 let err = registry
3883 .complete_operation(
3884 &op_id,
3885 OperationResult {
3886 id: op_id.clone(),
3887 content: "done".into(),
3888 is_error: false,
3889 duration_ms: 1,
3890 tokens_used: 0,
3891 },
3892 )
3893 .expect_err("persistence failure must fail the terminal transition");
3894 assert!(
3895 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
3896 "unexpected persistence error: {err:?}"
3897 );
3898 worker.join().expect("persistence worker");
3899
3900 let feed = registry.completion_feed_handle();
3901 let batch = feed.list_since(0);
3902 assert!(
3903 batch.entries.is_empty(),
3904 "completion feed must not publish entries before durable snapshot success"
3905 );
3906 assert_eq!(batch.watermark, 0);
3907 assert!(
3908 tokio::time::timeout(std::time::Duration::from_millis(10), watch)
3909 .await
3910 .is_err(),
3911 "watchers must not resolve before durable terminal snapshot success"
3912 );
3913 assert_eq!(
3914 registry.snapshot(&op_id).unwrap().unwrap().status,
3915 OperationStatus::Running,
3916 "failed persistence must roll the generated terminal transition back"
3917 );
3918
3919 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3920 registry.set_persistence_channel(
3921 tx,
3922 meerkat_core::RuntimeEpochId::new(),
3923 Arc::new(meerkat_core::EpochCursorState::new()),
3924 );
3925 let worker = std::thread::spawn(move || {
3926 let request = rx.blocking_recv().expect("second persistence request");
3927 let _ = request.result_tx.send(Ok(()));
3928 });
3929 registry
3930 .complete_operation(
3931 &op_id,
3932 OperationResult {
3933 id: op_id.clone(),
3934 content: "done".into(),
3935 is_error: false,
3936 duration_ms: 1,
3937 tokens_used: 0,
3938 },
3939 )
3940 .expect("rolled-back terminal transition should remain retryable");
3941 worker.join().expect("second persistence worker");
3942
3943 let batch = feed.list_since(0);
3944 assert_eq!(batch.entries.len(), 1);
3945 assert_eq!(batch.entries[0].operation_id, op_id);
3946 }
3947
3948 #[tokio::test]
3949 async fn dropping_wait_all_future_cancels_active_wait_request() {
3950 let registry = RuntimeOpsLifecycleRegistry::new();
3951 let run_id = test_run_id();
3952
3953 let spec = background_spec("cancelled-wait");
3954 let op_id = spec.id.clone();
3955 registry.register_operation(spec).unwrap();
3956 registry.provisioning_succeeded(&op_id).unwrap();
3957
3958 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3959 drop(wait_fut);
3960
3961 let state = registry.read_state().unwrap();
3962 assert!(state.wait_request_id.is_none());
3963 assert!(state.wait_operation_ids().unwrap().is_empty());
3964 assert!(!state.wait_active());
3965 }
3966
3967 #[tokio::test]
3973 async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
3974 let registry = RuntimeOpsLifecycleRegistry::new();
3975 let run_id = test_run_id();
3976
3977 let spec = background_spec("corrupt-barrier");
3978 let op_id = spec.id.clone();
3979 registry.register_operation(spec).unwrap();
3980 registry.provisioning_succeeded(&op_id).unwrap();
3981
3982 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3984 tokio::pin!(wait_fut);
3985 assert!(
3986 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3987 .await
3988 .is_err(),
3989 "barrier waiter must still be pending before corruption"
3990 );
3991 assert!(registry.read_state().unwrap().wait_request_id.is_some());
3992
3993 {
3997 let mut state = registry.write_state().unwrap();
3998 let mut machine_state = state.dsl.0.state().clone();
3999 machine_state.wait_run_id = None;
4000 state.dsl = DslAuthority(Box::new(
4001 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4002 ));
4003 }
4004
4005 let err = registry
4008 .complete_operation(
4009 &op_id,
4010 OperationResult {
4011 id: op_id.clone(),
4012 content: "done".into(),
4013 is_error: false,
4014 duration_ms: 1,
4015 tokens_used: 0,
4016 },
4017 )
4018 .expect_err("corrupt wait authority must fail the terminal transition");
4019 assert!(
4020 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
4021 "unexpected terminal error: {err:?}"
4022 );
4023
4024 let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
4026 .await
4027 .expect("waiter must resolve, not hang, after authority corruption");
4028 match waiter_result {
4029 Err(OpsLifecycleError::Internal(message)) => assert!(
4030 message.contains("wait_all completion channel dropped"),
4031 "unexpected waiter error message: {message}"
4032 ),
4033 other => panic!("expected dropped-channel Internal error, got {other:?}"),
4034 }
4035 }
4036
4037 #[test]
4041 fn completion_cursor_propagates_poison_not_none() {
4042 let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
4043
4044 let poison_registry = std::sync::Arc::clone(®istry);
4046 let join = std::thread::spawn(move || {
4047 let _guard = poison_registry.write_state().unwrap();
4048 panic!("intentional panic to poison ops lifecycle registry lock");
4049 });
4050 assert!(
4051 join.join().is_err(),
4052 "poisoning thread must have panicked while holding the write guard"
4053 );
4054
4055 let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
4056 let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
4057 match result {
4058 Err(OpsLifecycleError::Internal(message)) => assert!(
4059 message.contains("ops lifecycle registry poisoned"),
4060 "unexpected cursor error message: {message}"
4061 ),
4062 other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
4063 }
4064 }
4065
4066 #[test]
4067 fn terminate_owner_only_targets_non_terminal_operations() {
4068 let registry = RuntimeOpsLifecycleRegistry::new();
4069
4070 let running_spec = background_spec("running");
4071 let running_id = running_spec.id.clone();
4072 registry.register_operation(running_spec).unwrap();
4073 registry.provisioning_succeeded(&running_id).unwrap();
4074
4075 let completed_spec = background_spec("completed");
4076 let completed_id = completed_spec.id.clone();
4077 registry.register_operation(completed_spec).unwrap();
4078 registry.provisioning_succeeded(&completed_id).unwrap();
4079 registry
4080 .complete_operation(
4081 &completed_id,
4082 OperationResult {
4083 id: completed_id.clone(),
4084 content: "done".into(),
4085 is_error: false,
4086 duration_ms: 1,
4087 tokens_used: 0,
4088 },
4089 )
4090 .unwrap();
4091
4092 registry.terminate_owner("shutdown".into()).unwrap();
4093
4094 assert!(matches!(
4095 registry.snapshot(&running_id).unwrap().unwrap().status,
4096 OperationStatus::Terminated
4097 ));
4098 assert!(matches!(
4099 registry.snapshot(&completed_id).unwrap().unwrap().status,
4100 OperationStatus::Completed
4101 ));
4102 }
4103
4104 #[test]
4105 fn unregister_retirement_fences_detached_late_callbacks() {
4106 let registry = RuntimeOpsLifecycleRegistry::new();
4107 let running_spec = background_spec("detached-running");
4108 let running_id = running_spec.id.clone();
4109 registry.register_operation(running_spec).unwrap();
4110 registry.provisioning_succeeded(&running_id).unwrap();
4111
4112 registry
4113 .retire_owner_for_unregister("owner unregistered".into())
4114 .unwrap();
4115 assert!(matches!(
4116 registry.snapshot(&running_id).unwrap().unwrap().status,
4117 OperationStatus::Terminated
4118 ));
4119
4120 let late_result = OperationResult {
4121 id: running_id.clone(),
4122 content: "late detached completion".into(),
4123 is_error: false,
4124 duration_ms: 1,
4125 tokens_used: 0,
4126 };
4127 assert_eq!(
4128 registry.complete_operation(&running_id, late_result),
4129 Err(OpsLifecycleError::OwnerRetired)
4130 );
4131 assert_eq!(
4132 registry.register_operation(background_spec("late-new-op")),
4133 Err(OpsLifecycleError::OwnerRetired)
4134 );
4135 assert_eq!(
4136 registry.report_progress(
4137 &running_id,
4138 OperationProgressUpdate {
4139 message: "late progress".into(),
4140 percent: None,
4141 },
4142 ),
4143 Err(OpsLifecycleError::OwnerRetired)
4144 );
4145 }
4146
4147 #[test]
4148 fn collect_completed_drains_terminal_operations() {
4149 let registry = RuntimeOpsLifecycleRegistry::new();
4150
4151 let spec_a = background_spec("a");
4152 let id_a = spec_a.id.clone();
4153 registry.register_operation(spec_a).unwrap();
4154 registry.provisioning_succeeded(&id_a).unwrap();
4155 registry
4156 .complete_operation(
4157 &id_a,
4158 OperationResult {
4159 id: id_a.clone(),
4160 content: "done".into(),
4161 is_error: false,
4162 duration_ms: 1,
4163 tokens_used: 0,
4164 },
4165 )
4166 .unwrap();
4167
4168 let spec_b = background_spec("b");
4169 let id_b = spec_b.id.clone();
4170 registry.register_operation(spec_b).unwrap();
4171
4172 let collected = registry.collect_completed().unwrap();
4173 assert_eq!(collected.len(), 1);
4174 assert_eq!(collected[0].0, id_a);
4175
4176 assert!(registry.snapshot(&id_a).unwrap().is_none());
4177 assert!(registry.snapshot(&id_b).unwrap().is_some());
4178
4179 let collected2 = registry.collect_completed().unwrap();
4180 assert!(collected2.is_empty());
4181 }
4182
4183 #[test]
4184 fn bounded_completed_retention_evicts_oldest() {
4185 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4186 max_completed: 3,
4187 max_concurrent: None,
4188 });
4189
4190 let mut ids = Vec::new();
4191 for i in 0..5 {
4192 let spec = background_spec(&format!("op-{i}"));
4193 let id = spec.id.clone();
4194 registry.register_operation(spec).unwrap();
4195 registry.provisioning_succeeded(&id).unwrap();
4196 registry
4197 .complete_operation(
4198 &id,
4199 OperationResult {
4200 id: id.clone(),
4201 content: format!("done-{i}"),
4202 is_error: false,
4203 duration_ms: 1,
4204 tokens_used: 0,
4205 },
4206 )
4207 .unwrap();
4208 ids.push(id);
4209 }
4210
4211 assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4212 assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4213 assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4214 assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4215 assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4216 }
4217
4218 #[test]
4219 fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4220 let registry = RuntimeOpsLifecycleRegistry::new();
4221
4222 let completed_spec = background_spec("completed");
4223 let completed_id = completed_spec.id.clone();
4224 registry.register_operation(completed_spec).unwrap();
4225 registry.provisioning_succeeded(&completed_id).unwrap();
4226 registry
4227 .complete_operation(
4228 &completed_id,
4229 OperationResult {
4230 id: completed_id.clone(),
4231 content: "done".into(),
4232 is_error: false,
4233 duration_ms: 1,
4234 tokens_used: 0,
4235 },
4236 )
4237 .unwrap();
4238
4239 let running_spec = background_spec("running");
4240 let running_id = running_spec.id.clone();
4241 registry.register_operation(running_spec).unwrap();
4242 registry.provisioning_succeeded(&running_id).unwrap();
4243
4244 let cursor_state = meerkat_core::EpochCursorState::new();
4245 let snapshot = registry
4246 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4247 .unwrap();
4248 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4249
4250 assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4251 assert!(recovered.snapshot(&running_id).unwrap().is_none());
4252
4253 let collected = recovered.collect_completed().unwrap();
4254 assert_eq!(collected.len(), 1);
4255 assert_eq!(collected[0].0, completed_id);
4256 }
4257
4258 #[test]
4259 fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4260 let registry = RuntimeOpsLifecycleRegistry::new();
4261
4262 let mut spec = background_spec("capacity");
4263 spec.kind = OperationKind::BackgroundToolCapacitySlot;
4264 let operation_id = spec.id.clone();
4265 registry.register_operation(spec).unwrap();
4266 registry.provisioning_succeeded(&operation_id).unwrap();
4267 registry.mark_retired(&operation_id).unwrap();
4268
4269 assert!(registry.snapshot(&operation_id).unwrap().is_none());
4270
4271 let cursor_state = meerkat_core::EpochCursorState::new();
4272 let snapshot = registry
4273 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4274 .unwrap();
4275 assert!(
4276 !snapshot
4277 .authority_state
4278 .operations
4279 .contains_key(&operation_id)
4280 );
4281 assert!(!snapshot.operation_specs.contains_key(&operation_id));
4282 assert!(snapshot.completion_entries.is_empty());
4283
4284 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4285 assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4286 }
4287
4288 #[test]
4289 fn recovered_snapshot_uses_authority_operation_source() {
4290 let registry = RuntimeOpsLifecycleRegistry::new();
4291 let child_session_id = SessionId::new();
4292 let operation_source = OperationSource::session_child(child_session_id.clone());
4293 let spec = OperationSpec {
4294 id: OperationId::new(),
4295 kind: OperationKind::MobMemberChild,
4296 owner_session_id: SessionId::new(),
4297 display_name: "source-recovery".into(),
4298 source_label: "test".into(),
4299 operation_source: Some(operation_source.clone()),
4300 child_session_id: Some(child_session_id),
4301 expect_peer_channel: true,
4302 };
4303 let operation_id = spec.id.clone();
4304
4305 registry.register_operation(spec).unwrap();
4306 registry.provisioning_succeeded(&operation_id).unwrap();
4307 registry.mark_retired(&operation_id).unwrap();
4308
4309 let cursor_state = meerkat_core::EpochCursorState::new();
4310 let mut snapshot = registry
4311 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4312 .unwrap();
4313 assert_eq!(
4314 snapshot
4315 .authority_state
4316 .operations
4317 .get(&operation_id)
4318 .and_then(|state| state.operation_source.as_ref()),
4319 Some(&operation_source)
4320 );
4321
4322 snapshot
4323 .operation_specs
4324 .get_mut(&operation_id)
4325 .expect("persisted spec")
4326 .operation_source = None;
4327 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4328 assert_eq!(
4329 recovered
4330 .snapshot(&operation_id)
4331 .unwrap()
4332 .unwrap()
4333 .operation_source,
4334 Some(operation_source)
4335 );
4336 }
4337
4338 #[test]
4339 fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4340 let registry = RuntimeOpsLifecycleRegistry::new();
4341 let child_session_id = SessionId::new();
4342 let operation_source = OperationSource::session_child(child_session_id.clone());
4343 let spec = OperationSpec {
4344 id: OperationId::new(),
4345 kind: OperationKind::MobMemberChild,
4346 owner_session_id: SessionId::new(),
4347 display_name: "source-drift".into(),
4348 source_label: "test".into(),
4349 operation_source: Some(operation_source),
4350 child_session_id: Some(child_session_id),
4351 expect_peer_channel: true,
4352 };
4353 let operation_id = spec.id.clone();
4354
4355 registry.register_operation(spec).unwrap();
4356 registry.provisioning_succeeded(&operation_id).unwrap();
4357 registry.mark_retired(&operation_id).unwrap();
4358
4359 let cursor_state = meerkat_core::EpochCursorState::new();
4360 let mut snapshot = registry
4361 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4362 .unwrap();
4363 snapshot
4364 .operation_specs
4365 .get_mut(&operation_id)
4366 .expect("persisted spec")
4367 .operation_source = Some(OperationSource::session_child(SessionId::new()));
4368
4369 let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4370 .expect_err("source mirror drift must fail recovery");
4371 assert!(
4372 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4373 "unexpected recovery error: {err:?}"
4374 );
4375 }
4376
4377 #[test]
4378 fn persisted_authority_state_serializes_explicit_no_operation_source() {
4379 let registry = RuntimeOpsLifecycleRegistry::new();
4380
4381 let spec = background_spec("explicit-no-source");
4382 let operation_id = spec.id.clone();
4383 registry.register_operation(spec).unwrap();
4384 registry.provisioning_succeeded(&operation_id).unwrap();
4385
4386 let cursor_state = meerkat_core::EpochCursorState::new();
4387 let snapshot = registry
4388 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4389 .unwrap();
4390 let value = serde_json::to_value(&snapshot).unwrap();
4391 let operations = value
4392 .get("authority_state")
4393 .and_then(|state| state.get("operations"))
4394 .and_then(serde_json::Value::as_object)
4395 .expect("serialized authority operations");
4396 let persisted_state = operations
4397 .values()
4398 .next()
4399 .and_then(serde_json::Value::as_object)
4400 .expect("serialized operation state");
4401
4402 assert!(
4403 persisted_state
4404 .get("operation_source")
4405 .is_some_and(serde_json::Value::is_null),
4406 "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4407 );
4408
4409 let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4410 assert_eq!(
4411 recovered_snapshot
4412 .authority_state
4413 .operations
4414 .get(&operation_id)
4415 .expect("round-tripped operation")
4416 .operation_source,
4417 None
4418 );
4419 }
4420
4421 #[test]
4422 fn persisted_authority_state_rejects_missing_operation_source_fact() {
4423 let registry = RuntimeOpsLifecycleRegistry::new();
4424
4425 let spec = background_spec("missing-source-fact");
4426 registry.register_operation(spec).unwrap();
4427
4428 let cursor_state = meerkat_core::EpochCursorState::new();
4429 let snapshot = registry
4430 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4431 .unwrap();
4432 let mut value = serde_json::to_value(&snapshot).unwrap();
4433 let operations = value
4434 .get_mut("authority_state")
4435 .and_then(|state| state.get_mut("operations"))
4436 .and_then(serde_json::Value::as_object_mut)
4437 .expect("serialized authority operations");
4438 let operation_state = operations
4439 .values_mut()
4440 .next()
4441 .and_then(serde_json::Value::as_object_mut)
4442 .expect("serialized operation state");
4443 assert!(operation_state.remove("operation_source").is_some());
4444
4445 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4446 .expect_err("missing generated source fact must fail recovery snapshot decoding");
4447 assert!(
4448 err.to_string().contains("operation_source"),
4449 "unexpected decode error: {err}"
4450 );
4451 }
4452
4453 #[test]
4454 fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4455 let registry = RuntimeOpsLifecycleRegistry::new();
4456
4457 let spec = background_spec("missing-feed-authority");
4458 let operation_id = spec.id.clone();
4459 registry.register_operation(spec).unwrap();
4460 registry.provisioning_succeeded(&operation_id).unwrap();
4461 registry
4462 .complete_operation(
4463 &operation_id,
4464 OperationResult {
4465 id: operation_id.clone(),
4466 content: "done".into(),
4467 is_error: false,
4468 duration_ms: 1,
4469 tokens_used: 0,
4470 },
4471 )
4472 .unwrap();
4473
4474 let cursor_state = meerkat_core::EpochCursorState::new();
4475 let snapshot = registry
4476 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4477 .unwrap();
4478 let mut value = serde_json::to_value(&snapshot).unwrap();
4479 let authority_state = value
4480 .get_mut("authority_state")
4481 .and_then(serde_json::Value::as_object_mut)
4482 .expect("serialized authority state");
4483 assert!(authority_state.remove("completion_feed_entries").is_some());
4484
4485 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4486 .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4487 assert!(
4488 err.to_string().contains("completion_feed_entries"),
4489 "unexpected decode error: {err}"
4490 );
4491 }
4492
4493 #[test]
4494 fn public_child_session_projection_uses_authority_operation_source() {
4495 let registry = RuntimeOpsLifecycleRegistry::new();
4496 let authority_child_session_id = SessionId::new();
4497 let stale_shell_child_session_id = SessionId::new();
4498 let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4499 let spec = OperationSpec {
4500 id: OperationId::new(),
4501 kind: OperationKind::MobMemberChild,
4502 owner_session_id: SessionId::new(),
4503 display_name: "child-projection".into(),
4504 source_label: "test".into(),
4505 operation_source: Some(operation_source),
4506 child_session_id: Some(stale_shell_child_session_id),
4507 expect_peer_channel: true,
4508 };
4509 let operation_id = spec.id.clone();
4510
4511 registry.register_operation(spec).unwrap();
4512
4513 assert_eq!(
4514 registry
4515 .snapshot(&operation_id)
4516 .unwrap()
4517 .unwrap()
4518 .child_session_id,
4519 Some(authority_child_session_id.clone())
4520 );
4521
4522 let cursor_state = meerkat_core::EpochCursorState::new();
4523 let snapshot = registry
4524 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4525 .unwrap();
4526 assert_eq!(
4527 snapshot
4528 .operation_specs
4529 .get(&operation_id)
4530 .expect("persisted spec")
4531 .child_session_id,
4532 Some(authority_child_session_id)
4533 );
4534 }
4535
4536 #[test]
4537 fn generated_terminal_payload_projection_fails_closed() {
4538 let registry = RuntimeOpsLifecycleRegistry::new();
4539
4540 let spec = background_spec("terminal-payload-drift");
4541 let operation_id = spec.id.clone();
4542 registry.register_operation(spec).unwrap();
4543 registry.provisioning_succeeded(&operation_id).unwrap();
4544 registry
4545 .complete_operation(
4546 &operation_id,
4547 OperationResult {
4548 id: operation_id.clone(),
4549 content: "done".into(),
4550 is_error: false,
4551 duration_ms: 1,
4552 tokens_used: 0,
4553 },
4554 )
4555 .unwrap();
4556
4557 {
4558 let mut state = registry.write_state().unwrap();
4559 let mut machine_state = state.dsl.0.state().clone();
4560 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4561 machine_state
4562 .op_terminal_payload
4563 .insert(operation_id_key, OperationTerminalOutcome::Retired);
4564 state.dsl = DslAuthority(Box::new(
4565 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4566 ));
4567 }
4568
4569 let err = match registry.register_watcher(&operation_id) {
4570 Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4571 Err(err) => err,
4572 };
4573 assert!(
4574 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4575 "unexpected watcher error: {err:?}"
4576 );
4577 let err = registry
4578 .snapshot(&operation_id)
4579 .expect_err("invalid generated terminal payload must reject public snapshot");
4580 assert!(
4581 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
4582 "unexpected public snapshot error: {err:?}"
4583 );
4584
4585 let cursor_state = meerkat_core::EpochCursorState::new();
4586 let err = match registry
4587 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4588 {
4589 Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
4590 Err(err) => err,
4591 };
4592 assert!(
4593 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4594 "unexpected snapshot error: {err:?}"
4595 );
4596
4597 let err = match registry.collect_completed() {
4598 Ok(_) => panic!("invalid generated terminal payload must reject collection"),
4599 Err(err) => err,
4600 };
4601 assert!(
4602 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4603 "unexpected collection error: {err:?}"
4604 );
4605 }
4606
4607 #[test]
4608 fn generated_terminal_payload_missing_projection_fails_closed() {
4609 let registry = RuntimeOpsLifecycleRegistry::new();
4610
4611 let spec = background_spec("terminal-payload-missing");
4612 let operation_id = spec.id.clone();
4613 registry.register_operation(spec).unwrap();
4614 registry.provisioning_succeeded(&operation_id).unwrap();
4615 registry
4616 .fail_operation(&operation_id, "boom".into())
4617 .unwrap();
4618
4619 {
4620 let mut state = registry.write_state().unwrap();
4621 let mut machine_state = state.dsl.0.state().clone();
4622 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4623 machine_state.op_terminal_payload.remove(&operation_id_key);
4624 state.dsl = DslAuthority(Box::new(
4625 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4626 ));
4627 }
4628
4629 let err = match registry.register_watcher(&operation_id) {
4630 Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
4631 Err(err) => err,
4632 };
4633 assert!(
4634 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4635 "unexpected watcher error: {err:?}"
4636 );
4637 let err = registry
4638 .snapshot(&operation_id)
4639 .expect_err("missing generated terminal payload must reject public snapshot");
4640 assert!(
4641 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4642 "unexpected public snapshot error: {err:?}"
4643 );
4644 }
4645
4646 #[test]
4647 fn generated_terminal_status_without_outcome_fails_closed() {
4648 let registry = RuntimeOpsLifecycleRegistry::new();
4649
4650 let spec = background_spec("terminal-outcome-missing");
4651 let operation_id = spec.id.clone();
4652 registry.register_operation(spec).unwrap();
4653 registry.provisioning_succeeded(&operation_id).unwrap();
4654 registry
4655 .fail_operation(&operation_id, "boom".into())
4656 .unwrap();
4657
4658 {
4659 let mut state = registry.write_state().unwrap();
4660 let mut machine_state = state.dsl.0.state().clone();
4661 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4662 machine_state.op_terminal_outcomes.remove(&operation_id_key);
4663 state.dsl = DslAuthority(Box::new(
4664 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4665 ));
4666 }
4667
4668 let err = registry
4669 .snapshot(&operation_id)
4670 .expect_err("terminal status without outcome must reject public snapshot");
4671 assert!(
4672 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4673 "unexpected public snapshot error: {err:?}"
4674 );
4675
4676 let err = match registry.collect_completed() {
4677 Ok(_) => panic!("terminal status without outcome must reject collection"),
4678 Err(err) => err,
4679 };
4680 assert!(
4681 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4682 "unexpected collection error: {err:?}"
4683 );
4684 }
4685
4686 #[test]
4687 fn generated_operation_source_projection_fails_closed() {
4688 let registry = RuntimeOpsLifecycleRegistry::new();
4689 let child_session_id = SessionId::new();
4690 let operation_source = OperationSource::session_child(child_session_id.clone());
4691 let spec = OperationSpec {
4692 id: OperationId::new(),
4693 kind: OperationKind::MobMemberChild,
4694 owner_session_id: SessionId::new(),
4695 display_name: "source-authority-drift".into(),
4696 source_label: "test".into(),
4697 operation_source: Some(operation_source),
4698 child_session_id: Some(child_session_id),
4699 expect_peer_channel: true,
4700 };
4701 let operation_id = spec.id.clone();
4702
4703 registry.register_operation(spec).unwrap();
4704
4705 {
4706 let mut state = registry.write_state().unwrap();
4707 let mut machine_state = state.dsl.0.state().clone();
4708 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4709 machine_state
4710 .op_sources
4711 .get_mut(&operation_id_key)
4712 .expect("generated operation source")
4713 .session_id = None;
4714 state.dsl = DslAuthority(Box::new(
4715 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4716 ));
4717 }
4718
4719 let err = registry
4720 .snapshot(&operation_id)
4721 .expect_err("invalid generated operation source must reject public snapshot");
4722 assert!(
4723 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4724 "unexpected public snapshot error: {err:?}"
4725 );
4726 let err = registry
4727 .list_operations()
4728 .expect_err("invalid generated operation source must reject public operation list");
4729 assert!(
4730 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4731 "unexpected operation list error: {err:?}"
4732 );
4733
4734 let cursor_state = meerkat_core::EpochCursorState::new();
4735 let err = match registry
4736 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4737 {
4738 Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
4739 Err(err) => err,
4740 };
4741 assert!(
4742 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4743 "unexpected snapshot error: {err:?}"
4744 );
4745 }
4746
4747 #[test]
4748 fn generated_operation_id_projection_fails_closed() {
4749 let registry = RuntimeOpsLifecycleRegistry::new();
4750
4751 {
4752 let mut state = registry.write_state().unwrap();
4753 let mut machine_state = state.dsl.0.state().clone();
4754 machine_state.op_statuses.insert(
4755 "not-json-operation-id".into(),
4756 mm_dsl::OperationStatus::Running,
4757 );
4758 state.dsl = DslAuthority(Box::new(
4759 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4760 ));
4761 }
4762
4763 let err = registry
4764 .list_operations()
4765 .expect_err("invalid generated operation id must reject public operation list");
4766 assert!(
4767 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4768 "unexpected operation list error: {err:?}"
4769 );
4770
4771 let cursor_state = meerkat_core::EpochCursorState::new();
4772 let err = match registry
4773 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4774 {
4775 Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
4776 Err(err) => err,
4777 };
4778 assert!(
4779 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4780 "unexpected persistence snapshot error: {err:?}"
4781 );
4782 }
4783
4784 #[test]
4785 fn generated_missing_kind_projection_fails_closed() {
4786 let registry = RuntimeOpsLifecycleRegistry::new();
4787 let spec = background_spec("missing-kind");
4788 let operation_id = spec.id.clone();
4789 registry.register_operation(spec).unwrap();
4790
4791 {
4792 let mut state = registry.write_state().unwrap();
4793 let mut machine_state = state.dsl.0.state().clone();
4794 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4795 machine_state.op_kinds.remove(&operation_id_key);
4796 state.dsl = DslAuthority(Box::new(
4797 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4798 ));
4799 }
4800
4801 let err = registry
4802 .snapshot(&operation_id)
4803 .expect_err("missing generated kind must reject public snapshot");
4804 assert!(
4805 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4806 "unexpected public snapshot error: {err:?}"
4807 );
4808 let err = registry
4809 .list_operations()
4810 .expect_err("missing generated kind must reject public list");
4811 assert!(
4812 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4813 "unexpected public list error: {err:?}"
4814 );
4815
4816 let cursor_state = meerkat_core::EpochCursorState::new();
4817 let err = registry
4818 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4819 .expect_err("missing generated kind must reject persistence snapshot");
4820 assert!(
4821 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4822 "unexpected persistence snapshot error: {err:?}"
4823 );
4824 }
4825
4826 #[test]
4827 fn generated_missing_status_projection_fails_closed() {
4828 let registry = RuntimeOpsLifecycleRegistry::new();
4829 let spec = background_spec("missing-status");
4830 let operation_id = spec.id.clone();
4831 registry.register_operation(spec).unwrap();
4832
4833 {
4834 let mut state = registry.write_state().unwrap();
4835 let mut machine_state = state.dsl.0.state().clone();
4836 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4837 machine_state.op_statuses.remove(&operation_id_key);
4838 state.dsl = DslAuthority(Box::new(
4839 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4840 ));
4841 }
4842
4843 let err = registry
4844 .snapshot(&operation_id)
4845 .expect_err("missing generated status must reject public snapshot");
4846 assert!(
4847 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4848 "unexpected public snapshot error: {err:?}"
4849 );
4850 let err = registry
4851 .list_operations()
4852 .expect_err("missing generated status must reject public list");
4853 assert!(
4854 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4855 "unexpected public list error: {err:?}"
4856 );
4857 let err = registry
4858 .classify_operation_public_result(&operation_id)
4859 .expect_err("missing generated status must reject public-result classification");
4860 assert!(
4861 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4862 "unexpected public-result error: {err:?}"
4863 );
4864 }
4865
4866 #[test]
4867 fn generated_retiring_public_result_remains_running_until_terminal() {
4868 let registry = RuntimeOpsLifecycleRegistry::new();
4869 let spec = background_spec("retiring-public-result");
4870 let operation_id = spec.id.clone();
4871 registry.register_operation(spec).unwrap();
4872 registry.provisioning_succeeded(&operation_id).unwrap();
4873 registry.request_retire(&operation_id).unwrap();
4874
4875 let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
4876 assert_eq!(snapshot.status, OperationStatus::Retiring);
4877 assert!(snapshot.terminal_outcome.is_none());
4878 assert!(!snapshot.terminal);
4879 assert_eq!(
4880 snapshot.public_result_class,
4881 OperationPublicResultClass::Running
4882 );
4883 assert_eq!(
4884 registry
4885 .classify_operation_public_result(&operation_id)
4886 .unwrap(),
4887 OperationPublicResultClass::Running
4888 );
4889 }
4890
4891 #[test]
4892 fn generated_missing_peer_ready_projection_fails_closed() {
4893 let registry = RuntimeOpsLifecycleRegistry::new();
4894 let spec = background_spec("missing-peer-ready");
4895 let operation_id = spec.id.clone();
4896 registry.register_operation(spec).unwrap();
4897
4898 {
4899 let mut state = registry.write_state().unwrap();
4900 let mut machine_state = state.dsl.0.state().clone();
4901 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4902 machine_state.op_peer_ready.remove(&operation_id_key);
4903 state.dsl = DslAuthority(Box::new(
4904 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4905 ));
4906 }
4907
4908 let err = registry
4909 .snapshot(&operation_id)
4910 .expect_err("missing generated peer-ready fact must reject public snapshot");
4911 assert!(
4912 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4913 "unexpected public snapshot error: {err:?}"
4914 );
4915
4916 let cursor_state = meerkat_core::EpochCursorState::new();
4917 let err = registry
4918 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4919 .expect_err("missing generated peer-ready fact must reject persistence snapshot");
4920 assert!(
4921 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4922 "unexpected persistence snapshot error: {err:?}"
4923 );
4924 }
4925
4926 #[test]
4927 fn generated_missing_progress_count_projection_fails_closed() {
4928 let registry = RuntimeOpsLifecycleRegistry::new();
4929 let spec = background_spec("missing-progress-count");
4930 let operation_id = spec.id.clone();
4931 registry.register_operation(spec).unwrap();
4932
4933 {
4934 let mut state = registry.write_state().unwrap();
4935 let mut machine_state = state.dsl.0.state().clone();
4936 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4937 machine_state.op_progress_counts.remove(&operation_id_key);
4938 state.dsl = DslAuthority(Box::new(
4939 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4940 ));
4941 }
4942
4943 let err = registry
4944 .snapshot(&operation_id)
4945 .expect_err("missing generated progress count must reject public snapshot");
4946 assert!(
4947 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4948 "unexpected public snapshot error: {err:?}"
4949 );
4950
4951 let cursor_state = meerkat_core::EpochCursorState::new();
4952 let err = registry
4953 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4954 .expect_err("missing generated progress count must reject persistence snapshot");
4955 assert!(
4956 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4957 "unexpected persistence snapshot error: {err:?}"
4958 );
4959 }
4960
4961 #[test]
4962 fn generated_terminal_sequence_missing_persistence_fails_closed() {
4963 let registry = RuntimeOpsLifecycleRegistry::new();
4964 let spec = background_spec("terminal-sequence-missing");
4965 let operation_id = spec.id.clone();
4966 registry.register_operation(spec).unwrap();
4967 registry.provisioning_succeeded(&operation_id).unwrap();
4968 registry
4969 .complete_operation(
4970 &operation_id,
4971 OperationResult {
4972 id: operation_id.clone(),
4973 content: "done".into(),
4974 is_error: false,
4975 duration_ms: 1,
4976 tokens_used: 0,
4977 },
4978 )
4979 .unwrap();
4980
4981 {
4982 let mut state = registry.write_state().unwrap();
4983 let mut machine_state = state.dsl.0.state().clone();
4984 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4985 machine_state.op_completion_seq.remove(&operation_id_key);
4986 state.dsl = DslAuthority(Box::new(
4987 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4988 ));
4989 }
4990
4991 let cursor_state = meerkat_core::EpochCursorState::new();
4992 let err = registry
4993 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4994 .expect_err("missing generated terminal sequence must reject persistence snapshot");
4995 assert!(
4996 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
4997 "unexpected persistence snapshot error: {err:?}"
4998 );
4999 }
5000
5001 #[test]
5002 fn generated_record_without_shell_projection_fails_closed() {
5003 let registry = RuntimeOpsLifecycleRegistry::new();
5004 let spec = background_spec("missing-shell-record");
5005 let operation_id = spec.id.clone();
5006 registry.register_operation(spec).unwrap();
5007
5008 {
5009 let mut state = registry.write_state().unwrap();
5010 state.records.remove(&operation_id);
5011 }
5012
5013 let err = registry
5014 .snapshot(&operation_id)
5015 .expect_err("generated operation without shell record must reject public snapshot");
5016 assert!(
5017 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5018 "unexpected public snapshot error: {err:?}"
5019 );
5020 let err = registry
5021 .list_operations()
5022 .expect_err("generated operation without shell record must reject public list");
5023 assert!(
5024 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5025 "unexpected public list error: {err:?}"
5026 );
5027 }
5028
5029 #[test]
5030 fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
5031 let registry = RuntimeOpsLifecycleRegistry::new();
5032 let child_session_id = SessionId::new();
5033 let operation_source = OperationSource::session_child(child_session_id.clone());
5034 let spec = OperationSpec {
5035 id: OperationId::new(),
5036 kind: OperationKind::MobMemberChild,
5037 owner_session_id: SessionId::new(),
5038 display_name: "child-drift".into(),
5039 source_label: "test".into(),
5040 operation_source: Some(operation_source),
5041 child_session_id: Some(child_session_id.clone()),
5042 expect_peer_channel: true,
5043 };
5044 let operation_id = spec.id.clone();
5045
5046 registry.register_operation(spec).unwrap();
5047 registry.provisioning_succeeded(&operation_id).unwrap();
5048 registry.mark_retired(&operation_id).unwrap();
5049
5050 let cursor_state = meerkat_core::EpochCursorState::new();
5051 let mut snapshot = registry
5052 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5053 .unwrap();
5054 snapshot
5055 .operation_specs
5056 .get_mut(&operation_id)
5057 .expect("persisted spec")
5058 .child_session_id = Some(SessionId::new());
5059
5060 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5061 assert_eq!(
5062 recovered
5063 .snapshot(&operation_id)
5064 .unwrap()
5065 .unwrap()
5066 .child_session_id,
5067 Some(child_session_id)
5068 );
5069 }
5070
5071 #[test]
5072 fn completion_wake_class_is_generated_by_operation_kind() {
5073 let registry = RuntimeOpsLifecycleRegistry::new();
5074 let operation_id = OperationId::new();
5075
5076 assert_eq!(
5077 registry
5078 .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
5079 .unwrap(),
5080 OperationCompletionWakeClass::Wake
5081 );
5082 assert_eq!(
5083 registry
5084 .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
5085 .unwrap(),
5086 OperationCompletionWakeClass::Ignore
5087 );
5088 assert_eq!(
5089 registry
5090 .classify_operation_completion_wake(
5091 &operation_id,
5092 OperationKind::BackgroundToolCapacitySlot,
5093 )
5094 .unwrap(),
5095 OperationCompletionWakeClass::Ignore
5096 );
5097 }
5098
5099 #[test]
5100 fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
5101 let registry = RuntimeOpsLifecycleRegistry::new();
5102
5103 let running_spec = background_spec("running");
5104 let running_id = running_spec.id.clone();
5105 registry.register_operation(running_spec.clone()).unwrap();
5106 registry.provisioning_succeeded(&running_id).unwrap();
5107
5108 let cursor_state = meerkat_core::EpochCursorState::new();
5109 let mut snapshot = registry
5110 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5111 .unwrap();
5112 snapshot.completion_entries.push(CompletionEntry {
5113 seq: 1,
5114 operation_id: running_id.clone(),
5115 kind: running_spec.kind,
5116 display_name: running_spec.display_name,
5117 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5118 id: running_id,
5119 content: "phantom".into(),
5120 is_error: false,
5121 duration_ms: 1,
5122 tokens_used: 0,
5123 }),
5124 completed_at_ms: None,
5125 });
5126
5127 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5128 Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5129 Err(err) => err,
5130 };
5131 assert!(
5132 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5133 "unexpected recovery error: {err:?}"
5134 );
5135 }
5136
5137 #[test]
5138 fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5139 let registry = RuntimeOpsLifecycleRegistry::new();
5140 let phantom_id = OperationId::new();
5141 {
5142 let state = registry.read_state().unwrap();
5143 state.feed_buffer.push(CompletionEntry {
5144 seq: 1,
5145 operation_id: phantom_id.clone(),
5146 kind: OperationKind::BackgroundToolOp,
5147 display_name: "phantom".into(),
5148 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5149 id: phantom_id,
5150 content: "phantom".into(),
5151 is_error: false,
5152 duration_ms: 1,
5153 tokens_used: 0,
5154 }),
5155 completed_at_ms: None,
5156 });
5157 }
5158
5159 let cursor_state = meerkat_core::EpochCursorState::new();
5160 let err = match registry
5161 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5162 {
5163 Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5164 Err(err) => err,
5165 };
5166 assert!(
5167 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5168 "unexpected capture error: {err:?}"
5169 );
5170 }
5171
5172 #[test]
5173 fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5174 let registry = RuntimeOpsLifecycleRegistry::new();
5175
5176 let spec = background_spec("terminal");
5177 let operation_id = spec.id.clone();
5178 registry.register_operation(spec).unwrap();
5179 registry.provisioning_succeeded(&operation_id).unwrap();
5180 registry
5181 .complete_operation(
5182 &operation_id,
5183 OperationResult {
5184 id: operation_id.clone(),
5185 content: "done".into(),
5186 is_error: false,
5187 duration_ms: 1,
5188 tokens_used: 0,
5189 },
5190 )
5191 .unwrap();
5192
5193 let cursor_state = meerkat_core::EpochCursorState::new();
5194 let mut snapshot = registry
5195 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5196 .unwrap();
5197 let phantom_id = OperationId::new();
5198 let phantom_result = OperationResult {
5199 id: phantom_id.clone(),
5200 content: "phantom".into(),
5201 is_error: false,
5202 duration_ms: 1,
5203 tokens_used: 0,
5204 };
5205 let phantom_entry = CompletionFeedCanonicalState {
5206 seq: snapshot.authority_state.next_completion_seq + 1,
5207 kind: OperationKind::BackgroundToolOp,
5208 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5209 };
5210 snapshot
5211 .authority_state
5212 .completion_feed_entries
5213 .insert(phantom_id.clone(), phantom_entry);
5214 snapshot.completion_entries.push(CompletionEntry {
5215 seq: snapshot.authority_state.next_completion_seq + 1,
5216 operation_id: phantom_id.clone(),
5217 kind: OperationKind::BackgroundToolOp,
5218 display_name: "phantom".into(),
5219 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5220 completed_at_ms: None,
5221 });
5222
5223 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5224 Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5225 Err(err) => err,
5226 };
5227 assert!(
5228 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5229 "unexpected recovery error: {err:?}"
5230 );
5231 }
5232
5233 #[test]
5234 fn recovered_completed_order_uses_generated_completion_sequences() {
5235 let registry = RuntimeOpsLifecycleRegistry::new();
5236
5237 let spec_a = background_spec("a");
5238 let id_a = spec_a.id.clone();
5239 registry.register_operation(spec_a).unwrap();
5240 registry.provisioning_succeeded(&id_a).unwrap();
5241 registry
5242 .complete_operation(
5243 &id_a,
5244 OperationResult {
5245 id: id_a.clone(),
5246 content: "a".into(),
5247 is_error: false,
5248 duration_ms: 1,
5249 tokens_used: 0,
5250 },
5251 )
5252 .unwrap();
5253
5254 let spec_b = background_spec("b");
5255 let id_b = spec_b.id.clone();
5256 registry.register_operation(spec_b).unwrap();
5257 registry.provisioning_succeeded(&id_b).unwrap();
5258 registry
5259 .complete_operation(
5260 &id_b,
5261 OperationResult {
5262 id: id_b.clone(),
5263 content: "b".into(),
5264 is_error: false,
5265 duration_ms: 1,
5266 tokens_used: 0,
5267 },
5268 )
5269 .unwrap();
5270
5271 let cursor_state = meerkat_core::EpochCursorState::new();
5272 let mut snapshot = registry
5273 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5274 .unwrap();
5275 snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5276
5277 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5278 let collected = recovered.collect_completed().unwrap();
5279
5280 assert_eq!(collected[0].0, id_a);
5281 assert_eq!(collected[1].0, id_b);
5282 }
5283
5284 #[test]
5285 fn max_concurrent_enforcement() {
5286 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5287 max_completed: DEFAULT_MAX_COMPLETED,
5288 max_concurrent: Some(2),
5289 });
5290
5291 let spec_a = background_spec("a");
5292 let id_a = spec_a.id.clone();
5293 registry.register_operation(spec_a).unwrap();
5294
5295 let spec_b = background_spec("b");
5296 registry.register_operation(spec_b).unwrap();
5297
5298 let spec_c = background_spec("c");
5299 let result = registry.register_operation(spec_c);
5300 assert!(matches!(
5301 result,
5302 Err(OpsLifecycleError::MaxConcurrentExceeded {
5303 limit: 2,
5304 active: 2,
5305 })
5306 ));
5307
5308 registry.provisioning_succeeded(&id_a).unwrap();
5309 registry
5310 .complete_operation(
5311 &id_a,
5312 OperationResult {
5313 id: id_a.clone(),
5314 content: "done".into(),
5315 is_error: false,
5316 duration_ms: 1,
5317 tokens_used: 0,
5318 },
5319 )
5320 .unwrap();
5321
5322 let spec_d = background_spec("d");
5323 assert!(registry.register_operation(spec_d).is_ok());
5324 }
5325
5326 #[test]
5327 fn snapshot_includes_timestamps() {
5328 let registry = RuntimeOpsLifecycleRegistry::new();
5329 let spec = background_spec("timed");
5330 let op_id = spec.id.clone();
5331 registry.register_operation(spec).unwrap();
5332
5333 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5334 assert!(snap1.created_at_ms > 0);
5335 assert!(snap1.started_at_ms.is_none());
5336 assert!(snap1.completed_at_ms.is_none());
5337 assert!(snap1.elapsed_ms.is_none());
5338
5339 registry.provisioning_succeeded(&op_id).unwrap();
5340 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5341 assert!(snap2.started_at_ms.is_some());
5342 assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5343
5344 registry
5345 .complete_operation(
5346 &op_id,
5347 OperationResult {
5348 id: op_id.clone(),
5349 content: "done".into(),
5350 is_error: false,
5351 duration_ms: 1,
5352 tokens_used: 0,
5353 },
5354 )
5355 .unwrap();
5356 let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5357 assert!(snap3.completed_at_ms.is_some());
5358 assert!(snap3.elapsed_ms.is_some());
5359 assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5360 }
5361
5362 #[test]
5363 fn snapshot_includes_peer_handle() {
5364 let registry = RuntimeOpsLifecycleRegistry::new();
5365 let child_session_id = SessionId::new();
5366 let spec = OperationSpec {
5367 id: OperationId::new(),
5368 kind: OperationKind::MobMemberChild,
5369 owner_session_id: SessionId::new(),
5370 display_name: "peer-test".into(),
5371 source_label: "test".into(),
5372 operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5373 child_session_id: Some(child_session_id),
5374 expect_peer_channel: true,
5375 };
5376 let op_id = spec.id.clone();
5377 registry.register_operation(spec).unwrap();
5378 registry.provisioning_succeeded(&op_id).unwrap();
5379
5380 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5381 assert!(snap1.peer_handle.is_none());
5382
5383 let handle = OperationPeerHandle {
5384 peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5385 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5386 "member-x",
5387 PeerId::new(),
5388 "inproc://x",
5389 )
5390 .unwrap(),
5391 };
5392 registry.peer_ready(&op_id, handle).unwrap();
5393
5394 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5395 assert_eq!(
5396 snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5397 "member-x"
5398 );
5399 }
5400}