1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24 CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31 CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32 OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33 OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34 OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35 OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36 WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43fn deserialize_required_operation_source<'de, D>(
54 deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57 D: serde::Deserializer<'de>,
58{
59 <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65 status: OperationStatus,
66 kind: OperationKind,
67 #[serde(deserialize_with = "deserialize_required_operation_source")]
68 operation_source: Option<OperationSource>,
69 peer_ready: bool,
70 progress_count: u32,
71 watcher_count: u32,
72 terminal_outcome: Option<OperationTerminalOutcome>,
73 #[serde(default, skip_serializing_if = "Option::is_none")]
74 completion_sequence: Option<CompletionSeq>,
75 terminal_buffered: bool,
76}
77
78#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81 seq: CompletionSeq,
82 kind: OperationKind,
83 terminal_outcome: OperationTerminalOutcome,
84}
85
86#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89 operations: HashMap<OperationId, OperationCanonicalState>,
90 completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91 completed_order: VecDeque<OperationId>,
92 max_completed: usize,
93 max_concurrent: Option<usize>,
94 active_count: usize,
95 wait_request_id: Option<WaitRequestId>,
96 wait_operation_ids: Vec<OperationId>,
97 next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101 pub fn max_completed(&self) -> usize {
103 self.max_completed
104 }
105
106 pub fn max_concurrent(&self) -> Option<usize> {
108 self.max_concurrent
109 }
110
111 pub fn operation_count(&self) -> usize {
113 self.operations.len()
114 }
115
116 pub fn completion_feed_count(&self) -> usize {
118 self.completion_feed_entries.len()
119 }
120}
121
122#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134 pub epoch_id: meerkat_core::RuntimeEpochId,
136 pub authority_state: RegistryCanonicalState,
138 pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140 pub completion_entries: Vec<CompletionEntry>,
143 pub cursors: meerkat_core::EpochCursorSnapshot,
145}
146
147#[derive(Debug)]
148pub struct OpsLifecyclePersistenceRequest {
149 snapshot: PersistedOpsSnapshot,
150 result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
151}
152
153impl OpsLifecyclePersistenceRequest {
154 pub fn snapshot(&self) -> &PersistedOpsSnapshot {
155 &self.snapshot
156 }
157
158 pub fn complete(self, result: Result<(), OpsLifecycleError>) {
159 let _ = self.result_tx.send(result);
160 }
161}
162
163#[derive(Debug)]
172struct FeedBufferInner {
173 entries: VecDeque<CompletionEntry>,
174 watermark: CompletionSeq,
175 max_retained: usize,
176}
177
178#[derive(Debug)]
183struct FeedBuffer {
184 inner: RwLock<FeedBufferInner>,
185 watermark_atomic: AtomicU64,
187 notify: tokio::sync::Notify,
189}
190
191impl FeedBuffer {
192 fn new(max_retained: usize) -> Self {
193 Self {
194 inner: RwLock::new(FeedBufferInner {
195 entries: VecDeque::new(),
196 watermark: 0,
197 max_retained,
198 }),
199 watermark_atomic: AtomicU64::new(0),
200 notify: tokio::sync::Notify::new(),
201 }
202 }
203
204 fn push(&self, entry: CompletionEntry) {
205 let mut inner = self
206 .inner
207 .write()
208 .unwrap_or_else(std::sync::PoisonError::into_inner);
209 let seq = entry.seq;
210 inner.entries.push_back(entry);
211 inner.watermark = seq;
212
213 while inner.entries.len() > inner.max_retained {
215 inner.entries.pop_front();
216 }
217
218 drop(inner);
219
220 self.watermark_atomic.store(seq, Ordering::Release);
221 self.notify.notify_waiters();
222 }
223}
224
225#[derive(Debug, Clone)]
230pub struct RuntimeCompletionFeed {
231 buffer: Arc<FeedBuffer>,
232}
233
234impl CompletionFeed for RuntimeCompletionFeed {
235 fn watermark(&self) -> CompletionSeq {
236 self.buffer.watermark_atomic.load(Ordering::Acquire)
237 }
238
239 fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
240 let inner = self
241 .buffer
242 .inner
243 .read()
244 .unwrap_or_else(std::sync::PoisonError::into_inner);
245 let entries: Vec<CompletionEntry> = inner
246 .entries
247 .iter()
248 .filter(|e| e.seq > after_seq)
249 .cloned()
250 .collect();
251 let watermark = inner.watermark;
252 CompletionBatch { entries, watermark }
253 }
254
255 fn wait_for_advance(
256 &self,
257 after_seq: CompletionSeq,
258 ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
259 Box::pin(async move {
260 loop {
261 let notified = self.buffer.notify.notified();
266 let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
267 if current > after_seq {
268 return current;
269 }
270 notified.await;
271 }
272 })
273 }
274}
275
276#[derive(Debug)]
281struct OperationCompletionNotifier {
282 tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
283}
284
285impl OperationCompletionNotifier {
286 fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
287 Self { tx }
288 }
289
290 fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
291 let _ = self.tx.send(outcome.clone());
292 }
293}
294
295fn operation_completion_watch_from_receiver(
296 rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
297) -> OperationCompletionWatch {
298 Box::pin(async move {
299 rx.await
300 .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
301 })
302}
303
304fn resolved_operation_completion_watch(
305 outcome: OperationTerminalOutcome,
306) -> OperationCompletionWatch {
307 Box::pin(async move { Ok(outcome) })
308}
309
310#[derive(Debug)]
314struct ShellRecord {
315 spec: OperationSpec,
316 peer_handle: Option<OperationPeerHandle>,
317 watchers: Vec<OperationCompletionNotifier>,
321 created_at: Instant,
323 started_at: Option<Instant>,
324 completed_at: Option<Instant>,
325 created_at_wall: SystemTime,
327}
328
329#[derive(Debug)]
330struct PendingWaitState {
331 wait_request_id: WaitRequestId,
332 sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
333}
334
335enum WaitAllAuthorityPlan {
336 AlreadySatisfied(WaitAllSatisfied),
337 ActivateBarrier,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq)]
341enum RecoveredOperationRecordDisposition {
342 Retain,
343 Discard,
344}
345
346impl ShellRecord {
347 fn new(spec: OperationSpec) -> Self {
348 Self {
349 spec,
350 peer_handle: None,
351 watchers: Vec::new(),
352 created_at: Instant::now(),
353 started_at: None,
354 completed_at: None,
355 created_at_wall: SystemTime::now(),
356 }
357 }
358
359 fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
360 wall_anchor
361 .duration_since(UNIX_EPOCH)
362 .map(|d| d.as_millis() as u64)
363 .unwrap_or(0)
364 }
365
366 fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
367 let offset = instant.saturating_duration_since(self.created_at);
370 let wall = self.created_at_wall + offset;
371 Self::epoch_millis(&wall)
372 }
373
374 fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
376 for watcher in std::mem::take(&mut self.watchers) {
377 watcher.notify_after_generated_terminal(outcome);
378 }
379 }
380
381 fn mark_completed(&mut self) {
383 self.completed_at = Some(Instant::now());
384 }
385}
386
387#[derive(Debug)]
392struct ShellState {
393 dsl: DslAuthority,
395 records: HashMap<OperationId, ShellRecord>,
397 pending_wait: Option<PendingWaitState>,
399 completed_order: VecDeque<OperationId>,
401 max_completed: usize,
403 max_concurrent: Option<usize>,
405 wait_request_id: Option<WaitRequestId>,
411 feed_buffer: Arc<FeedBuffer>,
413 persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
415 persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
417 persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
419}
420
421struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
427
428impl std::fmt::Debug for DslAuthority {
429 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
430 f.debug_struct("DslAuthority")
431 .field("state", self.0.state())
432 .finish()
433 }
434}
435
436fn new_ops_dsl_authority() -> DslAuthority {
440 DslAuthority(Box::new(
441 crate::meerkat_machine::dsl_authority::new_initialized_authority(
442 "ops lifecycle DSL Initialize must be accepted",
443 ),
444 ))
445}
446
447impl ShellState {
448 fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
449 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
450 let dsl = new_ops_dsl_authority();
451 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
452 let feed_capacity = max_completed.saturating_mul(4).max(1024);
453 tracing::info!(
454 feed_capacity,
455 "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
456 );
457 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
458 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
459 Self {
460 dsl,
461 records: HashMap::new(),
462 pending_wait: None,
463 completed_order: VecDeque::new(),
464 max_completed,
465 max_concurrent,
466 wait_request_id: None,
467 feed_buffer,
472 persist_tx: None,
473 persist_epoch_id: None,
474 persist_cursor_state: None,
475 }
476 }
477
478 fn dsl_apply(
485 &mut self,
486 input: mm_dsl::MeerkatMachineInput,
487 context: &str,
488 ) -> Result<(), OpsLifecycleError> {
489 self.dsl_apply_raw(input).map_err(|err| {
490 OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
491 })
492 }
493
494 fn dsl_apply_raw(
501 &mut self,
502 input: mm_dsl::MeerkatMachineInput,
503 ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
504 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
505 }
506
507 fn dsl_apply_with_effects(
508 &mut self,
509 input: mm_dsl::MeerkatMachineInput,
510 context: &str,
511 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
512 let transition =
513 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
514 OpsLifecycleError::Internal(format!(
515 "DSL rejected ops transition ({context}): {err:?}"
516 ))
517 })?;
518 Ok(transition.into_effects())
519 }
520
521 fn checked_terminal_payload(
526 kind: mm_dsl::OperationTerminalOutcomeKind,
527 payload: &OperationTerminalOutcome,
528 authority: &str,
529 operation_id: &str,
530 ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
531 if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
532 return Err(OpsLifecycleError::Internal(format!(
533 "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
534 )));
535 }
536 Ok(payload.clone())
537 }
538
539 fn status(&self, id: &OperationId) -> Option<OperationStatus> {
541 let id_key = mm_dsl::OperationId::from_domain(id).0;
542 self.dsl
543 .0
544 .state()
545 .op_statuses
546 .get(&id_key)
547 .copied()
548 .map(OperationStatus::from)
549 }
550
551 fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
552 self.status(id).ok_or_else(|| {
553 OpsLifecycleError::Internal(format!(
554 "generated op lifecycle authority missing status for {id}"
555 ))
556 })
557 }
558
559 fn kind(&self, id: &OperationId) -> Option<OperationKind> {
561 let id_key = mm_dsl::OperationId::from_domain(id).0;
562 self.dsl
563 .0
564 .state()
565 .op_kinds
566 .get(&id_key)
567 .copied()
568 .map(OperationKind::from)
569 }
570
571 fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
572 self.kind(id).ok_or_else(|| {
573 OpsLifecycleError::Internal(format!(
574 "generated op lifecycle authority missing kind for {id}"
575 ))
576 })
577 }
578
579 fn operation_source(
580 &self,
581 id: &OperationId,
582 ) -> Result<Option<OperationSource>, OpsLifecycleError> {
583 let id_key = mm_dsl::OperationId::from_domain(id).0;
584 self.dsl
585 .0
586 .state()
587 .op_sources
588 .get(&id_key)
589 .map(|source| {
590 source.to_domain().map_err(|error| {
591 OpsLifecycleError::Internal(format!(
592 "generated operation source authority has invalid source for {id}: {error}"
593 ))
594 })
595 })
596 .transpose()
597 }
598
599 fn child_session_id_from_operation_source(
600 operation_source: Option<&OperationSource>,
601 ) -> Option<SessionId> {
602 match operation_source {
603 Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
604 Some(OperationSource::BackendPeer { .. }) | None => None,
605 }
606 }
607
608 fn align_spec_child_session_id_to_source(
609 spec: &mut OperationSpec,
610 operation_source: Option<&OperationSource>,
611 ) {
612 spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
613 }
614
615 fn peer_ready(&self, id: &OperationId) -> Option<bool> {
617 let id_key = mm_dsl::OperationId::from_domain(id).0;
618 self.dsl.0.state().op_peer_ready.get(&id_key).copied()
619 }
620
621 fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
622 self.peer_ready(id).ok_or_else(|| {
623 OpsLifecycleError::Internal(format!(
624 "generated op peer wiring authority missing peer-ready fact for {id}"
625 ))
626 })
627 }
628
629 fn progress_count(&self, id: &OperationId) -> Option<u32> {
631 let id_key = mm_dsl::OperationId::from_domain(id).0;
632 self.dsl
633 .0
634 .state()
635 .op_progress_counts
636 .get(&id_key)
637 .map(|v| (*v).min(u32::MAX as u64) as u32)
638 }
639
640 fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
641 self.progress_count(id).ok_or_else(|| {
642 OpsLifecycleError::Internal(format!(
643 "generated op progress authority missing progress count for {id}"
644 ))
645 })
646 }
647
648 fn terminal_outcome(
652 &self,
653 id: &OperationId,
654 ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
655 let id_key = mm_dsl::OperationId::from_domain(id).0;
656 let state = self.dsl.0.state();
657 let status = self.status(id);
658 let terminal = match status {
659 Some(status) => Self::operation_status_is_terminal(id, status)?,
660 None => false,
661 };
662 let kind = state.op_terminal_outcomes.get(&id_key).copied();
663 let Some(kind) = kind else {
664 if terminal {
665 return Err(OpsLifecycleError::Internal(format!(
666 "generated op terminal authority missing terminal outcome for {id}"
667 )));
668 }
669 return Ok(None);
670 };
671 if !terminal {
672 return Err(OpsLifecycleError::Internal(format!(
673 "generated op terminal authority has terminal outcome for non-terminal {id}"
674 )));
675 }
676 let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
677 OpsLifecycleError::Internal(format!(
678 "generated op terminal authority missing terminal payload for {id}"
679 ))
680 })?;
681 Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
682 .map(Some)
683 }
684
685 fn contains(&self, id: &OperationId) -> bool {
687 let id_key = mm_dsl::OperationId::from_domain(id).0;
688 self.dsl.0.state().op_statuses.contains_key(&id_key)
689 }
690
691 fn active_count(&self) -> usize {
693 self.dsl.0.state().active_op_count as usize
694 }
695
696 fn operation_count(&self) -> usize {
698 self.dsl.0.state().op_statuses.len()
699 }
700
701 fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
703 let mut ids = BTreeSet::new();
704 let state = self.dsl.0.state();
705 Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
706 Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
707 Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
708 Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
709 Self::collect_operation_id_keys(
710 &mut ids,
711 "op_progress_counts",
712 state.op_progress_counts.keys(),
713 )?;
714 Self::collect_operation_id_keys(
715 &mut ids,
716 "op_terminal_outcomes",
717 state.op_terminal_outcomes.keys(),
718 )?;
719 Self::collect_operation_id_keys(
720 &mut ids,
721 "op_terminal_payload",
722 state.op_terminal_payload.keys(),
723 )?;
724 Self::collect_operation_id_keys(
725 &mut ids,
726 "op_completion_seq",
727 state.op_completion_seq.keys(),
728 )?;
729 ids.extend(self.records.keys().cloned());
730 Ok(ids.into_iter().collect())
731 }
732
733 fn collect_operation_id_keys<'a, I>(
734 ids: &mut BTreeSet<OperationId>,
735 field: &str,
736 keys: I,
737 ) -> Result<(), OpsLifecycleError>
738 where
739 I: IntoIterator<Item = &'a String>,
740 {
741 for key in keys {
742 let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
743 OpsLifecycleError::Internal(format!(
744 "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
745 ))
746 })?;
747 ids.insert(id);
748 }
749 Ok(())
750 }
751
752 fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
753 let id_key = mm_dsl::OperationId::from_domain(id).0;
754 let state = self.dsl.0.state();
755 state.op_statuses.contains_key(&id_key)
756 || state.op_kinds.contains_key(&id_key)
757 || state.op_sources.contains_key(&id_key)
758 || state.op_peer_ready.contains_key(&id_key)
759 || state.op_progress_counts.contains_key(&id_key)
760 || state.op_terminal_outcomes.contains_key(&id_key)
761 || state.op_terminal_payload.contains_key(&id_key)
762 || state.op_completion_seq.contains_key(&id_key)
763 }
764
765 fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
767 let id_key = mm_dsl::OperationId::from_domain(id).0;
768 self.dsl.0.state().op_completion_seq.get(&id_key).copied()
769 }
770
771 fn completion_feed_authority_entries(
772 &self,
773 ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
774 let state = self.dsl.0.state();
775 let sequence_keys: BTreeSet<String> =
776 state.completion_feed_sequences.keys().cloned().collect();
777 let companion_domains: [(&str, BTreeSet<String>); 3] = [
778 (
779 "completion_feed_kinds",
780 state.completion_feed_kinds.keys().cloned().collect(),
781 ),
782 (
783 "completion_feed_terminal_outcomes",
784 state
785 .completion_feed_terminal_outcomes
786 .keys()
787 .cloned()
788 .collect(),
789 ),
790 (
791 "completion_feed_terminal_payload",
792 state
793 .completion_feed_terminal_payload
794 .keys()
795 .cloned()
796 .collect(),
797 ),
798 ];
799 for (field, keys) in companion_domains {
800 if keys != sequence_keys {
801 return Err(OpsLifecycleError::Internal(format!(
802 "generated completion feed authority has mismatched {field} domain"
803 )));
804 }
805 }
806
807 let mut entries = HashMap::new();
808 for (id_key, seq) in &state.completion_feed_sequences {
809 if !state.completion_sequence_claims.contains(seq) {
810 return Err(OpsLifecycleError::Internal(format!(
811 "generated completion feed authority sequence {seq} for {id_key} is not claimed"
812 )));
813 }
814 let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
815 OpsLifecycleError::Internal(format!(
816 "generated completion feed authority used invalid operation id key {id_key}: {error}"
817 ))
818 })?;
819 let kind = state
820 .completion_feed_kinds
821 .get(id_key)
822 .copied()
823 .map(OperationKind::from)
824 .ok_or_else(|| {
825 OpsLifecycleError::Internal(format!(
826 "generated completion feed authority missing kind for {id_key}"
827 ))
828 })?;
829 let outcome_kind = state
830 .completion_feed_terminal_outcomes
831 .get(id_key)
832 .copied()
833 .ok_or_else(|| {
834 OpsLifecycleError::Internal(format!(
835 "generated completion feed authority missing terminal outcome for {id_key}"
836 ))
837 })?;
838 let payload = state
839 .completion_feed_terminal_payload
840 .get(id_key)
841 .ok_or_else(|| {
842 OpsLifecycleError::Internal(format!(
843 "generated completion feed authority missing terminal payload for {id_key}"
844 ))
845 })?;
846 let terminal_outcome = Self::checked_terminal_payload(
847 outcome_kind,
848 payload,
849 "generated completion feed authority",
850 id_key,
851 )?;
852 entries.insert(
853 operation_id,
854 CompletionFeedCanonicalState {
855 seq: *seq,
856 kind,
857 terminal_outcome,
858 },
859 );
860 }
861 Ok(entries)
862 }
863
864 fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
865 let state = self.dsl.0.state();
866 match consumer {
867 CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
868 CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
869 CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
870 }
871 }
872
873 fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
874 meerkat_core::EpochCursorSnapshot {
875 agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
876 runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
877 runtime_last_injected_seq: self
878 .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
879 }
880 }
881
882 fn snapshot(
884 &self,
885 id: &OperationId,
886 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
887 let Some(shell) = self.records.get(id) else {
888 if self.has_generated_operation_record_fact(id) {
889 return Err(OpsLifecycleError::Internal(format!(
890 "generated op lifecycle authority has operation facts without shell projection record for {id}"
891 )));
892 }
893 return Ok(None);
894 };
895 let kind = self.require_kind(id)?;
896 let status = self.require_status(id)?;
897 let terminal = Self::operation_status_is_terminal(id, status)?;
898 let public_result_class = Self::operation_public_result_class(id, status)?;
899 let peer_ready = self.require_peer_ready(id)?;
900 let progress_count = self.require_progress_count(id)?;
901 let operation_source = self.operation_source(id)?;
902 let terminal_outcome = self.terminal_outcome(id)?;
903
904 let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
905 let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
906 let completed_at_ms = shell
907 .completed_at
908 .map(|i| shell.epoch_millis_for_instant(i));
909 let elapsed_ms = shell.completed_at.map(|completed| {
910 completed
911 .saturating_duration_since(shell.created_at)
912 .as_millis() as u64
913 });
914
915 Ok(Some(OperationLifecycleSnapshot {
916 id: shell.spec.id.clone(),
917 kind,
918 display_name: shell.spec.display_name.clone(),
919 child_session_id: Self::child_session_id_from_operation_source(
920 operation_source.as_ref(),
921 ),
922 operation_source,
923 status,
924 terminal,
925 public_result_class,
926 peer_ready,
927 progress_count,
928 watcher_count: shell.watchers.len() as u32,
929 terminal_outcome,
930 peer_handle: shell.peer_handle.clone(),
931 created_at_ms,
932 started_at_ms,
933 completed_at_ms,
934 elapsed_ms,
935 }))
936 }
937
938 fn finalize_terminal(
943 &mut self,
944 id: &OperationId,
945 ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
946 let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
947 OpsLifecycleError::Internal(format!(
948 "generated op terminal transition did not mint terminal outcome for {id}"
949 ))
950 })?;
951 let kind = self.require_kind(id)?;
952
953 if let Some(shell) = self.records.get_mut(id) {
955 shell.notify_watchers(&outcome);
956 shell.mark_completed();
957 }
958
959 if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
960 {
961 self.dsl_apply(
962 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
963 operation_id: mm_dsl::OperationId::from_domain(id).0,
964 },
965 "CollectCompletedOp",
966 )?;
967 self.records.remove(id);
968 self.completed_order.retain(|queued| queued != id);
969 return Ok(None);
970 }
971
972 let mut completion_entry = None;
973 if Self::operation_completion_feed_class(id, kind)?
974 == mm_dsl::OperationCompletionFeedClass::Emit
975 {
976 let feed_authority = self
977 .completion_feed_authority_entries()?
978 .remove(id)
979 .ok_or_else(|| {
980 OpsLifecycleError::Internal(format!(
981 "generated op terminal transition did not mint completion feed authority for {id}"
982 ))
983 })?;
984 if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
985 return Err(OpsLifecycleError::Internal(format!(
986 "generated completion feed authority drifted from terminal op authority for {id}"
987 )));
988 }
989 let seq = self.completion_sequence(id).ok_or_else(|| {
990 OpsLifecycleError::Internal(format!(
991 "generated op terminal transition did not mint completion sequence for {id}"
992 ))
993 })?;
994 if feed_authority.seq != seq {
995 return Err(OpsLifecycleError::Internal(format!(
996 "generated completion feed authority sequence drifted for {id}"
997 )));
998 }
999 let display_name = self
1000 .records
1001 .get(id)
1002 .map(|r| r.spec.display_name.clone())
1003 .unwrap_or_default();
1004 let completed_at_ms = self
1005 .records
1006 .get(id)
1007 .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1008 completion_entry = Some(CompletionEntry {
1009 seq: feed_authority.seq,
1010 operation_id: id.clone(),
1011 kind: feed_authority.kind,
1012 display_name,
1013 terminal_outcome: feed_authority.terminal_outcome,
1014 completed_at_ms,
1015 });
1016 }
1017
1018 self.completed_order.push_back(id.clone());
1020 while self.completed_order.len() > self.max_completed {
1021 if let Some(evicted) = self.completed_order.pop_front() {
1022 self.dsl_apply(
1023 mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1024 operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1025 },
1026 "EvictCompletedOp",
1027 )?;
1028 self.records.remove(&evicted);
1029 }
1030 }
1031
1032 self.maybe_satisfy_wait()?;
1037 Ok(completion_entry)
1038 }
1039
1040 fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1041 if let Some(entry) = entry {
1042 self.feed_buffer.push(entry);
1043 }
1044 }
1045
1046 fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1048 self.dsl
1049 .0
1050 .state()
1051 .wait_operation_ids
1052 .iter()
1053 .map(|key| {
1054 serde_json::from_str::<OperationId>(key).map_err(|error| {
1055 OpsLifecycleError::Internal(format!(
1056 "generated wait operation identity authority used invalid operation id key {key}: {error}"
1057 ))
1058 })
1059 })
1060 .collect()
1061 }
1062
1063 #[cfg(test)]
1065 fn wait_active(&self) -> bool {
1066 self.dsl.0.state().wait_active
1067 }
1068
1069 fn wait_all_satisfied_from_effects(
1070 effects: &[mm_dsl::MeerkatMachineEffect],
1071 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1072 let mut satisfied = None;
1073 for effect in effects {
1074 let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1075 wait_request_id,
1076 run_id,
1077 operation_ids,
1078 } = effect
1079 else {
1080 continue;
1081 };
1082 if satisfied.is_some() {
1083 return Err(OpsLifecycleError::Internal(
1084 "generated wait_all authority emitted multiple satisfaction effects".into(),
1085 ));
1086 }
1087 let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1088 OpsLifecycleError::Internal(format!(
1089 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1090 wait_request_id.0
1091 ))
1092 })?;
1093 let mut ids = Vec::with_capacity(operation_ids.len());
1094 for operation_id in operation_ids {
1095 ids.push(
1096 serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1097 OpsLifecycleError::Internal(format!(
1098 "generated wait_all authority emitted invalid operation id '{}': {err}",
1099 operation_id.0
1100 ))
1101 })?,
1102 );
1103 }
1104 satisfied = Some(WaitAllSatisfied {
1105 wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1106 run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1107 OpsLifecycleError::Internal(format!(
1108 "generated wait_all authority emitted invalid run id '{}': {err}",
1109 run_id.0
1110 ))
1111 })?),
1112 operation_ids: ids,
1113 });
1114 }
1115 Ok(satisfied)
1116 }
1117
1118 fn parse_wait_all_operation_id(
1119 raw: &str,
1120 context: &str,
1121 ) -> Result<OperationId, OpsLifecycleError> {
1122 serde_json::from_str::<OperationId>(raw).map_err(|err| {
1123 OpsLifecycleError::Internal(format!(
1124 "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1125 ))
1126 })
1127 }
1128
1129 fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1130 let mut seen = HashSet::new();
1131 operation_ids
1132 .iter()
1133 .find(|operation_id| !seen.insert((*operation_id).clone()))
1134 .cloned()
1135 }
1136
1137 fn wait_all_admission_error_from_effects(
1138 wait_request_id: &WaitRequestId,
1139 effects: &[mm_dsl::MeerkatMachineEffect],
1140 ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1141 let mut admission = None;
1142 for effect in effects {
1143 let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1144 wait_request_id: resolved_wait_request_id,
1145 result,
1146 reject_reason,
1147 rejected_operation_id,
1148 } = effect
1149 else {
1150 continue;
1151 };
1152 if admission.is_some() {
1153 return Err(OpsLifecycleError::Internal(
1154 "generated wait_all authority emitted multiple admission results".into(),
1155 ));
1156 }
1157 let resolved_uuid =
1158 uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1159 OpsLifecycleError::Internal(format!(
1160 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1161 resolved_wait_request_id.0
1162 ))
1163 })?;
1164 let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1165 if &resolved_wait_request_id != wait_request_id {
1166 return Err(OpsLifecycleError::Internal(format!(
1167 "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1168 )));
1169 }
1170 admission = Some(match result {
1171 mm_dsl::WaitAllAdmissionResultKind::Accept => {
1172 if reject_reason.is_some() || rejected_operation_id.is_some() {
1173 return Err(OpsLifecycleError::Internal(
1174 "generated wait_all authority accepted with rejection payload".into(),
1175 ));
1176 }
1177 None
1178 }
1179 mm_dsl::WaitAllAdmissionResultKind::Reject => {
1180 let reason = reject_reason.ok_or_else(|| {
1181 OpsLifecycleError::Internal(
1182 "generated wait_all authority rejected without reason".into(),
1183 )
1184 })?;
1185 let error = match reason {
1186 mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1187 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1188 OpsLifecycleError::Internal(
1189 "generated wait_all authority rejected duplicate without operation id"
1190 .into(),
1191 )
1192 })?;
1193 OpsLifecycleError::DuplicateWaitOperation(
1194 Self::parse_wait_all_operation_id(raw, "duplicate")?,
1195 )
1196 }
1197 mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1198 if rejected_operation_id.is_some() {
1199 return Err(OpsLifecycleError::Internal(
1200 "generated wait_all authority rejected active wait with operation id"
1201 .into(),
1202 ));
1203 }
1204 OpsLifecycleError::WaitAlreadyActive
1205 }
1206 mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1207 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1208 OpsLifecycleError::Internal(
1209 "generated wait_all authority rejected missing operation without operation id"
1210 .into(),
1211 )
1212 })?;
1213 OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1214 raw, "missing",
1215 )?)
1216 }
1217 };
1218 Some(error)
1219 }
1220 });
1221 }
1222 admission.ok_or_else(|| {
1223 OpsLifecycleError::Internal(
1224 "generated wait_all authority emitted no admission result".into(),
1225 )
1226 })
1227 }
1228
1229 fn resolve_wait_all_admission(
1230 &mut self,
1231 wait_request_id: &WaitRequestId,
1232 operation_ids: &[OperationId],
1233 dsl_ids: &BTreeSet<String>,
1234 dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1235 operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1236 operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1237 ) -> Result<(), OpsLifecycleError> {
1238 let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1239 .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1240 let not_found = operation_ids
1241 .iter()
1242 .find(|operation_id| !self.contains(operation_id))
1243 .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1244 let dsl_id_sequence: Vec<String> = operation_ids
1245 .iter()
1246 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1247 .collect();
1248 let effects = self.dsl_apply_with_effects(
1249 mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1250 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1251 operation_id_sequence: dsl_id_sequence,
1252 operation_ids: dsl_ids.clone(),
1253 operation_id_tokens: dsl_id_tokens.clone(),
1254 operation_token_by_id: operation_token_by_id.clone(),
1255 operation_id_by_token: operation_id_by_token.clone(),
1256 duplicate_operation_id: duplicate,
1257 not_found_operation_id: not_found,
1258 },
1259 "ResolveWaitAllAdmission",
1260 )?;
1261 if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1262 {
1263 return Err(error);
1264 }
1265 Ok(())
1266 }
1267
1268 fn try_satisfy_wait_all_authority(
1269 &mut self,
1270 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1271 let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1272 return Ok(None);
1273 };
1274 let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1275 return Err(OpsLifecycleError::Internal(
1276 "generated wait_all authority has active wait without run id".into(),
1277 ));
1278 };
1279 let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1280 let transition = match mm_dsl::MeerkatMachineMutator::apply(
1281 &mut *self.dsl.0,
1282 mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1283 wait_request_id: dsl_wait_request_id,
1284 run_id: dsl_run_id,
1285 operation_id_tokens: dsl_operation_id_tokens,
1286 },
1287 ) {
1288 Ok(transition) => transition,
1289 Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1290 Err(err) => {
1291 return Err(OpsLifecycleError::Internal(format!(
1292 "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1293 )));
1294 }
1295 };
1296 Self::wait_all_satisfied_from_effects(transition.effects())?
1297 .ok_or_else(|| {
1298 OpsLifecycleError::Internal(
1299 "generated wait_all authority accepted satisfaction without effect".into(),
1300 )
1301 })
1302 .map(Some)
1303 }
1304
1305 fn begin_wait_all_authority(
1306 &mut self,
1307 run_id: &RunId,
1308 wait_request_id: &WaitRequestId,
1309 operation_ids: &[OperationId],
1310 ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1311 let mut dsl_ids = BTreeSet::new();
1312 let mut dsl_id_tokens = BTreeSet::new();
1313 let mut operation_token_by_id = BTreeMap::new();
1314 let mut operation_id_by_token = BTreeMap::new();
1315 for id in operation_ids {
1316 let token = mm_dsl::OperationId::from_domain(id);
1317 let raw_id = token.0.clone();
1318 dsl_ids.insert(raw_id.clone());
1319 dsl_id_tokens.insert(token.clone());
1320 operation_token_by_id.insert(raw_id.clone(), token.clone());
1321 operation_id_by_token.insert(token, raw_id);
1322 }
1323 self.resolve_wait_all_admission(
1324 wait_request_id,
1325 operation_ids,
1326 &dsl_ids,
1327 &dsl_id_tokens,
1328 &operation_token_by_id,
1329 &operation_id_by_token,
1330 )?;
1331 self.dsl_apply(
1332 mm_dsl::MeerkatMachineInput::RequestWaitAll {
1333 run_id: mm_dsl::RunId::from_domain(run_id),
1334 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1335 operation_id_sequence: operation_ids
1336 .iter()
1337 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1338 .collect(),
1339 operation_ids: dsl_ids,
1340 operation_id_tokens: dsl_id_tokens,
1341 operation_token_by_id,
1342 operation_id_by_token,
1343 },
1344 "RequestWaitAll",
1345 )?;
1346 if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1347 return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1348 }
1349 Ok(WaitAllAuthorityPlan::ActivateBarrier)
1350 }
1351
1352 fn owner_termination_targets(
1353 &self,
1354 ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1355 let mut targets = Vec::new();
1356 for id in self.operation_ids()? {
1357 let status = self.require_status(&id)?;
1358 if !Self::operation_status_is_terminal(&id, status)? {
1359 targets.push((id, status));
1360 }
1361 }
1362 Ok(targets)
1363 }
1364
1365 fn operation_status_is_terminal(
1366 operation_id: &OperationId,
1367 status: OperationStatus,
1368 ) -> Result<bool, OpsLifecycleError> {
1369 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1370 let effects = Self::apply_stateless_classifier(
1371 mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1372 operation_id: operation_id_key.clone(),
1373 status: mm_dsl::OperationStatus::from(status),
1374 },
1375 "ClassifyOperationTerminality",
1376 )?;
1377 let mut terminal = None;
1378 for effect in effects {
1379 match effect {
1380 mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1381 if operation_id == operation_id_key =>
1382 {
1383 terminal = Some(true);
1384 }
1385 mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1386 if operation_id == operation_id_key =>
1387 {
1388 terminal = Some(false);
1389 }
1390 other => {
1391 return Err(OpsLifecycleError::Internal(format!(
1392 "unexpected generated operation terminality effect: {other:?}"
1393 )));
1394 }
1395 }
1396 }
1397 terminal.ok_or_else(|| {
1398 OpsLifecycleError::Internal(format!(
1399 "generated operation terminality authority emitted no effect for {operation_id}"
1400 ))
1401 })
1402 }
1403
1404 fn operation_public_result_class(
1405 operation_id: &OperationId,
1406 status: OperationStatus,
1407 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1408 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1409 let effects = Self::apply_stateless_classifier(
1410 mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1411 operation_id: operation_id_key.clone(),
1412 status: mm_dsl::OperationStatus::from(status),
1413 },
1414 "ClassifyOperationPublicResult",
1415 )?;
1416 let mut result = None;
1417 for effect in effects {
1418 match effect {
1419 mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1420 operation_id,
1421 result: classified,
1422 } if operation_id == operation_id_key => {
1423 result = Some(OperationPublicResultClass::from(classified));
1424 }
1425 other => {
1426 return Err(OpsLifecycleError::Internal(format!(
1427 "unexpected generated operation public-result effect: {other:?}"
1428 )));
1429 }
1430 }
1431 }
1432 result.ok_or_else(|| {
1433 OpsLifecycleError::Internal(format!(
1434 "generated operation public-result authority emitted no effect for {operation_id}"
1435 ))
1436 })
1437 }
1438
1439 fn operation_transition_rejection_is_idempotent(
1440 operation_id: &OperationId,
1441 action: OperationLifecycleAction,
1442 status: OperationStatus,
1443 ) -> Result<bool, OpsLifecycleError> {
1444 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1445 let action = mm_dsl::OpLifecycleActionKind::from(action);
1446 let status = mm_dsl::OperationStatus::from(status);
1447 let effects = Self::apply_stateless_classifier(
1448 mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1449 operation_id: operation_id_key.clone(),
1450 action,
1451 status,
1452 },
1453 "ClassifyOperationTransitionIdempotence",
1454 )?;
1455 let mut idempotent = None;
1456 for effect in effects {
1457 match effect {
1458 mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1459 operation_id,
1460 action: effect_action,
1461 status: effect_status,
1462 } if operation_id == operation_id_key
1463 && effect_action == action
1464 && effect_status == status =>
1465 {
1466 idempotent = Some(true);
1467 }
1468 mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1469 operation_id,
1470 action: effect_action,
1471 status: effect_status,
1472 } if operation_id == operation_id_key
1473 && effect_action == action
1474 && effect_status == status =>
1475 {
1476 idempotent = Some(false);
1477 }
1478 other => {
1479 return Err(OpsLifecycleError::Internal(format!(
1480 "unexpected generated operation transition-idempotence effect: {other:?}"
1481 )));
1482 }
1483 }
1484 }
1485 idempotent.ok_or_else(|| {
1486 OpsLifecycleError::Internal(format!(
1487 "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1488 ))
1489 })
1490 }
1491
1492 fn operation_completion_feed_class(
1493 operation_id: &OperationId,
1494 kind: OperationKind,
1495 ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1496 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1497 let kind = mm_dsl::OperationKind::from(kind);
1498 let effects = Self::apply_stateless_classifier(
1499 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1500 operation_id: operation_id_key.clone(),
1501 kind,
1502 },
1503 "ClassifyOperationCompletionFeed",
1504 )?;
1505 let mut class = None;
1506 for effect in effects {
1507 match effect {
1508 mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1509 operation_id,
1510 result,
1511 } if operation_id == operation_id_key => {
1512 class = Some(result);
1513 }
1514 other => {
1515 return Err(OpsLifecycleError::Internal(format!(
1516 "unexpected generated operation completion-feed effect: {other:?}"
1517 )));
1518 }
1519 }
1520 }
1521 class.ok_or_else(|| {
1522 OpsLifecycleError::Internal(format!(
1523 "generated operation completion-feed authority emitted no effect for {operation_id}"
1524 ))
1525 })
1526 }
1527
1528 fn operation_completion_wake_class(
1529 operation_id: &OperationId,
1530 kind: OperationKind,
1531 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1532 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1533 let kind = mm_dsl::OperationKind::from(kind);
1534 let effects = Self::apply_stateless_classifier(
1535 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1536 operation_id: operation_id_key.clone(),
1537 kind,
1538 },
1539 "ClassifyOperationCompletionWake",
1540 )?;
1541 let mut class = None;
1542 for effect in effects {
1543 match effect {
1544 mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1545 operation_id,
1546 result,
1547 } if operation_id == operation_id_key => {
1548 class = Some(OperationCompletionWakeClass::from(result));
1549 }
1550 other => {
1551 return Err(OpsLifecycleError::Internal(format!(
1552 "unexpected generated operation completion-wake effect: {other:?}"
1553 )));
1554 }
1555 }
1556 }
1557 class.ok_or_else(|| {
1558 OpsLifecycleError::Internal(format!(
1559 "generated operation completion-wake authority emitted no effect for {operation_id}"
1560 ))
1561 })
1562 }
1563
1564 fn operation_durability_class(
1565 operation_id: &OperationId,
1566 kind: OperationKind,
1567 ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1568 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1569 let kind = mm_dsl::OperationKind::from(kind);
1570 let effects = Self::apply_stateless_classifier(
1571 mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1572 operation_id: operation_id_key.clone(),
1573 kind,
1574 },
1575 "ClassifyOperationDurability",
1576 )?;
1577 let mut class = None;
1578 for effect in effects {
1579 match effect {
1580 mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1581 operation_id,
1582 result,
1583 } if operation_id == operation_id_key => {
1584 class = Some(result);
1585 }
1586 other => {
1587 return Err(OpsLifecycleError::Internal(format!(
1588 "unexpected generated operation durability effect: {other:?}"
1589 )));
1590 }
1591 }
1592 }
1593 class.ok_or_else(|| {
1594 OpsLifecycleError::Internal(format!(
1595 "generated operation durability authority emitted no effect for {operation_id}"
1596 ))
1597 })
1598 }
1599
1600 fn recovered_operation_record_disposition(
1601 operation_id: &OperationId,
1602 status: OperationStatus,
1603 kind: OperationKind,
1604 terminal_outcome_present: bool,
1605 terminal_payload_present: bool,
1606 completion_sequence_present: bool,
1607 ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1608 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1609 let effects = Self::apply_stateless_classifier(
1610 mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1611 operation_id: operation_id_key.clone(),
1612 status: mm_dsl::OperationStatus::from(status),
1613 kind: mm_dsl::OperationKind::from(kind),
1614 terminal_outcome_present,
1615 terminal_payload_present,
1616 completion_sequence_present,
1617 },
1618 "ClassifyRecoveredOperationRecord",
1619 )?;
1620 let mut disposition = None;
1621 for effect in effects {
1622 match effect {
1623 mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1624 if operation_id == operation_id_key =>
1625 {
1626 disposition = Some(RecoveredOperationRecordDisposition::Retain);
1627 }
1628 mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1629 if operation_id == operation_id_key =>
1630 {
1631 disposition = Some(RecoveredOperationRecordDisposition::Discard);
1632 }
1633 other => {
1634 return Err(OpsLifecycleError::Internal(format!(
1635 "unexpected generated recovered-operation classification effect: {other:?}"
1636 )));
1637 }
1638 }
1639 }
1640 disposition.ok_or_else(|| {
1641 OpsLifecycleError::Internal(format!(
1642 "generated recovered-operation classifier emitted no effect for {operation_id}"
1643 ))
1644 })
1645 }
1646
1647 fn apply_stateless_classifier(
1648 input: mm_dsl::MeerkatMachineInput,
1649 label: &'static str,
1650 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1651 let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1652 "ops stateless classifier Initialize must be accepted",
1653 );
1654 let transition =
1655 mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1656 OpsLifecycleError::Internal(format!(
1657 "DSL rejected ops transition ({label}): {err:?}"
1658 ))
1659 })?;
1660 Ok(transition.into_effects())
1661 }
1662
1663 fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1687 let satisfied = match self.try_satisfy_wait_all_authority() {
1688 Ok(Some(satisfied)) => satisfied,
1689 Ok(None) => return Ok(()),
1690 Err(err) => {
1691 if let Some(pending) = self.pending_wait.take() {
1699 drop(pending.sender);
1700 }
1701 self.wait_request_id = None;
1702 return Err(err);
1703 }
1704 };
1705 let shell_wait_id = self.wait_request_id.take();
1706 if shell_wait_id
1707 .as_ref()
1708 .is_some_and(|id| id != &satisfied.wait_request_id)
1709 {
1710 tracing::error!(
1711 shell_wait_request_id = ?shell_wait_id,
1712 authority_wait_request_id = %satisfied.wait_request_id,
1713 "generated wait_all authority satisfied a different wait request"
1714 );
1715 }
1716 if let Some(pending) = self.pending_wait.take() {
1717 if pending.wait_request_id == satisfied.wait_request_id {
1718 let _ = pending.sender.send(satisfied);
1719 } else if let Some(shell_wait_id) = shell_wait_id {
1720 tracing::error!(
1721 shell_wait_request_id = %shell_wait_id,
1722 pending_wait_request_id = %pending.wait_request_id,
1723 authority_wait_request_id = %satisfied.wait_request_id,
1724 "generated wait_all authority satisfied without a matching pending waiter"
1725 );
1726 }
1727 }
1728 Ok(())
1729 }
1730
1731 fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1737 let (tx, epoch_id, cursor_state) = match (
1738 &self.persist_tx,
1739 &self.persist_epoch_id,
1740 &self.persist_cursor_state,
1741 ) {
1742 (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1743 _ => return Ok(()),
1744 };
1745
1746 let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1747 let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1748 let request = OpsLifecyclePersistenceRequest {
1749 snapshot,
1750 result_tx,
1751 };
1752
1753 tx.send(request).map_err(|_| {
1754 OpsLifecycleError::Internal(
1755 "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1756 .into(),
1757 )
1758 })?;
1759 result_rx.recv().map_err(|_| {
1760 OpsLifecycleError::Internal(
1761 "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1762 .into(),
1763 )
1764 })?
1765 }
1766
1767 fn capture_snapshot(
1769 &self,
1770 epoch_id: meerkat_core::RuntimeEpochId,
1771 _cursor_state: &meerkat_core::EpochCursorState,
1772 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1773 let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1774 for op_id in self.operation_ids()? {
1775 let status = self.require_status(&op_id)?;
1776 let kind = self.require_kind(&op_id)?;
1777 if Self::operation_durability_class(&op_id, kind)?
1778 != mm_dsl::OperationDurabilityClass::Retain
1779 {
1780 continue;
1781 }
1782 let peer_ready = self.require_peer_ready(&op_id)?;
1783 let progress_count = self.require_progress_count(&op_id)?;
1784 let operation_source = self.operation_source(&op_id)?;
1785 let terminal_outcome = self.terminal_outcome(&op_id)?;
1786 let completion_sequence = self.completion_sequence(&op_id);
1787 if terminal_outcome.is_some() && completion_sequence.is_none() {
1788 return Err(OpsLifecycleError::Internal(format!(
1789 "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1790 )));
1791 }
1792 if terminal_outcome.is_none() && completion_sequence.is_some() {
1793 return Err(OpsLifecycleError::Internal(format!(
1794 "generated op terminal authority has completion sequence for non-terminal {op_id}"
1795 )));
1796 }
1797 let terminal_buffered = terminal_outcome.is_some();
1798 let watcher_count = self
1799 .records
1800 .get(&op_id)
1801 .map(|r| r.watchers.len() as u32)
1802 .unwrap_or(0);
1803 operations.insert(
1804 op_id,
1805 OperationCanonicalState {
1806 status,
1807 kind,
1808 operation_source,
1809 peer_ready,
1810 progress_count,
1811 watcher_count,
1812 terminal_outcome,
1813 completion_sequence,
1814 terminal_buffered,
1815 },
1816 );
1817 }
1818 let operation_specs: HashMap<OperationId, OperationSpec> = self
1819 .records
1820 .iter()
1821 .filter(|(id, _)| operations.contains_key(*id))
1822 .map(|(id, record)| {
1823 let mut spec = record.spec.clone();
1824 let operation_source = operations
1825 .get(id)
1826 .and_then(|state| state.operation_source.as_ref());
1827 Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1828 (id.clone(), spec)
1829 })
1830 .collect();
1831 let completed_order: VecDeque<OperationId> = self
1832 .completed_order
1833 .iter()
1834 .filter(|id| operations.contains_key(*id))
1835 .cloned()
1836 .collect();
1837 let active_count = operations
1838 .iter()
1839 .filter(|(id, state)| {
1840 matches!(
1841 Self::operation_status_is_terminal(id, state.status),
1842 Ok(false)
1843 )
1844 })
1845 .count();
1846 let authority_completion_entries = self.completion_feed_authority_entries()?;
1847 let published_completion_entries_by_id: HashMap<OperationId, CompletionEntry> = {
1848 let inner = self
1849 .feed_buffer
1850 .inner
1851 .read()
1852 .unwrap_or_else(std::sync::PoisonError::into_inner);
1853 let mut entries = HashMap::new();
1854 for entry in &inner.entries {
1855 if !authority_completion_entries.contains_key(&entry.operation_id) {
1856 return Err(OpsLifecycleError::Internal(format!(
1857 "public completion feed projection for {} has no generated authority",
1858 entry.operation_id
1859 )));
1860 }
1861 if entries
1862 .insert(entry.operation_id.clone(), entry.clone())
1863 .is_some()
1864 {
1865 return Err(OpsLifecycleError::Internal(format!(
1866 "public completion feed projection for {} appeared more than once",
1867 entry.operation_id
1868 )));
1869 }
1870 }
1871 entries
1872 };
1873 let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
1874 .iter()
1875 .map(|(operation_id, authority_entry)| {
1876 if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
1877 if projection.seq != authority_entry.seq
1878 || projection.kind != authority_entry.kind
1879 || projection.terminal_outcome != authority_entry.terminal_outcome
1880 {
1881 return Err(OpsLifecycleError::Internal(format!(
1882 "public completion feed projection for {operation_id} drifted from generated authority"
1883 )));
1884 }
1885 return Ok(projection.clone());
1886 }
1887
1888 let display_name = self
1889 .records
1890 .get(operation_id)
1891 .map(|record| record.spec.display_name.clone())
1892 .unwrap_or_default();
1893 let completed_at_ms = self.records.get(operation_id).and_then(|record| {
1894 record
1895 .completed_at
1896 .map(|completed_at| record.epoch_millis_for_instant(completed_at))
1897 });
1898
1899 Ok(CompletionEntry {
1900 seq: authority_entry.seq,
1901 operation_id: operation_id.clone(),
1902 kind: authority_entry.kind,
1903 display_name,
1904 terminal_outcome: authority_entry.terminal_outcome.clone(),
1905 completed_at_ms,
1906 })
1907 })
1908 .collect::<Result<_, _>>()?;
1909 completion_entries.sort_by_key(|entry| entry.seq);
1910
1911 let authority_state = RegistryCanonicalState {
1912 operations,
1913 completion_feed_entries: authority_completion_entries,
1914 completed_order,
1915 max_completed: self.max_completed,
1916 max_concurrent: self.max_concurrent,
1917 active_count,
1918 wait_request_id: self.wait_request_id.clone(),
1919 wait_operation_ids: self.wait_operation_ids()?,
1920 next_completion_seq: self.dsl.0.state().next_completion_seq,
1921 };
1922
1923 Ok(PersistedOpsSnapshot {
1924 epoch_id,
1925 authority_state,
1926 operation_specs,
1927 completion_entries,
1928 cursors: self.completion_cursor_snapshot(),
1929 })
1930 }
1931
1932 fn shell_record_mut(
1933 &mut self,
1934 id: &OperationId,
1935 ) -> Result<&mut ShellRecord, OpsLifecycleError> {
1936 self.records
1937 .get_mut(id)
1938 .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
1939 }
1940
1941 fn collect_wait_outcomes(
1942 &self,
1943 operation_ids: &[OperationId],
1944 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
1945 operation_ids
1946 .iter()
1947 .map(|operation_id| {
1948 let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
1949 OpsLifecycleError::Internal(format!(
1950 "wait_all completed without terminal outcome for {operation_id}"
1951 ))
1952 })?;
1953 Ok((operation_id.clone(), outcome))
1954 })
1955 .collect()
1956 }
1957}
1958
1959impl Default for ShellState {
1960 fn default() -> Self {
1961 Self::new(DEFAULT_MAX_COMPLETED, None)
1962 }
1963}
1964
1965#[derive(Debug, Clone)]
1971pub struct OpsLifecycleConfig {
1972 pub max_completed: usize,
1974 pub max_concurrent: Option<usize>,
1976}
1977
1978impl Default for OpsLifecycleConfig {
1979 fn default() -> Self {
1980 Self {
1981 max_completed: DEFAULT_MAX_COMPLETED,
1982 max_concurrent: None,
1983 }
1984 }
1985}
1986
1987#[derive(Debug)]
1994pub struct RuntimeOpsLifecycleRegistry {
1995 state: RwLock<ShellState>,
1996}
1997
1998#[derive(Debug, Clone)]
1999pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2000 pub operation_count: usize,
2001 pub active_count: usize,
2002 pub wait_request_id: Option<WaitRequestId>,
2003 pub pending_wait_present: bool,
2004 pub pending_wait_request_id: Option<WaitRequestId>,
2005 pub wait_operation_ids: Vec<OperationId>,
2006 pub operations: Vec<OperationLifecycleSnapshot>,
2007}
2008
2009impl Default for RuntimeOpsLifecycleRegistry {
2010 fn default() -> Self {
2011 Self {
2012 state: RwLock::new(ShellState::default()),
2013 }
2014 }
2015}
2016
2017impl RuntimeOpsLifecycleRegistry {
2018 pub fn new() -> Self {
2019 let dsl = new_ops_dsl_authority();
2020 let feed_capacity = DEFAULT_MAX_COMPLETED.saturating_mul(4).max(1024);
2021 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2022 Self {
2023 state: RwLock::new(ShellState {
2024 dsl,
2025 records: HashMap::new(),
2026 pending_wait: None,
2027 completed_order: VecDeque::new(),
2028 max_completed: DEFAULT_MAX_COMPLETED,
2029 max_concurrent: None,
2030 wait_request_id: None,
2031 feed_buffer,
2032 persist_tx: None,
2033 persist_epoch_id: None,
2034 persist_cursor_state: None,
2035 }),
2036 }
2037 }
2038
2039 pub fn with_config(config: OpsLifecycleConfig) -> Self {
2040 Self {
2041 state: RwLock::new(ShellState::new(config.max_completed, config.max_concurrent)),
2042 }
2043 }
2044
2045 fn recover_completion_feed_entry(
2046 shell: &mut ShellState,
2047 operation_id: &OperationId,
2048 entry: &CompletionFeedCanonicalState,
2049 ) -> Result<(), OpsLifecycleError> {
2050 let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2051 let terminal_outcome_kind =
2052 mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2053 let effects = shell.dsl_apply_with_effects(
2054 mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2055 operation_id: expected_operation_id.clone(),
2056 kind: mm_dsl::OperationKind::from(entry.kind),
2057 terminal_outcome: terminal_outcome_kind,
2058 terminal_payload: entry.terminal_outcome.clone(),
2059 completion_sequence: entry.seq,
2060 },
2061 "RecoverCompletionFeedEntry",
2062 )?;
2063 let recovered = effects.iter().find_map(|effect| match effect {
2064 mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2065 operation_id,
2066 seq,
2067 kind,
2068 terminal_outcome,
2069 terminal_payload,
2070 } => Some((
2071 operation_id,
2072 *seq,
2073 OperationKind::from(*kind),
2074 *terminal_outcome,
2075 terminal_payload,
2076 )),
2077 _ => None,
2078 });
2079 let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2080 return Err(OpsLifecycleError::Internal(
2081 "generated completion-feed recovery emitted no recovered entry".into(),
2082 ));
2083 };
2084 if operation_id != &expected_operation_id
2085 || seq != entry.seq
2086 || kind != entry.kind
2087 || terminal_outcome != terminal_outcome_kind
2088 || terminal_payload != &entry.terminal_outcome
2089 {
2090 return Err(OpsLifecycleError::Internal(format!(
2091 "generated completion-feed recovery drifted for {operation_id}"
2092 )));
2093 }
2094 Ok(())
2095 }
2096
2097 pub fn set_persistence_channel(
2103 &self,
2104 tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2105 epoch_id: meerkat_core::RuntimeEpochId,
2106 cursor_state: Arc<meerkat_core::EpochCursorState>,
2107 ) {
2108 if let Ok(mut state) = self.state.write() {
2109 state.persist_tx = Some(tx);
2110 state.persist_epoch_id = Some(epoch_id);
2111 state.persist_cursor_state = Some(cursor_state);
2112 }
2113 }
2114
2115 pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2122 let PersistedOpsSnapshot {
2123 authority_state,
2124 operation_specs,
2125 completion_entries,
2126 cursors,
2127 ..
2128 } = snapshot;
2129 let max_completed = authority_state.max_completed;
2130 let max_concurrent = authority_state.max_concurrent;
2131 let next_completion_seq = authority_state.next_completion_seq;
2132 let authority_completion_entries = authority_state.completion_feed_entries;
2133 let authority_operations = authority_state.operations;
2134 let mut shell = ShellState::new(max_completed, max_concurrent);
2135
2136 let mut retained_ids: HashSet<OperationId> = HashSet::new();
2142 for (op_id, op_state) in authority_operations {
2143 let terminal_outcome = op_state
2144 .terminal_outcome
2145 .as_ref()
2146 .map(mm_dsl::OperationTerminalOutcomeKind::from);
2147 let terminal_payload = op_state.terminal_outcome.clone();
2148 let disposition = ShellState::recovered_operation_record_disposition(
2149 &op_id,
2150 op_state.status,
2151 op_state.kind,
2152 terminal_outcome.is_some(),
2153 terminal_payload.is_some(),
2154 op_state.completion_sequence.is_some(),
2155 )?;
2156 if disposition == RecoveredOperationRecordDisposition::Discard {
2157 continue;
2158 }
2159 if let Some(spec_source) = operation_specs
2160 .get(&op_id)
2161 .and_then(|spec| spec.operation_source.as_ref())
2162 && op_state.operation_source.as_ref() != Some(spec_source)
2163 {
2164 return Err(OpsLifecycleError::Internal(format!(
2165 "persisted operation source mirror for {op_id} drifted from generated authority"
2166 )));
2167 }
2168 let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2169 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2170 status: mm_dsl::OperationStatus::from(op_state.status),
2171 kind: mm_dsl::OperationKind::from(op_state.kind),
2172 source: op_state
2173 .operation_source
2174 .as_ref()
2175 .map(mm_dsl::OperationSource::from_domain),
2176 peer_ready: op_state.peer_ready,
2177 progress_count: u64::from(op_state.progress_count),
2178 terminal_outcome,
2179 terminal_payload,
2180 completion_sequence: op_state.completion_sequence,
2181 };
2182 shell.dsl_apply(recovery, "RecoverOpRecord")?;
2183 let recovered_seq = shell.completion_sequence(&op_id).ok_or_else(|| {
2184 OpsLifecycleError::Internal(format!(
2185 "generated op recovery accepted {op_id} without completion sequence"
2186 ))
2187 })?;
2188 if op_state.completion_sequence != Some(recovered_seq) {
2189 return Err(OpsLifecycleError::Internal(format!(
2190 "generated op recovery completion sequence mismatch for {op_id}"
2191 )));
2192 }
2193 retained_ids.insert(op_id);
2194 }
2195 shell.dsl_apply(
2196 mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2197 next_completion_seq,
2198 },
2199 "RecoverOpsCompletionCursor",
2200 )?;
2201 shell.dsl_apply(
2202 mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2203 agent_applied_cursor: cursors.agent_applied_cursor,
2204 runtime_observed_cursor: cursors.runtime_observed_seq,
2205 runtime_injected_cursor: cursors.runtime_last_injected_seq,
2206 },
2207 "RecoverCompletionConsumerCursors",
2208 )?;
2209
2210 let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2213 .iter()
2214 .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2215 .collect();
2216 recovered_completed.sort_by_key(|(seq, _)| *seq);
2217 shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2218
2219 for (operation_id, entry) in &authority_completion_entries {
2223 if !retained_ids.contains(operation_id) {
2224 Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2225 }
2226 }
2227
2228 let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2229 for (operation_id, entry) in &authority_completion_entries {
2230 let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2231 return Err(OpsLifecycleError::Internal(format!(
2232 "persisted completion feed authority for {operation_id} was not recovered"
2233 )));
2234 };
2235 if recovered_entry != entry {
2236 return Err(OpsLifecycleError::Internal(format!(
2237 "persisted completion feed authority drifted from generated recovery for {operation_id}"
2238 )));
2239 }
2240 }
2241
2242 let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2246 for entry in completion_entries {
2247 let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2248 return Err(OpsLifecycleError::Internal(format!(
2249 "persisted completion feed projection for {} has no generated feed authority",
2250 entry.operation_id
2251 )));
2252 };
2253 if authority_entry.seq != entry.seq
2254 || authority_entry.kind != entry.kind
2255 || authority_entry.terminal_outcome != entry.terminal_outcome
2256 {
2257 return Err(OpsLifecycleError::Internal(format!(
2258 "persisted completion feed projection for {} drifted from generated feed authority",
2259 entry.operation_id
2260 )));
2261 }
2262 projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2263 }
2264
2265 let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2266 canonical_feed_entries.into_iter().collect();
2267 recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2268 for (operation_id, entry) in recovered_entries {
2269 let projection = projection_entries_by_id.get(&operation_id);
2270 let display_name = operation_specs
2271 .get(&operation_id)
2272 .map(|spec| spec.display_name.clone())
2273 .or_else(|| projection.map(|entry| entry.display_name.clone()))
2274 .unwrap_or_default();
2275 let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2276 shell.feed_buffer.push(CompletionEntry {
2277 seq: entry.seq,
2278 operation_id,
2279 kind: entry.kind,
2280 display_name,
2281 terminal_outcome: entry.terminal_outcome,
2282 completed_at_ms,
2283 });
2284 }
2285
2286 for (op_id, spec) in operation_specs {
2289 if retained_ids.contains(&op_id) {
2290 let mut spec = spec;
2291 let operation_source = shell.operation_source(&op_id)?;
2292 ShellState::align_spec_child_session_id_to_source(
2293 &mut spec,
2294 operation_source.as_ref(),
2295 );
2296 shell.records.insert(
2297 op_id,
2298 ShellRecord {
2299 spec,
2300 peer_handle: None,
2301 watchers: Vec::new(),
2302 created_at: Instant::now(),
2303 started_at: None,
2304 completed_at: None,
2305 created_at_wall: SystemTime::now(),
2306 },
2307 );
2308 }
2309 }
2310
2311 Ok(Self {
2312 state: RwLock::new(shell),
2313 })
2314 }
2315
2316 pub fn capture_persistence_snapshot(
2321 &self,
2322 epoch_id: meerkat_core::RuntimeEpochId,
2323 cursor_state: &meerkat_core::EpochCursorState,
2324 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2325 let state = self
2326 .state
2327 .read()
2328 .unwrap_or_else(std::sync::PoisonError::into_inner);
2329 state.capture_snapshot(epoch_id, cursor_state)
2330 }
2331
2332 pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2334 let state = self
2335 .state
2336 .read()
2337 .unwrap_or_else(std::sync::PoisonError::into_inner);
2338 state.completion_cursor_snapshot()
2339 }
2340
2341 pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2343 let state = self
2344 .state
2345 .read()
2346 .unwrap_or_else(std::sync::PoisonError::into_inner);
2347 Arc::new(RuntimeCompletionFeed {
2348 buffer: Arc::clone(&state.feed_buffer),
2349 })
2350 }
2351
2352 pub(crate) fn diagnostic_snapshot(
2354 &self,
2355 ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2356 let state = self
2357 .state
2358 .read()
2359 .unwrap_or_else(std::sync::PoisonError::into_inner);
2360 let mut operations = state
2361 .operation_ids()?
2362 .into_iter()
2363 .map(|id| state.snapshot(&id))
2364 .collect::<Result<Vec<_>, _>>()?
2365 .into_iter()
2366 .flatten()
2367 .collect::<Vec<_>>();
2368 operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2369 Ok(RuntimeOpsDiagnosticSnapshot {
2370 operation_count: state.operation_count(),
2371 active_count: state.active_count(),
2372 wait_request_id: state.wait_request_id.clone(),
2373 pending_wait_present: state.pending_wait.is_some(),
2374 pending_wait_request_id: state
2375 .pending_wait
2376 .as_ref()
2377 .map(|pending_wait| pending_wait.wait_request_id.clone()),
2378 wait_operation_ids: state.wait_operation_ids()?,
2379 operations,
2380 })
2381 }
2382
2383 fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2384 self.state
2385 .read()
2386 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2387 }
2388
2389 fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2390 self.state
2391 .write()
2392 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2393 }
2394
2395 fn cancel_wait_all_internal(
2396 &self,
2397 wait_request_id: &WaitRequestId,
2398 ) -> Result<(), OpsLifecycleError> {
2399 let mut state = self.write_state()?;
2400 match state.wait_request_id.as_ref() {
2401 Some(active) if active == wait_request_id => {
2402 state.dsl_apply(
2407 mm_dsl::MeerkatMachineInput::CancelWaitAll,
2408 "CancelWaitAll(cancel)",
2409 )?;
2410 state.wait_request_id = None;
2411 if state
2412 .pending_wait
2413 .as_ref()
2414 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2415 {
2416 state.pending_wait = None;
2417 }
2418 Ok(())
2419 }
2420 _ => {
2421 if state
2422 .pending_wait
2423 .as_ref()
2424 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2425 {
2426 state.pending_wait = None;
2427 }
2428 Ok(())
2429 }
2430 }
2431 }
2432}
2433
2434enum WaitAllFutureState {
2435 Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2436 Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2437 Done,
2438}
2439
2440struct WaitAllFuture<'a> {
2441 registry: &'a RuntimeOpsLifecycleRegistry,
2442 wait_request_id: WaitRequestId,
2443 state: WaitAllFutureState,
2444}
2445
2446impl Future for WaitAllFuture<'_> {
2447 type Output = Result<WaitAllResult, OpsLifecycleError>;
2448
2449 fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2450 match &mut self.state {
2451 WaitAllFutureState::Ready(result) => {
2452 let ready = result.take().unwrap_or_else(|| {
2453 Err(OpsLifecycleError::Internal(
2454 "wait_all future polled after completion".into(),
2455 ))
2456 });
2457 self.state = WaitAllFutureState::Done;
2458 Poll::Ready(ready)
2459 }
2460 WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2461 Poll::Pending => Poll::Pending,
2462 Poll::Ready(Ok(satisfied)) => {
2463 let outcomes = match self.registry.read_state() {
2464 Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2465 Err(err) => Err(err),
2466 };
2467 self.state = WaitAllFutureState::Done;
2468 Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2469 outcomes,
2470 satisfied,
2471 }))
2472 }
2473 Poll::Ready(Err(_)) => {
2474 self.state = WaitAllFutureState::Done;
2475 Poll::Ready(Err(OpsLifecycleError::Internal(
2476 "wait_all completion channel dropped".into(),
2477 )))
2478 }
2479 },
2480 WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2481 "wait_all future polled after completion".into(),
2482 ))),
2483 }
2484 }
2485}
2486
2487impl Drop for WaitAllFuture<'_> {
2488 fn drop(&mut self) {
2489 if matches!(self.state, WaitAllFutureState::Waiting(_))
2490 && let Err(err) = self
2491 .registry
2492 .cancel_wait_all_internal(&self.wait_request_id)
2493 {
2494 tracing::error!(
2495 wait_request_id = %self.wait_request_id,
2496 error = %err,
2497 "generated wait_all authority rejected cancellation during drop"
2498 );
2499 }
2500 }
2501}
2502
2503fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2508 match action {
2509 mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2510 mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2511 mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2512 mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2513 mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2514 mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2515 mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2516 mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2517 mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2518 mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2519 }
2520}
2521
2522fn op_lifecycle_rejection_error_from_effects(
2523 id: &OperationId,
2524 requested_action: mm_dsl::OpLifecycleActionKind,
2525 effects: &[mm_dsl::MeerkatMachineEffect],
2526) -> Result<OpsLifecycleError, OpsLifecycleError> {
2527 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2528 let mut rejection = None;
2529 for effect in effects {
2530 let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2531 operation_id,
2532 action,
2533 reason,
2534 status,
2535 } = effect
2536 else {
2537 continue;
2538 };
2539 if rejection.is_some() {
2540 return Err(OpsLifecycleError::Internal(
2541 "generated op lifecycle authority emitted multiple rejection results".into(),
2542 ));
2543 }
2544 if operation_id != &expected_id || *action != requested_action {
2545 return Err(OpsLifecycleError::Internal(format!(
2546 "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2547 )));
2548 }
2549 let error = match reason {
2550 mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2551 if status.is_some() {
2552 return Err(OpsLifecycleError::Internal(
2553 "generated op lifecycle authority emitted not-found with status".into(),
2554 ));
2555 }
2556 OpsLifecycleError::NotFound(id.clone())
2557 }
2558 mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2559 let status = status.ok_or_else(|| {
2560 OpsLifecycleError::Internal(
2561 "generated op lifecycle authority emitted invalid-transition without status"
2562 .into(),
2563 )
2564 })?;
2565 OpsLifecycleError::InvalidTransition {
2566 id: id.clone(),
2567 status: OperationStatus::from(status),
2568 action: op_lifecycle_action_label(requested_action),
2569 }
2570 }
2571 mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2572 if status.is_none() {
2573 return Err(OpsLifecycleError::Internal(
2574 "generated op lifecycle authority emitted peer-not-expected without status"
2575 .into(),
2576 ));
2577 }
2578 OpsLifecycleError::PeerNotExpected(id.clone())
2579 }
2580 mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2581 if status.is_none() {
2582 return Err(OpsLifecycleError::Internal(
2583 "generated op lifecycle authority emitted already-peer-ready without status"
2584 .into(),
2585 ));
2586 }
2587 OpsLifecycleError::AlreadyPeerReady(id.clone())
2588 }
2589 };
2590 rejection = Some(error);
2591 }
2592 rejection.ok_or_else(|| {
2593 OpsLifecycleError::Internal(
2594 "generated op lifecycle authority emitted no rejection result".into(),
2595 )
2596 })
2597}
2598
2599fn classify_generated_op_rejection(
2600 state: &mut ShellState,
2601 err: mm_dsl::MeerkatMachineTransitionError,
2602 id: &OperationId,
2603 action: mm_dsl::OpLifecycleActionKind,
2604) -> OpsLifecycleError {
2605 match err {
2606 mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2607 match state.dsl_apply_with_effects(
2608 mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2609 operation_id: mm_dsl::OperationId::from_domain(id).0,
2610 action,
2611 },
2612 "ResolveOpLifecycleTransitionRejection",
2613 ) {
2614 Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2615 .unwrap_or_else(|err| err),
2616 Err(err) => err,
2617 }
2618 }
2619 other => OpsLifecycleError::Internal(format!(
2620 "DSL rejected ops transition ({}): {other:?}",
2621 op_lifecycle_action_label(action)
2622 )),
2623 }
2624}
2625
2626fn apply_op_transition(
2627 state: &mut ShellState,
2628 id: &OperationId,
2629 input: mm_dsl::MeerkatMachineInput,
2630 action: mm_dsl::OpLifecycleActionKind,
2631) -> Result<(), OpsLifecycleError> {
2632 state
2633 .dsl_apply_raw(input)
2634 .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2635}
2636
2637fn apply_terminal_op_transition_and_persist(
2638 state: &mut ShellState,
2639 id: &OperationId,
2640 input: mm_dsl::MeerkatMachineInput,
2641 action: mm_dsl::OpLifecycleActionKind,
2642) -> Result<(), OpsLifecycleError> {
2643 let previous_snapshot = state.dsl.0.snapshot();
2644 apply_op_transition(state, id, input, action)?;
2645 if let Err(err) = state.maybe_persist() {
2646 state.dsl.0.restore_snapshot(previous_snapshot);
2647 return Err(err);
2648 }
2649 Ok(())
2650}
2651
2652fn op_registration_error_from_effects(
2653 id: &OperationId,
2654 effects: &[mm_dsl::MeerkatMachineEffect],
2655) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2656 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2657 let mut admission = None;
2658 for effect in effects {
2659 let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2660 operation_id,
2661 result,
2662 reject_reason,
2663 max_concurrent_limit,
2664 active_op_count,
2665 } = effect
2666 else {
2667 continue;
2668 };
2669 if admission.is_some() {
2670 return Err(OpsLifecycleError::Internal(
2671 "generated op registration authority emitted multiple admission results".into(),
2672 ));
2673 }
2674 if operation_id != &expected_id {
2675 return Err(OpsLifecycleError::Internal(format!(
2676 "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2677 )));
2678 }
2679 admission = Some(match result {
2680 mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2681 if reject_reason.is_some() {
2682 return Err(OpsLifecycleError::Internal(
2683 "generated op registration authority accepted with rejection reason".into(),
2684 ));
2685 }
2686 None
2687 }
2688 mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2689 let reason = reject_reason.ok_or_else(|| {
2690 OpsLifecycleError::Internal(
2691 "generated op registration authority rejected without reason".into(),
2692 )
2693 })?;
2694 let error = match reason {
2695 mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2696 OpsLifecycleError::AlreadyRegistered(id.clone())
2697 }
2698 mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2699 let limit = max_concurrent_limit.ok_or_else(|| {
2700 OpsLifecycleError::Internal(
2701 "generated op registration authority rejected capacity without limit"
2702 .into(),
2703 )
2704 })?;
2705 OpsLifecycleError::MaxConcurrentExceeded {
2706 limit: limit as usize,
2707 active: *active_op_count as usize,
2708 }
2709 }
2710 };
2711 Some(error)
2712 }
2713 });
2714 }
2715 admission.ok_or_else(|| {
2716 OpsLifecycleError::Internal(
2717 "generated op registration authority emitted no admission result".into(),
2718 )
2719 })
2720}
2721
2722impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2723 fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2724 self.register_operation_with_admission_limit(spec, None)
2725 }
2726
2727 fn register_operation_with_admission_limit(
2728 &self,
2729 mut spec: OperationSpec,
2730 max_concurrent: Option<usize>,
2731 ) -> Result<(), OpsLifecycleError> {
2732 let mut state = self.write_state()?;
2733 let operation_id = spec.id.clone();
2734 let kind = spec.kind;
2735 let max_concurrent = max_concurrent
2736 .or(state.max_concurrent)
2737 .map(|limit| limit as u64);
2738
2739 let effects = state.dsl_apply_with_effects(
2740 mm_dsl::MeerkatMachineInput::RegisterOp {
2741 operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2742 kind: mm_dsl::OperationKind::from_domain(&kind),
2743 source: spec
2744 .operation_source
2745 .as_ref()
2746 .map(mm_dsl::OperationSource::from_domain),
2747 max_concurrent,
2748 },
2749 "RegisterOp",
2750 )?;
2751 if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
2752 return Err(error);
2753 }
2754
2755 let authority_operation_source = state.operation_source(&operation_id)?;
2756 ShellState::align_spec_child_session_id_to_source(
2757 &mut spec,
2758 authority_operation_source.as_ref(),
2759 );
2760
2761 state.records.insert(operation_id, ShellRecord::new(spec));
2763 Ok(())
2764 }
2765
2766 fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2767 let mut state = self.write_state()?;
2768
2769 apply_op_transition(
2770 &mut state,
2771 id,
2772 mm_dsl::MeerkatMachineInput::StartOp {
2773 operation_id: mm_dsl::OperationId::from_domain(id).0,
2774 },
2775 mm_dsl::OpLifecycleActionKind::Start,
2776 )?;
2777
2778 if let Some(shell) = state.records.get_mut(id) {
2780 shell.started_at = Some(Instant::now());
2781 }
2782 Ok(())
2783 }
2784
2785 fn provisioning_failed(
2786 &self,
2787 id: &OperationId,
2788 error: String,
2789 ) -> Result<(), OpsLifecycleError> {
2790 let mut state = self.write_state()?;
2791
2792 let terminal_outcome = OperationTerminalOutcome::Failed { error };
2793 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2794
2795 apply_terminal_op_transition_and_persist(
2796 &mut state,
2797 id,
2798 mm_dsl::MeerkatMachineInput::FailOp {
2799 operation_id: mm_dsl::OperationId::from_domain(id).0,
2800 outcome: outcome_kind,
2801 payload: terminal_outcome,
2802 },
2803 mm_dsl::OpLifecycleActionKind::Fail,
2804 )?;
2805
2806 let completion_entry = state.finalize_terminal(id)?;
2807 state.publish_completion_entry(completion_entry);
2808 Ok(())
2809 }
2810
2811 fn peer_ready(
2812 &self,
2813 id: &OperationId,
2814 peer: OperationPeerHandle,
2815 ) -> Result<(), OpsLifecycleError> {
2816 let mut state = self.write_state()?;
2817
2818 apply_op_transition(
2819 &mut state,
2820 id,
2821 mm_dsl::MeerkatMachineInput::PeerReadyOp {
2822 operation_id: mm_dsl::OperationId::from_domain(id).0,
2823 },
2824 mm_dsl::OpLifecycleActionKind::PeerReady,
2825 )?;
2826
2827 if let Some(shell) = state.records.get_mut(id) {
2829 shell.peer_handle = Some(peer);
2830 }
2831 Ok(())
2832 }
2833
2834 fn register_watcher(
2835 &self,
2836 id: &OperationId,
2837 ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
2838 let mut state = self.write_state()?;
2839
2840 if !state.contains(id) {
2841 return Err(OpsLifecycleError::NotFound(id.clone()));
2842 }
2843
2844 if let Some(outcome) = state.terminal_outcome(id)? {
2846 return Ok(resolved_operation_completion_watch(outcome));
2847 }
2848
2849 let shell = state.shell_record_mut(id)?;
2851 let (tx, rx) = tokio::sync::oneshot::channel();
2852 let watch = operation_completion_watch_from_receiver(rx);
2853 shell.watchers.push(OperationCompletionNotifier::new(tx));
2854 Ok(watch)
2855 }
2856
2857 fn report_progress(
2858 &self,
2859 id: &OperationId,
2860 _update: OperationProgressUpdate,
2861 ) -> Result<(), OpsLifecycleError> {
2862 let mut state = self.write_state()?;
2863
2864 apply_op_transition(
2865 &mut state,
2866 id,
2867 mm_dsl::MeerkatMachineInput::ProgressReportedOp {
2868 operation_id: mm_dsl::OperationId::from_domain(id).0,
2869 },
2870 mm_dsl::OpLifecycleActionKind::ProgressReported,
2871 )?;
2872 Ok(())
2873 }
2874
2875 fn complete_operation(
2876 &self,
2877 id: &OperationId,
2878 result: OperationResult,
2879 ) -> Result<(), OpsLifecycleError> {
2880 let mut state = self.write_state()?;
2881
2882 let terminal_outcome = OperationTerminalOutcome::Completed(result);
2883 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2884
2885 apply_terminal_op_transition_and_persist(
2886 &mut state,
2887 id,
2888 mm_dsl::MeerkatMachineInput::CompleteOp {
2889 operation_id: mm_dsl::OperationId::from_domain(id).0,
2890 outcome: outcome_kind,
2891 payload: terminal_outcome,
2892 },
2893 mm_dsl::OpLifecycleActionKind::Complete,
2894 )?;
2895
2896 let completion_entry = state.finalize_terminal(id)?;
2897 state.publish_completion_entry(completion_entry);
2898 Ok(())
2899 }
2900
2901 fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
2902 let mut state = self.write_state()?;
2903
2904 let terminal_outcome = OperationTerminalOutcome::Failed { error };
2905 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2906
2907 apply_terminal_op_transition_and_persist(
2908 &mut state,
2909 id,
2910 mm_dsl::MeerkatMachineInput::FailOp {
2911 operation_id: mm_dsl::OperationId::from_domain(id).0,
2912 outcome: outcome_kind,
2913 payload: terminal_outcome,
2914 },
2915 mm_dsl::OpLifecycleActionKind::Fail,
2916 )?;
2917
2918 let completion_entry = state.finalize_terminal(id)?;
2919 state.publish_completion_entry(completion_entry);
2920 Ok(())
2921 }
2922
2923 fn abort_provisioning(
2924 &self,
2925 id: &OperationId,
2926 reason: Option<String>,
2927 ) -> Result<(), OpsLifecycleError> {
2928 let mut state = self.write_state()?;
2929
2930 let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
2931 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2932
2933 apply_terminal_op_transition_and_persist(
2934 &mut state,
2935 id,
2936 mm_dsl::MeerkatMachineInput::AbortOp {
2937 operation_id: mm_dsl::OperationId::from_domain(id).0,
2938 outcome: outcome_kind,
2939 payload: terminal_outcome,
2940 },
2941 mm_dsl::OpLifecycleActionKind::Abort,
2942 )?;
2943
2944 let completion_entry = state.finalize_terminal(id)?;
2945 state.publish_completion_entry(completion_entry);
2946 Ok(())
2947 }
2948
2949 fn cancel_operation(
2950 &self,
2951 id: &OperationId,
2952 reason: Option<String>,
2953 ) -> Result<(), OpsLifecycleError> {
2954 let mut state = self.write_state()?;
2955
2956 let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
2957 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2958
2959 apply_terminal_op_transition_and_persist(
2960 &mut state,
2961 id,
2962 mm_dsl::MeerkatMachineInput::CancelOp {
2963 operation_id: mm_dsl::OperationId::from_domain(id).0,
2964 outcome: outcome_kind,
2965 payload: terminal_outcome,
2966 },
2967 mm_dsl::OpLifecycleActionKind::Cancel,
2968 )?;
2969
2970 let completion_entry = state.finalize_terminal(id)?;
2971 state.publish_completion_entry(completion_entry);
2972 Ok(())
2973 }
2974
2975 fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2976 let mut state = self.write_state()?;
2977
2978 apply_op_transition(
2979 &mut state,
2980 id,
2981 mm_dsl::MeerkatMachineInput::RetireRequestedOp {
2982 operation_id: mm_dsl::OperationId::from_domain(id).0,
2983 },
2984 mm_dsl::OpLifecycleActionKind::RetireRequested,
2985 )?;
2986 Ok(())
2987 }
2988
2989 fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2990 let mut state = self.write_state()?;
2991
2992 let terminal_outcome = OperationTerminalOutcome::Retired;
2993 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2994
2995 apply_terminal_op_transition_and_persist(
2996 &mut state,
2997 id,
2998 mm_dsl::MeerkatMachineInput::RetireCompletedOp {
2999 operation_id: mm_dsl::OperationId::from_domain(id).0,
3000 outcome: outcome_kind,
3001 payload: terminal_outcome,
3002 },
3003 mm_dsl::OpLifecycleActionKind::RetireCompleted,
3004 )?;
3005
3006 let completion_entry = state.finalize_terminal(id)?;
3007 state.publish_completion_entry(completion_entry);
3008 Ok(())
3009 }
3010
3011 fn snapshot(
3012 &self,
3013 id: &OperationId,
3014 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3015 let state = self.read_state()?;
3016 state.snapshot(id)
3017 }
3018
3019 fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3020 let state = self.read_state()?;
3021 let mut snapshots = Vec::new();
3022 for id in state.operation_ids()? {
3023 let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3024 OpsLifecycleError::Internal(format!(
3025 "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3026 ))
3027 })?;
3028 snapshots.push(snapshot);
3029 }
3030 snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3031 Ok(snapshots)
3032 }
3033
3034 fn classify_operation_terminality(
3035 &self,
3036 id: &OperationId,
3037 status: OperationStatus,
3038 ) -> Result<bool, OpsLifecycleError> {
3039 ShellState::operation_status_is_terminal(id, status)
3040 }
3041
3042 fn classify_operation_public_result(
3043 &self,
3044 id: &OperationId,
3045 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3046 let state = self.read_state()?;
3047 let status = match state.status(id) {
3048 Some(status) => status,
3049 None if state.records.contains_key(id)
3050 || state.has_generated_operation_record_fact(id) =>
3051 {
3052 return Err(OpsLifecycleError::Internal(format!(
3053 "generated op lifecycle authority missing status for {id}"
3054 )));
3055 }
3056 None => OperationStatus::Absent,
3057 };
3058 ShellState::operation_public_result_class(id, status)
3059 }
3060
3061 fn classify_operation_completion_wake(
3062 &self,
3063 id: &OperationId,
3064 kind: OperationKind,
3065 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3066 ShellState::operation_completion_wake_class(id, kind)
3067 }
3068
3069 fn classify_operation_transition_idempotence(
3070 &self,
3071 id: &OperationId,
3072 action: OperationLifecycleAction,
3073 ) -> Result<bool, OpsLifecycleError> {
3074 let state = self.read_state()?;
3075 let status = match state.status(id) {
3076 Some(status) => status,
3077 None if state.records.contains_key(id)
3078 || state.has_generated_operation_record_fact(id) =>
3079 {
3080 return Err(OpsLifecycleError::Internal(format!(
3081 "generated op lifecycle authority missing status for {id}"
3082 )));
3083 }
3084 None => OperationStatus::Absent,
3085 };
3086 ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3087 }
3088
3089 fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3090 let mut state = self.write_state()?;
3091
3092 let to_terminate = state.owner_termination_targets()?;
3093
3094 for (op_id, _status) in &to_terminate {
3095 let terminal_outcome = OperationTerminalOutcome::Terminated {
3096 reason: reason.clone(),
3097 };
3098 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3099
3100 apply_terminal_op_transition_and_persist(
3101 &mut state,
3102 op_id,
3103 mm_dsl::MeerkatMachineInput::TerminateOp {
3104 operation_id: mm_dsl::OperationId::from_domain(op_id).0,
3105 outcome: outcome_kind,
3106 payload: terminal_outcome,
3107 },
3108 mm_dsl::OpLifecycleActionKind::Terminate,
3109 )?;
3110
3111 if let Some(entry) = state.finalize_terminal(op_id)? {
3112 state.publish_completion_entry(Some(entry));
3113 }
3114 }
3115 Ok(())
3116 }
3117
3118 fn collect_completed(
3119 &self,
3120 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3121 let mut state = self.write_state()?;
3122
3123 let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3124 let mut collected = Vec::with_capacity(ids.len());
3125 for id in ids {
3126 let outcome = state.terminal_outcome(&id)?;
3127 state.dsl_apply(
3128 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3129 operation_id: mm_dsl::OperationId::from_domain(&id).0,
3130 },
3131 "CollectCompletedOp",
3132 )?;
3133 state.completed_order.retain(|queued| queued != &id);
3134 state.records.remove(&id);
3135 if let Some(outcome) = outcome {
3136 collected.push((id, outcome));
3137 }
3138 }
3139 Ok(collected)
3140 }
3141
3142 fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3143 Some(self.completion_feed_handle())
3144 }
3145
3146 fn completion_cursor(
3147 &self,
3148 consumer: CompletionCursorConsumer,
3149 ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3150 let state = self.read_state()?;
3151 Ok(Some(state.completion_cursor(consumer)))
3152 }
3153
3154 fn advance_completion_cursor(
3155 &self,
3156 consumer: CompletionCursorConsumer,
3157 cursor: CompletionSeq,
3158 projection: Option<&meerkat_core::EpochCursorState>,
3159 ) -> Result<CompletionSeq, OpsLifecycleError> {
3160 let mut state = self.write_state()?;
3161 let input = match consumer {
3162 CompletionCursorConsumer::AgentApplied => {
3163 mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3164 }
3165 CompletionCursorConsumer::RuntimeObserved => {
3166 mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3167 }
3168 CompletionCursorConsumer::RuntimeInjected => {
3169 mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3170 }
3171 };
3172 let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3173 let advanced = effects
3174 .iter()
3175 .find_map(|effect| match (consumer, effect) {
3176 (
3177 CompletionCursorConsumer::AgentApplied,
3178 mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3179 ) => Some(*cursor),
3180 (
3181 CompletionCursorConsumer::RuntimeObserved,
3182 mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3183 cursor,
3184 },
3185 ) => Some(*cursor),
3186 (
3187 CompletionCursorConsumer::RuntimeInjected,
3188 mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3189 cursor,
3190 },
3191 ) => Some(*cursor),
3192 _ => None,
3193 })
3194 .ok_or_else(|| {
3195 OpsLifecycleError::Internal(format!(
3196 "generated completion cursor transition emitted no feedback for {consumer:?}"
3197 ))
3198 })?;
3199 if let Some(projection) = projection {
3200 projection.project_authorized_completion_cursor(consumer, advanced);
3201 }
3202 Ok(advanced)
3203 }
3204
3205 fn wait_all(
3206 &self,
3207 run_id: &RunId,
3208 ids: &[OperationId],
3209 ) -> std::pin::Pin<
3210 Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3211 > {
3212 let wait_request_id = WaitRequestId::new();
3213 let owned_ids = ids.to_vec();
3214
3215 let state = match self.write_state() {
3216 Ok(mut state) => {
3217 match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3218 Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3219 let outcomes =
3220 state
3221 .collect_wait_outcomes(&satisfied.operation_ids)
3222 .map(|outcomes| WaitAllResult {
3223 outcomes,
3224 satisfied,
3225 });
3226 WaitAllFutureState::Ready(Some(outcomes))
3227 }
3228 Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3229 if state.pending_wait.is_some() {
3230 let rollback = state.dsl_apply(
3236 mm_dsl::MeerkatMachineInput::CancelWaitAll,
3237 "CancelWaitAll(rollback)",
3238 );
3239 return Box::pin(WaitAllFuture {
3240 registry: self,
3241 wait_request_id,
3242 state: WaitAllFutureState::Ready(Some(Err(match rollback {
3243 Ok(()) => OpsLifecycleError::Internal(
3244 "wait_all started while a pending wait sender already existed"
3245 .into(),
3246 ),
3247 Err(err) => err,
3248 }))),
3249 });
3250 }
3251 state.wait_request_id = Some(wait_request_id.clone());
3252 let (sender, receiver) = tokio::sync::oneshot::channel();
3253 state.pending_wait = Some(PendingWaitState {
3254 wait_request_id: wait_request_id.clone(),
3255 sender,
3256 });
3257 WaitAllFutureState::Waiting(receiver)
3258 }
3259 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3260 }
3261 }
3262 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3263 };
3264
3265 Box::pin(WaitAllFuture {
3266 registry: self,
3267 wait_request_id,
3268 state,
3269 })
3270 }
3271}
3272
3273#[cfg(test)]
3274#[allow(clippy::unwrap_used, clippy::panic)]
3275mod tests {
3276 use super::*;
3277 use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3278 use meerkat_core::lifecycle::RunId;
3279 use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3280 use meerkat_core::types::SessionId;
3281 use std::sync::atomic::Ordering;
3282 use uuid::Uuid;
3283
3284 fn test_run_id() -> RunId {
3285 RunId(Uuid::from_u128(1))
3286 }
3287
3288 fn background_spec(name: &str) -> OperationSpec {
3289 OperationSpec {
3290 id: OperationId::new(),
3291 kind: OperationKind::BackgroundToolOp,
3292 owner_session_id: SessionId::new(),
3293 display_name: name.into(),
3294 source_label: "test".into(),
3295 operation_source: None,
3296 child_session_id: None,
3297 expect_peer_channel: false,
3298 }
3299 }
3300
3301 #[tokio::test]
3302 async fn late_watchers_resolve_immediately() {
3303 let registry = RuntimeOpsLifecycleRegistry::new();
3304 let spec = background_spec("late");
3305 let op_id = spec.id.clone();
3306 registry.register_operation(spec).unwrap();
3307 registry.provisioning_succeeded(&op_id).unwrap();
3308 registry
3309 .complete_operation(
3310 &op_id,
3311 OperationResult {
3312 id: op_id.clone(),
3313 content: "done".into(),
3314 is_error: false,
3315 duration_ms: 1,
3316 tokens_used: 0,
3317 },
3318 )
3319 .unwrap();
3320
3321 let watch = registry.register_watcher(&op_id).unwrap();
3322 match watch
3323 .await
3324 .expect("operation completion watch should resolve")
3325 {
3326 OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3327 other => panic!("expected completed outcome, got {other:?}"),
3328 }
3329 }
3330
3331 #[tokio::test]
3332 async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3333 let (tx, rx) = tokio::sync::oneshot::channel();
3334 let watch = operation_completion_watch_from_receiver(rx);
3335 drop(tx);
3336
3337 assert_eq!(
3338 watch.await,
3339 Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3340 );
3341 }
3342
3343 #[test]
3344 fn peer_ready_requires_peer_expectation() {
3345 let registry = RuntimeOpsLifecycleRegistry::new();
3346 let spec = background_spec("no-peer");
3347 let op_id = spec.id.clone();
3348 registry.register_operation(spec).unwrap();
3349 registry.provisioning_succeeded(&op_id).unwrap();
3350
3351 let result = registry.peer_ready(
3352 &op_id,
3353 OperationPeerHandle {
3354 peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3355 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3356 "peer",
3357 PeerId::new(),
3358 "inproc://peer",
3359 )
3360 .unwrap(),
3361 },
3362 );
3363 assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3364 }
3365
3366 #[test]
3372 fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3373 let op_id = OperationId::new();
3374 let outcomes = vec![
3375 (
3376 OperationTerminalOutcome::Completed(OperationResult {
3377 id: op_id.clone(),
3378 content: "done".into(),
3379 is_error: false,
3380 duration_ms: 7,
3381 tokens_used: 42,
3382 }),
3383 mm_dsl::OperationTerminalOutcomeKind::Completed,
3384 ),
3385 (
3386 OperationTerminalOutcome::Failed {
3387 error: "boom".into(),
3388 },
3389 mm_dsl::OperationTerminalOutcomeKind::Failed,
3390 ),
3391 (
3392 OperationTerminalOutcome::Aborted {
3393 reason: Some("user aborted".into()),
3394 },
3395 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3396 ),
3397 (
3398 OperationTerminalOutcome::Aborted { reason: None },
3399 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3400 ),
3401 (
3402 OperationTerminalOutcome::Cancelled {
3403 reason: Some("cancelled".into()),
3404 },
3405 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3406 ),
3407 (
3408 OperationTerminalOutcome::Cancelled { reason: None },
3409 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3410 ),
3411 (
3412 OperationTerminalOutcome::Retired,
3413 mm_dsl::OperationTerminalOutcomeKind::Retired,
3414 ),
3415 (
3416 OperationTerminalOutcome::Terminated {
3417 reason: "owner stopped".into(),
3418 },
3419 mm_dsl::OperationTerminalOutcomeKind::Terminated,
3420 ),
3421 ];
3422
3423 for (outcome, expected_kind) in &outcomes {
3424 assert_eq!(
3425 mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3426 *expected_kind,
3427 "typed payload {outcome:?} must classify to {expected_kind:?}"
3428 );
3429 let read = ShellState::checked_terminal_payload(
3430 *expected_kind,
3431 outcome,
3432 "test authority",
3433 "test-op",
3434 )
3435 .expect("matching discriminant must read back the exact payload");
3436 assert_eq!(&read, outcome);
3437 }
3438
3439 let err = ShellState::checked_terminal_payload(
3441 mm_dsl::OperationTerminalOutcomeKind::Completed,
3442 &OperationTerminalOutcome::Retired,
3443 "test authority",
3444 "test-op",
3445 )
3446 .expect_err("variant mismatch must be rejected");
3447 assert!(matches!(err, OpsLifecycleError::Internal(_)));
3448 }
3449
3450 #[test]
3456 fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3457 let registry = RuntimeOpsLifecycleRegistry::new();
3458 let spec = background_spec("variant-mismatch");
3459 let op_id = spec.id.clone();
3460 registry.register_operation(spec).unwrap();
3461 registry.provisioning_succeeded(&op_id).unwrap();
3462
3463 let mut state = registry.write_state().unwrap();
3464 let err = state
3465 .dsl_apply(
3466 mm_dsl::MeerkatMachineInput::CompleteOp {
3467 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3468 outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3469 payload: OperationTerminalOutcome::Retired,
3471 },
3472 "CompleteOp",
3473 )
3474 .expect_err("machine must reject payload variant mismatch");
3475 assert!(
3482 matches!(
3483 &err,
3484 OpsLifecycleError::Internal(message)
3485 if message.contains("GuardRejected") && message.contains("CompleteOp")
3486 ),
3487 "expected generated guard rejection for CompleteOp, got: {err:?}"
3488 );
3489 drop(state);
3490
3491 registry
3494 .complete_operation(
3495 &op_id,
3496 OperationResult {
3497 id: op_id.clone(),
3498 content: "done".into(),
3499 is_error: false,
3500 duration_ms: 1,
3501 tokens_used: 0,
3502 },
3503 )
3504 .expect("matching variant must complete");
3505 }
3506
3507 #[test]
3508 fn duplicate_registration_rejection_is_generated() {
3509 let registry = RuntimeOpsLifecycleRegistry::new();
3510 let spec = background_spec("duplicate");
3511 let op_id = spec.id.clone();
3512
3513 registry.register_operation(spec.clone()).unwrap();
3514 let result = registry.register_operation(spec);
3515
3516 assert!(matches!(
3517 result,
3518 Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3519 ));
3520 }
3521
3522 #[test]
3523 fn invalid_transition_rejection_is_generated() {
3524 let registry = RuntimeOpsLifecycleRegistry::new();
3525 let spec = background_spec("invalid-transition");
3526 let op_id = spec.id.clone();
3527 registry.register_operation(spec).unwrap();
3528
3529 let result = registry.complete_operation(
3530 &op_id,
3531 OperationResult {
3532 id: op_id.clone(),
3533 content: "too-early".into(),
3534 is_error: false,
3535 duration_ms: 1,
3536 tokens_used: 0,
3537 },
3538 );
3539
3540 assert!(matches!(
3541 result,
3542 Err(OpsLifecycleError::InvalidTransition {
3543 id,
3544 status: OperationStatus::Provisioning,
3545 action: "complete_operation",
3546 }) if id == op_id
3547 ));
3548 }
3549
3550 #[tokio::test]
3551 async fn multi_listener_completion() {
3552 let registry = RuntimeOpsLifecycleRegistry::new();
3553 let spec = background_spec("multi");
3554 let op_id = spec.id.clone();
3555 registry.register_operation(spec).unwrap();
3556 registry.provisioning_succeeded(&op_id).unwrap();
3557
3558 let watch1 = registry.register_watcher(&op_id).unwrap();
3559 let watch2 = registry.register_watcher(&op_id).unwrap();
3560 let watch3 = registry.register_watcher(&op_id).unwrap();
3561
3562 registry
3563 .complete_operation(
3564 &op_id,
3565 OperationResult {
3566 id: op_id.clone(),
3567 content: "multi-done".into(),
3568 is_error: false,
3569 duration_ms: 1,
3570 tokens_used: 0,
3571 },
3572 )
3573 .unwrap();
3574
3575 for watch in [watch1, watch2, watch3] {
3576 match watch
3577 .await
3578 .expect("operation completion watch should resolve")
3579 {
3580 OperationTerminalOutcome::Completed(result) => {
3581 assert_eq!(result.content, "multi-done");
3582 }
3583 other => panic!("expected completed, got {other:?}"),
3584 }
3585 }
3586 }
3587
3588 #[tokio::test]
3589 async fn wait_all_returns_all_outcomes() {
3590 let registry = RuntimeOpsLifecycleRegistry::new();
3591
3592 let spec_a = background_spec("a");
3593 let id_a = spec_a.id.clone();
3594 registry.register_operation(spec_a).unwrap();
3595 registry.provisioning_succeeded(&id_a).unwrap();
3596
3597 let spec_b = background_spec("b");
3598 let id_b = spec_b.id.clone();
3599 registry.register_operation(spec_b).unwrap();
3600 registry.provisioning_succeeded(&id_b).unwrap();
3601
3602 registry
3603 .complete_operation(
3604 &id_a,
3605 OperationResult {
3606 id: id_a.clone(),
3607 content: "a-done".into(),
3608 is_error: false,
3609 duration_ms: 1,
3610 tokens_used: 0,
3611 },
3612 )
3613 .unwrap();
3614 registry.fail_operation(&id_b, "b-error".into()).unwrap();
3615
3616 let wait_result = registry
3617 .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3618 .await
3619 .unwrap();
3620 assert_eq!(wait_result.outcomes.len(), 2);
3621 assert_eq!(wait_result.outcomes[0].0, id_a);
3622 assert!(matches!(
3623 wait_result.outcomes[0].1,
3624 OperationTerminalOutcome::Completed(_)
3625 ));
3626 assert_eq!(wait_result.outcomes[1].0, id_b);
3627 assert!(matches!(
3628 wait_result.outcomes[1].1,
3629 OperationTerminalOutcome::Failed { .. }
3630 ));
3631 assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3633 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3634 }
3635
3636 #[tokio::test]
3639 async fn wait_all_trait_path_submits_through_authority() {
3640 let registry = RuntimeOpsLifecycleRegistry::new();
3641 let spec = background_spec("trait-wait");
3642 let op_id = spec.id.clone();
3643 registry.register_operation(spec).unwrap();
3644 registry.provisioning_succeeded(&op_id).unwrap();
3645 registry
3646 .complete_operation(
3647 &op_id,
3648 OperationResult {
3649 id: op_id.clone(),
3650 content: "done".into(),
3651 is_error: false,
3652 duration_ms: 1,
3653 tokens_used: 0,
3654 },
3655 )
3656 .unwrap();
3657
3658 let trait_ref: &dyn OpsLifecycleRegistry = ®istry;
3660 let wait_result = trait_ref
3661 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3662 .await
3663 .unwrap();
3664 assert_eq!(wait_result.outcomes.len(), 1);
3665 assert!(matches!(
3666 wait_result.outcomes[0].1,
3667 OperationTerminalOutcome::Completed(_)
3668 ));
3669 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3671 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3672 let state = registry.read_state().unwrap();
3673 assert!(
3674 !state.wait_active(),
3675 "already-satisfied wait_all must be cleared by generated satisfaction authority"
3676 );
3677 assert!(state.wait_operation_ids().unwrap().is_empty());
3678 }
3679
3680 #[tokio::test]
3681 async fn wait_all_duplicate_rejection_is_generated() {
3682 let registry = RuntimeOpsLifecycleRegistry::new();
3683 let spec = background_spec("duplicate-wait");
3684 let op_id = spec.id.clone();
3685 registry.register_operation(spec).unwrap();
3686 registry.provisioning_succeeded(&op_id).unwrap();
3687
3688 let result = registry
3689 .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3690 .await;
3691
3692 assert!(matches!(
3693 result,
3694 Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3695 ));
3696 let state = registry.read_state().unwrap();
3697 assert!(
3698 !state.wait_active(),
3699 "duplicate wait rejection must not create a shell or machine barrier"
3700 );
3701 assert!(state.wait_operation_ids().unwrap().is_empty());
3702 }
3703
3704 #[tokio::test]
3705 async fn wait_all_active_rejection_is_generated() {
3706 let registry = RuntimeOpsLifecycleRegistry::new();
3707 let spec = background_spec("active-wait");
3708 let op_id = spec.id.clone();
3709 registry.register_operation(spec).unwrap();
3710 registry.provisioning_succeeded(&op_id).unwrap();
3711
3712 let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3713 let result = registry
3714 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3715 .await;
3716
3717 assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3718 drop(active_wait);
3719 let state = registry.read_state().unwrap();
3720 assert!(!state.wait_active());
3721 assert!(state.wait_operation_ids().unwrap().is_empty());
3722 }
3723
3724 #[tokio::test]
3725 async fn wait_all_unknown_operation_rejection_is_generated() {
3726 let registry = RuntimeOpsLifecycleRegistry::new();
3727 let op_id = OperationId::new();
3728
3729 let result = registry
3730 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3731 .await;
3732
3733 assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3734 let state = registry.read_state().unwrap();
3735 assert!(!state.wait_active());
3736 assert!(state.wait_operation_ids().unwrap().is_empty());
3737 }
3738
3739 #[tokio::test]
3740 async fn wait_all_resolves_from_authority_owned_wait_request() {
3741 let registry = RuntimeOpsLifecycleRegistry::new();
3742 let run_id = test_run_id();
3743
3744 let spec = background_spec("pending");
3745 let op_id = spec.id.clone();
3746 registry.register_operation(spec).unwrap();
3747 registry.provisioning_succeeded(&op_id).unwrap();
3748
3749 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3750 tokio::pin!(wait_fut);
3751 assert!(
3752 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3753 .await
3754 .is_err()
3755 );
3756
3757 let active_wait_request_id = {
3758 let state = registry.read_state().unwrap();
3759 let wait_request_id = match state.wait_request_id.clone() {
3760 Some(wait_request_id) => wait_request_id,
3761 None => panic!("wait request should be active"),
3762 };
3763 assert_eq!(
3764 state.wait_operation_ids().unwrap().as_slice(),
3765 std::slice::from_ref(&op_id)
3766 );
3767 wait_request_id
3768 };
3769
3770 registry
3771 .complete_operation(
3772 &op_id,
3773 OperationResult {
3774 id: op_id.clone(),
3775 content: "done".into(),
3776 is_error: false,
3777 duration_ms: 1,
3778 tokens_used: 0,
3779 },
3780 )
3781 .unwrap();
3782
3783 let wait_result = wait_fut.await.unwrap();
3784 assert_eq!(
3785 wait_result.satisfied.wait_request_id,
3786 active_wait_request_id
3787 );
3788 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
3789 assert!(matches!(
3790 wait_result.outcomes.as_slice(),
3791 [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
3792 ));
3793 assert!(registry.read_state().unwrap().wait_request_id.is_none());
3794 }
3795
3796 #[tokio::test]
3797 async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
3798 let registry = RuntimeOpsLifecycleRegistry::new();
3799 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3800 registry.set_persistence_channel(
3801 tx,
3802 meerkat_core::RuntimeEpochId::new(),
3803 Arc::new(meerkat_core::EpochCursorState::new()),
3804 );
3805 let worker = std::thread::spawn(move || {
3806 let request = rx.blocking_recv().expect("persistence request");
3807 let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
3808 "injected persistence failure".to_string(),
3809 )));
3810 });
3811
3812 let spec = background_spec("feed-after-persist");
3813 let op_id = spec.id.clone();
3814 registry.register_operation(spec).unwrap();
3815 registry.provisioning_succeeded(&op_id).unwrap();
3816 let watch = registry.register_watcher(&op_id).unwrap();
3817
3818 let err = registry
3819 .complete_operation(
3820 &op_id,
3821 OperationResult {
3822 id: op_id.clone(),
3823 content: "done".into(),
3824 is_error: false,
3825 duration_ms: 1,
3826 tokens_used: 0,
3827 },
3828 )
3829 .expect_err("persistence failure must fail the terminal transition");
3830 assert!(
3831 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
3832 "unexpected persistence error: {err:?}"
3833 );
3834 worker.join().expect("persistence worker");
3835
3836 let feed = registry.completion_feed_handle();
3837 let batch = feed.list_since(0);
3838 assert!(
3839 batch.entries.is_empty(),
3840 "completion feed must not publish entries before durable snapshot success"
3841 );
3842 assert_eq!(batch.watermark, 0);
3843 assert!(
3844 tokio::time::timeout(std::time::Duration::from_millis(10), watch)
3845 .await
3846 .is_err(),
3847 "watchers must not resolve before durable terminal snapshot success"
3848 );
3849 assert_eq!(
3850 registry.snapshot(&op_id).unwrap().unwrap().status,
3851 OperationStatus::Running,
3852 "failed persistence must roll the generated terminal transition back"
3853 );
3854
3855 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3856 registry.set_persistence_channel(
3857 tx,
3858 meerkat_core::RuntimeEpochId::new(),
3859 Arc::new(meerkat_core::EpochCursorState::new()),
3860 );
3861 let worker = std::thread::spawn(move || {
3862 let request = rx.blocking_recv().expect("second persistence request");
3863 let _ = request.result_tx.send(Ok(()));
3864 });
3865 registry
3866 .complete_operation(
3867 &op_id,
3868 OperationResult {
3869 id: op_id.clone(),
3870 content: "done".into(),
3871 is_error: false,
3872 duration_ms: 1,
3873 tokens_used: 0,
3874 },
3875 )
3876 .expect("rolled-back terminal transition should remain retryable");
3877 worker.join().expect("second persistence worker");
3878
3879 let batch = feed.list_since(0);
3880 assert_eq!(batch.entries.len(), 1);
3881 assert_eq!(batch.entries[0].operation_id, op_id);
3882 }
3883
3884 #[tokio::test]
3885 async fn dropping_wait_all_future_cancels_active_wait_request() {
3886 let registry = RuntimeOpsLifecycleRegistry::new();
3887 let run_id = test_run_id();
3888
3889 let spec = background_spec("cancelled-wait");
3890 let op_id = spec.id.clone();
3891 registry.register_operation(spec).unwrap();
3892 registry.provisioning_succeeded(&op_id).unwrap();
3893
3894 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3895 drop(wait_fut);
3896
3897 let state = registry.read_state().unwrap();
3898 assert!(state.wait_request_id.is_none());
3899 assert!(state.wait_operation_ids().unwrap().is_empty());
3900 assert!(!state.wait_active());
3901 }
3902
3903 #[tokio::test]
3909 async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
3910 let registry = RuntimeOpsLifecycleRegistry::new();
3911 let run_id = test_run_id();
3912
3913 let spec = background_spec("corrupt-barrier");
3914 let op_id = spec.id.clone();
3915 registry.register_operation(spec).unwrap();
3916 registry.provisioning_succeeded(&op_id).unwrap();
3917
3918 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3920 tokio::pin!(wait_fut);
3921 assert!(
3922 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3923 .await
3924 .is_err(),
3925 "barrier waiter must still be pending before corruption"
3926 );
3927 assert!(registry.read_state().unwrap().wait_request_id.is_some());
3928
3929 {
3933 let mut state = registry.write_state().unwrap();
3934 let mut machine_state = state.dsl.0.state().clone();
3935 machine_state.wait_run_id = None;
3936 state.dsl = DslAuthority(Box::new(
3937 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
3938 ));
3939 }
3940
3941 let err = registry
3944 .complete_operation(
3945 &op_id,
3946 OperationResult {
3947 id: op_id.clone(),
3948 content: "done".into(),
3949 is_error: false,
3950 duration_ms: 1,
3951 tokens_used: 0,
3952 },
3953 )
3954 .expect_err("corrupt wait authority must fail the terminal transition");
3955 assert!(
3956 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
3957 "unexpected terminal error: {err:?}"
3958 );
3959
3960 let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
3962 .await
3963 .expect("waiter must resolve, not hang, after authority corruption");
3964 match waiter_result {
3965 Err(OpsLifecycleError::Internal(message)) => assert!(
3966 message.contains("wait_all completion channel dropped"),
3967 "unexpected waiter error message: {message}"
3968 ),
3969 other => panic!("expected dropped-channel Internal error, got {other:?}"),
3970 }
3971 }
3972
3973 #[test]
3977 fn completion_cursor_propagates_poison_not_none() {
3978 let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
3979
3980 let poison_registry = std::sync::Arc::clone(®istry);
3982 let join = std::thread::spawn(move || {
3983 let _guard = poison_registry.write_state().unwrap();
3984 panic!("intentional panic to poison ops lifecycle registry lock");
3985 });
3986 assert!(
3987 join.join().is_err(),
3988 "poisoning thread must have panicked while holding the write guard"
3989 );
3990
3991 let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
3992 let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
3993 match result {
3994 Err(OpsLifecycleError::Internal(message)) => assert!(
3995 message.contains("ops lifecycle registry poisoned"),
3996 "unexpected cursor error message: {message}"
3997 ),
3998 other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
3999 }
4000 }
4001
4002 #[test]
4003 fn terminate_owner_only_targets_non_terminal_operations() {
4004 let registry = RuntimeOpsLifecycleRegistry::new();
4005
4006 let running_spec = background_spec("running");
4007 let running_id = running_spec.id.clone();
4008 registry.register_operation(running_spec).unwrap();
4009 registry.provisioning_succeeded(&running_id).unwrap();
4010
4011 let completed_spec = background_spec("completed");
4012 let completed_id = completed_spec.id.clone();
4013 registry.register_operation(completed_spec).unwrap();
4014 registry.provisioning_succeeded(&completed_id).unwrap();
4015 registry
4016 .complete_operation(
4017 &completed_id,
4018 OperationResult {
4019 id: completed_id.clone(),
4020 content: "done".into(),
4021 is_error: false,
4022 duration_ms: 1,
4023 tokens_used: 0,
4024 },
4025 )
4026 .unwrap();
4027
4028 registry.terminate_owner("shutdown".into()).unwrap();
4029
4030 assert!(matches!(
4031 registry.snapshot(&running_id).unwrap().unwrap().status,
4032 OperationStatus::Terminated
4033 ));
4034 assert!(matches!(
4035 registry.snapshot(&completed_id).unwrap().unwrap().status,
4036 OperationStatus::Completed
4037 ));
4038 }
4039
4040 #[test]
4041 fn collect_completed_drains_terminal_operations() {
4042 let registry = RuntimeOpsLifecycleRegistry::new();
4043
4044 let spec_a = background_spec("a");
4045 let id_a = spec_a.id.clone();
4046 registry.register_operation(spec_a).unwrap();
4047 registry.provisioning_succeeded(&id_a).unwrap();
4048 registry
4049 .complete_operation(
4050 &id_a,
4051 OperationResult {
4052 id: id_a.clone(),
4053 content: "done".into(),
4054 is_error: false,
4055 duration_ms: 1,
4056 tokens_used: 0,
4057 },
4058 )
4059 .unwrap();
4060
4061 let spec_b = background_spec("b");
4062 let id_b = spec_b.id.clone();
4063 registry.register_operation(spec_b).unwrap();
4064
4065 let collected = registry.collect_completed().unwrap();
4066 assert_eq!(collected.len(), 1);
4067 assert_eq!(collected[0].0, id_a);
4068
4069 assert!(registry.snapshot(&id_a).unwrap().is_none());
4070 assert!(registry.snapshot(&id_b).unwrap().is_some());
4071
4072 let collected2 = registry.collect_completed().unwrap();
4073 assert!(collected2.is_empty());
4074 }
4075
4076 #[test]
4077 fn bounded_completed_retention_evicts_oldest() {
4078 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4079 max_completed: 3,
4080 max_concurrent: None,
4081 });
4082
4083 let mut ids = Vec::new();
4084 for i in 0..5 {
4085 let spec = background_spec(&format!("op-{i}"));
4086 let id = spec.id.clone();
4087 registry.register_operation(spec).unwrap();
4088 registry.provisioning_succeeded(&id).unwrap();
4089 registry
4090 .complete_operation(
4091 &id,
4092 OperationResult {
4093 id: id.clone(),
4094 content: format!("done-{i}"),
4095 is_error: false,
4096 duration_ms: 1,
4097 tokens_used: 0,
4098 },
4099 )
4100 .unwrap();
4101 ids.push(id);
4102 }
4103
4104 assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4105 assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4106 assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4107 assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4108 assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4109 }
4110
4111 #[test]
4112 fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4113 let registry = RuntimeOpsLifecycleRegistry::new();
4114
4115 let completed_spec = background_spec("completed");
4116 let completed_id = completed_spec.id.clone();
4117 registry.register_operation(completed_spec).unwrap();
4118 registry.provisioning_succeeded(&completed_id).unwrap();
4119 registry
4120 .complete_operation(
4121 &completed_id,
4122 OperationResult {
4123 id: completed_id.clone(),
4124 content: "done".into(),
4125 is_error: false,
4126 duration_ms: 1,
4127 tokens_used: 0,
4128 },
4129 )
4130 .unwrap();
4131
4132 let running_spec = background_spec("running");
4133 let running_id = running_spec.id.clone();
4134 registry.register_operation(running_spec).unwrap();
4135 registry.provisioning_succeeded(&running_id).unwrap();
4136
4137 let cursor_state = meerkat_core::EpochCursorState::new();
4138 let snapshot = registry
4139 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4140 .unwrap();
4141 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4142
4143 assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4144 assert!(recovered.snapshot(&running_id).unwrap().is_none());
4145
4146 let collected = recovered.collect_completed().unwrap();
4147 assert_eq!(collected.len(), 1);
4148 assert_eq!(collected[0].0, completed_id);
4149 }
4150
4151 #[test]
4152 fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4153 let registry = RuntimeOpsLifecycleRegistry::new();
4154
4155 let mut spec = background_spec("capacity");
4156 spec.kind = OperationKind::BackgroundToolCapacitySlot;
4157 let operation_id = spec.id.clone();
4158 registry.register_operation(spec).unwrap();
4159 registry.provisioning_succeeded(&operation_id).unwrap();
4160 registry.mark_retired(&operation_id).unwrap();
4161
4162 assert!(registry.snapshot(&operation_id).unwrap().is_none());
4163
4164 let cursor_state = meerkat_core::EpochCursorState::new();
4165 let snapshot = registry
4166 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4167 .unwrap();
4168 assert!(
4169 !snapshot
4170 .authority_state
4171 .operations
4172 .contains_key(&operation_id)
4173 );
4174 assert!(!snapshot.operation_specs.contains_key(&operation_id));
4175 assert!(snapshot.completion_entries.is_empty());
4176
4177 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4178 assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4179 }
4180
4181 #[test]
4182 fn recovered_snapshot_uses_authority_operation_source() {
4183 let registry = RuntimeOpsLifecycleRegistry::new();
4184 let child_session_id = SessionId::new();
4185 let operation_source = OperationSource::session_child(child_session_id.clone());
4186 let spec = OperationSpec {
4187 id: OperationId::new(),
4188 kind: OperationKind::MobMemberChild,
4189 owner_session_id: SessionId::new(),
4190 display_name: "source-recovery".into(),
4191 source_label: "test".into(),
4192 operation_source: Some(operation_source.clone()),
4193 child_session_id: Some(child_session_id),
4194 expect_peer_channel: true,
4195 };
4196 let operation_id = spec.id.clone();
4197
4198 registry.register_operation(spec).unwrap();
4199 registry.provisioning_succeeded(&operation_id).unwrap();
4200 registry.mark_retired(&operation_id).unwrap();
4201
4202 let cursor_state = meerkat_core::EpochCursorState::new();
4203 let mut snapshot = registry
4204 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4205 .unwrap();
4206 assert_eq!(
4207 snapshot
4208 .authority_state
4209 .operations
4210 .get(&operation_id)
4211 .and_then(|state| state.operation_source.as_ref()),
4212 Some(&operation_source)
4213 );
4214
4215 snapshot
4216 .operation_specs
4217 .get_mut(&operation_id)
4218 .expect("persisted spec")
4219 .operation_source = None;
4220 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4221 assert_eq!(
4222 recovered
4223 .snapshot(&operation_id)
4224 .unwrap()
4225 .unwrap()
4226 .operation_source,
4227 Some(operation_source)
4228 );
4229 }
4230
4231 #[test]
4232 fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4233 let registry = RuntimeOpsLifecycleRegistry::new();
4234 let child_session_id = SessionId::new();
4235 let operation_source = OperationSource::session_child(child_session_id.clone());
4236 let spec = OperationSpec {
4237 id: OperationId::new(),
4238 kind: OperationKind::MobMemberChild,
4239 owner_session_id: SessionId::new(),
4240 display_name: "source-drift".into(),
4241 source_label: "test".into(),
4242 operation_source: Some(operation_source),
4243 child_session_id: Some(child_session_id),
4244 expect_peer_channel: true,
4245 };
4246 let operation_id = spec.id.clone();
4247
4248 registry.register_operation(spec).unwrap();
4249 registry.provisioning_succeeded(&operation_id).unwrap();
4250 registry.mark_retired(&operation_id).unwrap();
4251
4252 let cursor_state = meerkat_core::EpochCursorState::new();
4253 let mut snapshot = registry
4254 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4255 .unwrap();
4256 snapshot
4257 .operation_specs
4258 .get_mut(&operation_id)
4259 .expect("persisted spec")
4260 .operation_source = Some(OperationSource::session_child(SessionId::new()));
4261
4262 let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4263 .expect_err("source mirror drift must fail recovery");
4264 assert!(
4265 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4266 "unexpected recovery error: {err:?}"
4267 );
4268 }
4269
4270 #[test]
4271 fn persisted_authority_state_serializes_explicit_no_operation_source() {
4272 let registry = RuntimeOpsLifecycleRegistry::new();
4273
4274 let spec = background_spec("explicit-no-source");
4275 let operation_id = spec.id.clone();
4276 registry.register_operation(spec).unwrap();
4277 registry.provisioning_succeeded(&operation_id).unwrap();
4278
4279 let cursor_state = meerkat_core::EpochCursorState::new();
4280 let snapshot = registry
4281 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4282 .unwrap();
4283 let value = serde_json::to_value(&snapshot).unwrap();
4284 let operations = value
4285 .get("authority_state")
4286 .and_then(|state| state.get("operations"))
4287 .and_then(serde_json::Value::as_object)
4288 .expect("serialized authority operations");
4289 let persisted_state = operations
4290 .values()
4291 .next()
4292 .and_then(serde_json::Value::as_object)
4293 .expect("serialized operation state");
4294
4295 assert!(
4296 persisted_state
4297 .get("operation_source")
4298 .is_some_and(serde_json::Value::is_null),
4299 "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4300 );
4301
4302 let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4303 assert_eq!(
4304 recovered_snapshot
4305 .authority_state
4306 .operations
4307 .get(&operation_id)
4308 .expect("round-tripped operation")
4309 .operation_source,
4310 None
4311 );
4312 }
4313
4314 #[test]
4315 fn persisted_authority_state_rejects_missing_operation_source_fact() {
4316 let registry = RuntimeOpsLifecycleRegistry::new();
4317
4318 let spec = background_spec("missing-source-fact");
4319 registry.register_operation(spec).unwrap();
4320
4321 let cursor_state = meerkat_core::EpochCursorState::new();
4322 let snapshot = registry
4323 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4324 .unwrap();
4325 let mut value = serde_json::to_value(&snapshot).unwrap();
4326 let operations = value
4327 .get_mut("authority_state")
4328 .and_then(|state| state.get_mut("operations"))
4329 .and_then(serde_json::Value::as_object_mut)
4330 .expect("serialized authority operations");
4331 let operation_state = operations
4332 .values_mut()
4333 .next()
4334 .and_then(serde_json::Value::as_object_mut)
4335 .expect("serialized operation state");
4336 assert!(operation_state.remove("operation_source").is_some());
4337
4338 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4339 .expect_err("missing generated source fact must fail recovery snapshot decoding");
4340 assert!(
4341 err.to_string().contains("operation_source"),
4342 "unexpected decode error: {err}"
4343 );
4344 }
4345
4346 #[test]
4347 fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4348 let registry = RuntimeOpsLifecycleRegistry::new();
4349
4350 let spec = background_spec("missing-feed-authority");
4351 let operation_id = spec.id.clone();
4352 registry.register_operation(spec).unwrap();
4353 registry.provisioning_succeeded(&operation_id).unwrap();
4354 registry
4355 .complete_operation(
4356 &operation_id,
4357 OperationResult {
4358 id: operation_id.clone(),
4359 content: "done".into(),
4360 is_error: false,
4361 duration_ms: 1,
4362 tokens_used: 0,
4363 },
4364 )
4365 .unwrap();
4366
4367 let cursor_state = meerkat_core::EpochCursorState::new();
4368 let snapshot = registry
4369 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4370 .unwrap();
4371 let mut value = serde_json::to_value(&snapshot).unwrap();
4372 let authority_state = value
4373 .get_mut("authority_state")
4374 .and_then(serde_json::Value::as_object_mut)
4375 .expect("serialized authority state");
4376 assert!(authority_state.remove("completion_feed_entries").is_some());
4377
4378 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4379 .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4380 assert!(
4381 err.to_string().contains("completion_feed_entries"),
4382 "unexpected decode error: {err}"
4383 );
4384 }
4385
4386 #[test]
4387 fn public_child_session_projection_uses_authority_operation_source() {
4388 let registry = RuntimeOpsLifecycleRegistry::new();
4389 let authority_child_session_id = SessionId::new();
4390 let stale_shell_child_session_id = SessionId::new();
4391 let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4392 let spec = OperationSpec {
4393 id: OperationId::new(),
4394 kind: OperationKind::MobMemberChild,
4395 owner_session_id: SessionId::new(),
4396 display_name: "child-projection".into(),
4397 source_label: "test".into(),
4398 operation_source: Some(operation_source),
4399 child_session_id: Some(stale_shell_child_session_id),
4400 expect_peer_channel: true,
4401 };
4402 let operation_id = spec.id.clone();
4403
4404 registry.register_operation(spec).unwrap();
4405
4406 assert_eq!(
4407 registry
4408 .snapshot(&operation_id)
4409 .unwrap()
4410 .unwrap()
4411 .child_session_id,
4412 Some(authority_child_session_id.clone())
4413 );
4414
4415 let cursor_state = meerkat_core::EpochCursorState::new();
4416 let snapshot = registry
4417 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4418 .unwrap();
4419 assert_eq!(
4420 snapshot
4421 .operation_specs
4422 .get(&operation_id)
4423 .expect("persisted spec")
4424 .child_session_id,
4425 Some(authority_child_session_id)
4426 );
4427 }
4428
4429 #[test]
4430 fn generated_terminal_payload_projection_fails_closed() {
4431 let registry = RuntimeOpsLifecycleRegistry::new();
4432
4433 let spec = background_spec("terminal-payload-drift");
4434 let operation_id = spec.id.clone();
4435 registry.register_operation(spec).unwrap();
4436 registry.provisioning_succeeded(&operation_id).unwrap();
4437 registry
4438 .complete_operation(
4439 &operation_id,
4440 OperationResult {
4441 id: operation_id.clone(),
4442 content: "done".into(),
4443 is_error: false,
4444 duration_ms: 1,
4445 tokens_used: 0,
4446 },
4447 )
4448 .unwrap();
4449
4450 {
4451 let mut state = registry.write_state().unwrap();
4452 let mut machine_state = state.dsl.0.state().clone();
4453 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4454 machine_state
4455 .op_terminal_payload
4456 .insert(operation_id_key, OperationTerminalOutcome::Retired);
4457 state.dsl = DslAuthority(Box::new(
4458 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4459 ));
4460 }
4461
4462 let err = match registry.register_watcher(&operation_id) {
4463 Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4464 Err(err) => err,
4465 };
4466 assert!(
4467 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4468 "unexpected watcher error: {err:?}"
4469 );
4470 let err = registry
4471 .snapshot(&operation_id)
4472 .expect_err("invalid generated terminal payload must reject public snapshot");
4473 assert!(
4474 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
4475 "unexpected public snapshot error: {err:?}"
4476 );
4477
4478 let cursor_state = meerkat_core::EpochCursorState::new();
4479 let err = match registry
4480 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4481 {
4482 Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
4483 Err(err) => err,
4484 };
4485 assert!(
4486 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4487 "unexpected snapshot error: {err:?}"
4488 );
4489
4490 let err = match registry.collect_completed() {
4491 Ok(_) => panic!("invalid generated terminal payload must reject collection"),
4492 Err(err) => err,
4493 };
4494 assert!(
4495 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4496 "unexpected collection error: {err:?}"
4497 );
4498 }
4499
4500 #[test]
4501 fn generated_terminal_payload_missing_projection_fails_closed() {
4502 let registry = RuntimeOpsLifecycleRegistry::new();
4503
4504 let spec = background_spec("terminal-payload-missing");
4505 let operation_id = spec.id.clone();
4506 registry.register_operation(spec).unwrap();
4507 registry.provisioning_succeeded(&operation_id).unwrap();
4508 registry
4509 .fail_operation(&operation_id, "boom".into())
4510 .unwrap();
4511
4512 {
4513 let mut state = registry.write_state().unwrap();
4514 let mut machine_state = state.dsl.0.state().clone();
4515 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4516 machine_state.op_terminal_payload.remove(&operation_id_key);
4517 state.dsl = DslAuthority(Box::new(
4518 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4519 ));
4520 }
4521
4522 let err = match registry.register_watcher(&operation_id) {
4523 Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
4524 Err(err) => err,
4525 };
4526 assert!(
4527 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4528 "unexpected watcher error: {err:?}"
4529 );
4530 let err = registry
4531 .snapshot(&operation_id)
4532 .expect_err("missing generated terminal payload must reject public snapshot");
4533 assert!(
4534 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4535 "unexpected public snapshot error: {err:?}"
4536 );
4537 }
4538
4539 #[test]
4540 fn generated_terminal_status_without_outcome_fails_closed() {
4541 let registry = RuntimeOpsLifecycleRegistry::new();
4542
4543 let spec = background_spec("terminal-outcome-missing");
4544 let operation_id = spec.id.clone();
4545 registry.register_operation(spec).unwrap();
4546 registry.provisioning_succeeded(&operation_id).unwrap();
4547 registry
4548 .fail_operation(&operation_id, "boom".into())
4549 .unwrap();
4550
4551 {
4552 let mut state = registry.write_state().unwrap();
4553 let mut machine_state = state.dsl.0.state().clone();
4554 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4555 machine_state.op_terminal_outcomes.remove(&operation_id_key);
4556 state.dsl = DslAuthority(Box::new(
4557 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4558 ));
4559 }
4560
4561 let err = registry
4562 .snapshot(&operation_id)
4563 .expect_err("terminal status without outcome must reject public snapshot");
4564 assert!(
4565 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4566 "unexpected public snapshot error: {err:?}"
4567 );
4568
4569 let err = match registry.collect_completed() {
4570 Ok(_) => panic!("terminal status without outcome must reject collection"),
4571 Err(err) => err,
4572 };
4573 assert!(
4574 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4575 "unexpected collection error: {err:?}"
4576 );
4577 }
4578
4579 #[test]
4580 fn generated_operation_source_projection_fails_closed() {
4581 let registry = RuntimeOpsLifecycleRegistry::new();
4582 let child_session_id = SessionId::new();
4583 let operation_source = OperationSource::session_child(child_session_id.clone());
4584 let spec = OperationSpec {
4585 id: OperationId::new(),
4586 kind: OperationKind::MobMemberChild,
4587 owner_session_id: SessionId::new(),
4588 display_name: "source-authority-drift".into(),
4589 source_label: "test".into(),
4590 operation_source: Some(operation_source),
4591 child_session_id: Some(child_session_id),
4592 expect_peer_channel: true,
4593 };
4594 let operation_id = spec.id.clone();
4595
4596 registry.register_operation(spec).unwrap();
4597
4598 {
4599 let mut state = registry.write_state().unwrap();
4600 let mut machine_state = state.dsl.0.state().clone();
4601 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4602 machine_state
4603 .op_sources
4604 .get_mut(&operation_id_key)
4605 .expect("generated operation source")
4606 .session_id = None;
4607 state.dsl = DslAuthority(Box::new(
4608 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4609 ));
4610 }
4611
4612 let err = registry
4613 .snapshot(&operation_id)
4614 .expect_err("invalid generated operation source must reject public snapshot");
4615 assert!(
4616 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4617 "unexpected public snapshot error: {err:?}"
4618 );
4619 let err = registry
4620 .list_operations()
4621 .expect_err("invalid generated operation source must reject public operation list");
4622 assert!(
4623 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4624 "unexpected operation list error: {err:?}"
4625 );
4626
4627 let cursor_state = meerkat_core::EpochCursorState::new();
4628 let err = match registry
4629 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4630 {
4631 Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
4632 Err(err) => err,
4633 };
4634 assert!(
4635 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4636 "unexpected snapshot error: {err:?}"
4637 );
4638 }
4639
4640 #[test]
4641 fn generated_operation_id_projection_fails_closed() {
4642 let registry = RuntimeOpsLifecycleRegistry::new();
4643
4644 {
4645 let mut state = registry.write_state().unwrap();
4646 let mut machine_state = state.dsl.0.state().clone();
4647 machine_state.op_statuses.insert(
4648 "not-json-operation-id".into(),
4649 mm_dsl::OperationStatus::Running,
4650 );
4651 state.dsl = DslAuthority(Box::new(
4652 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4653 ));
4654 }
4655
4656 let err = registry
4657 .list_operations()
4658 .expect_err("invalid generated operation id must reject public operation list");
4659 assert!(
4660 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4661 "unexpected operation list error: {err:?}"
4662 );
4663
4664 let cursor_state = meerkat_core::EpochCursorState::new();
4665 let err = match registry
4666 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4667 {
4668 Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
4669 Err(err) => err,
4670 };
4671 assert!(
4672 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4673 "unexpected persistence snapshot error: {err:?}"
4674 );
4675 }
4676
4677 #[test]
4678 fn generated_missing_kind_projection_fails_closed() {
4679 let registry = RuntimeOpsLifecycleRegistry::new();
4680 let spec = background_spec("missing-kind");
4681 let operation_id = spec.id.clone();
4682 registry.register_operation(spec).unwrap();
4683
4684 {
4685 let mut state = registry.write_state().unwrap();
4686 let mut machine_state = state.dsl.0.state().clone();
4687 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4688 machine_state.op_kinds.remove(&operation_id_key);
4689 state.dsl = DslAuthority(Box::new(
4690 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4691 ));
4692 }
4693
4694 let err = registry
4695 .snapshot(&operation_id)
4696 .expect_err("missing generated kind must reject public snapshot");
4697 assert!(
4698 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4699 "unexpected public snapshot error: {err:?}"
4700 );
4701 let err = registry
4702 .list_operations()
4703 .expect_err("missing generated kind must reject public list");
4704 assert!(
4705 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4706 "unexpected public list error: {err:?}"
4707 );
4708
4709 let cursor_state = meerkat_core::EpochCursorState::new();
4710 let err = registry
4711 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4712 .expect_err("missing generated kind must reject persistence snapshot");
4713 assert!(
4714 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4715 "unexpected persistence snapshot error: {err:?}"
4716 );
4717 }
4718
4719 #[test]
4720 fn generated_missing_status_projection_fails_closed() {
4721 let registry = RuntimeOpsLifecycleRegistry::new();
4722 let spec = background_spec("missing-status");
4723 let operation_id = spec.id.clone();
4724 registry.register_operation(spec).unwrap();
4725
4726 {
4727 let mut state = registry.write_state().unwrap();
4728 let mut machine_state = state.dsl.0.state().clone();
4729 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4730 machine_state.op_statuses.remove(&operation_id_key);
4731 state.dsl = DslAuthority(Box::new(
4732 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4733 ));
4734 }
4735
4736 let err = registry
4737 .snapshot(&operation_id)
4738 .expect_err("missing generated status must reject public snapshot");
4739 assert!(
4740 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4741 "unexpected public snapshot error: {err:?}"
4742 );
4743 let err = registry
4744 .list_operations()
4745 .expect_err("missing generated status must reject public list");
4746 assert!(
4747 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4748 "unexpected public list error: {err:?}"
4749 );
4750 let err = registry
4751 .classify_operation_public_result(&operation_id)
4752 .expect_err("missing generated status must reject public-result classification");
4753 assert!(
4754 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4755 "unexpected public-result error: {err:?}"
4756 );
4757 }
4758
4759 #[test]
4760 fn generated_retiring_public_result_remains_running_until_terminal() {
4761 let registry = RuntimeOpsLifecycleRegistry::new();
4762 let spec = background_spec("retiring-public-result");
4763 let operation_id = spec.id.clone();
4764 registry.register_operation(spec).unwrap();
4765 registry.provisioning_succeeded(&operation_id).unwrap();
4766 registry.request_retire(&operation_id).unwrap();
4767
4768 let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
4769 assert_eq!(snapshot.status, OperationStatus::Retiring);
4770 assert!(snapshot.terminal_outcome.is_none());
4771 assert!(!snapshot.terminal);
4772 assert_eq!(
4773 snapshot.public_result_class,
4774 OperationPublicResultClass::Running
4775 );
4776 assert_eq!(
4777 registry
4778 .classify_operation_public_result(&operation_id)
4779 .unwrap(),
4780 OperationPublicResultClass::Running
4781 );
4782 }
4783
4784 #[test]
4785 fn generated_missing_peer_ready_projection_fails_closed() {
4786 let registry = RuntimeOpsLifecycleRegistry::new();
4787 let spec = background_spec("missing-peer-ready");
4788 let operation_id = spec.id.clone();
4789 registry.register_operation(spec).unwrap();
4790
4791 {
4792 let mut state = registry.write_state().unwrap();
4793 let mut machine_state = state.dsl.0.state().clone();
4794 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4795 machine_state.op_peer_ready.remove(&operation_id_key);
4796 state.dsl = DslAuthority(Box::new(
4797 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4798 ));
4799 }
4800
4801 let err = registry
4802 .snapshot(&operation_id)
4803 .expect_err("missing generated peer-ready fact must reject public snapshot");
4804 assert!(
4805 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4806 "unexpected public snapshot error: {err:?}"
4807 );
4808
4809 let cursor_state = meerkat_core::EpochCursorState::new();
4810 let err = registry
4811 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4812 .expect_err("missing generated peer-ready fact must reject persistence snapshot");
4813 assert!(
4814 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4815 "unexpected persistence snapshot error: {err:?}"
4816 );
4817 }
4818
4819 #[test]
4820 fn generated_missing_progress_count_projection_fails_closed() {
4821 let registry = RuntimeOpsLifecycleRegistry::new();
4822 let spec = background_spec("missing-progress-count");
4823 let operation_id = spec.id.clone();
4824 registry.register_operation(spec).unwrap();
4825
4826 {
4827 let mut state = registry.write_state().unwrap();
4828 let mut machine_state = state.dsl.0.state().clone();
4829 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4830 machine_state.op_progress_counts.remove(&operation_id_key);
4831 state.dsl = DslAuthority(Box::new(
4832 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4833 ));
4834 }
4835
4836 let err = registry
4837 .snapshot(&operation_id)
4838 .expect_err("missing generated progress count must reject public snapshot");
4839 assert!(
4840 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4841 "unexpected public snapshot error: {err:?}"
4842 );
4843
4844 let cursor_state = meerkat_core::EpochCursorState::new();
4845 let err = registry
4846 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4847 .expect_err("missing generated progress count must reject persistence snapshot");
4848 assert!(
4849 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4850 "unexpected persistence snapshot error: {err:?}"
4851 );
4852 }
4853
4854 #[test]
4855 fn generated_terminal_sequence_missing_persistence_fails_closed() {
4856 let registry = RuntimeOpsLifecycleRegistry::new();
4857 let spec = background_spec("terminal-sequence-missing");
4858 let operation_id = spec.id.clone();
4859 registry.register_operation(spec).unwrap();
4860 registry.provisioning_succeeded(&operation_id).unwrap();
4861 registry
4862 .complete_operation(
4863 &operation_id,
4864 OperationResult {
4865 id: operation_id.clone(),
4866 content: "done".into(),
4867 is_error: false,
4868 duration_ms: 1,
4869 tokens_used: 0,
4870 },
4871 )
4872 .unwrap();
4873
4874 {
4875 let mut state = registry.write_state().unwrap();
4876 let mut machine_state = state.dsl.0.state().clone();
4877 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4878 machine_state.op_completion_seq.remove(&operation_id_key);
4879 state.dsl = DslAuthority(Box::new(
4880 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4881 ));
4882 }
4883
4884 let cursor_state = meerkat_core::EpochCursorState::new();
4885 let err = registry
4886 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4887 .expect_err("missing generated terminal sequence must reject persistence snapshot");
4888 assert!(
4889 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
4890 "unexpected persistence snapshot error: {err:?}"
4891 );
4892 }
4893
4894 #[test]
4895 fn generated_record_without_shell_projection_fails_closed() {
4896 let registry = RuntimeOpsLifecycleRegistry::new();
4897 let spec = background_spec("missing-shell-record");
4898 let operation_id = spec.id.clone();
4899 registry.register_operation(spec).unwrap();
4900
4901 {
4902 let mut state = registry.write_state().unwrap();
4903 state.records.remove(&operation_id);
4904 }
4905
4906 let err = registry
4907 .snapshot(&operation_id)
4908 .expect_err("generated operation without shell record must reject public snapshot");
4909 assert!(
4910 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
4911 "unexpected public snapshot error: {err:?}"
4912 );
4913 let err = registry
4914 .list_operations()
4915 .expect_err("generated operation without shell record must reject public list");
4916 assert!(
4917 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
4918 "unexpected public list error: {err:?}"
4919 );
4920 }
4921
4922 #[test]
4923 fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
4924 let registry = RuntimeOpsLifecycleRegistry::new();
4925 let child_session_id = SessionId::new();
4926 let operation_source = OperationSource::session_child(child_session_id.clone());
4927 let spec = OperationSpec {
4928 id: OperationId::new(),
4929 kind: OperationKind::MobMemberChild,
4930 owner_session_id: SessionId::new(),
4931 display_name: "child-drift".into(),
4932 source_label: "test".into(),
4933 operation_source: Some(operation_source),
4934 child_session_id: Some(child_session_id.clone()),
4935 expect_peer_channel: true,
4936 };
4937 let operation_id = spec.id.clone();
4938
4939 registry.register_operation(spec).unwrap();
4940 registry.provisioning_succeeded(&operation_id).unwrap();
4941 registry.mark_retired(&operation_id).unwrap();
4942
4943 let cursor_state = meerkat_core::EpochCursorState::new();
4944 let mut snapshot = registry
4945 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4946 .unwrap();
4947 snapshot
4948 .operation_specs
4949 .get_mut(&operation_id)
4950 .expect("persisted spec")
4951 .child_session_id = Some(SessionId::new());
4952
4953 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4954 assert_eq!(
4955 recovered
4956 .snapshot(&operation_id)
4957 .unwrap()
4958 .unwrap()
4959 .child_session_id,
4960 Some(child_session_id)
4961 );
4962 }
4963
4964 #[test]
4965 fn completion_wake_class_is_generated_by_operation_kind() {
4966 let registry = RuntimeOpsLifecycleRegistry::new();
4967 let operation_id = OperationId::new();
4968
4969 assert_eq!(
4970 registry
4971 .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
4972 .unwrap(),
4973 OperationCompletionWakeClass::Wake
4974 );
4975 assert_eq!(
4976 registry
4977 .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
4978 .unwrap(),
4979 OperationCompletionWakeClass::Ignore
4980 );
4981 assert_eq!(
4982 registry
4983 .classify_operation_completion_wake(
4984 &operation_id,
4985 OperationKind::BackgroundToolCapacitySlot,
4986 )
4987 .unwrap(),
4988 OperationCompletionWakeClass::Ignore
4989 );
4990 }
4991
4992 #[test]
4993 fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
4994 let registry = RuntimeOpsLifecycleRegistry::new();
4995
4996 let running_spec = background_spec("running");
4997 let running_id = running_spec.id.clone();
4998 registry.register_operation(running_spec.clone()).unwrap();
4999 registry.provisioning_succeeded(&running_id).unwrap();
5000
5001 let cursor_state = meerkat_core::EpochCursorState::new();
5002 let mut snapshot = registry
5003 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5004 .unwrap();
5005 snapshot.completion_entries.push(CompletionEntry {
5006 seq: 1,
5007 operation_id: running_id.clone(),
5008 kind: running_spec.kind,
5009 display_name: running_spec.display_name,
5010 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5011 id: running_id,
5012 content: "phantom".into(),
5013 is_error: false,
5014 duration_ms: 1,
5015 tokens_used: 0,
5016 }),
5017 completed_at_ms: None,
5018 });
5019
5020 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5021 Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5022 Err(err) => err,
5023 };
5024 assert!(
5025 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5026 "unexpected recovery error: {err:?}"
5027 );
5028 }
5029
5030 #[test]
5031 fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5032 let registry = RuntimeOpsLifecycleRegistry::new();
5033 let phantom_id = OperationId::new();
5034 {
5035 let state = registry.read_state().unwrap();
5036 state.feed_buffer.push(CompletionEntry {
5037 seq: 1,
5038 operation_id: phantom_id.clone(),
5039 kind: OperationKind::BackgroundToolOp,
5040 display_name: "phantom".into(),
5041 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5042 id: phantom_id,
5043 content: "phantom".into(),
5044 is_error: false,
5045 duration_ms: 1,
5046 tokens_used: 0,
5047 }),
5048 completed_at_ms: None,
5049 });
5050 }
5051
5052 let cursor_state = meerkat_core::EpochCursorState::new();
5053 let err = match registry
5054 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5055 {
5056 Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5057 Err(err) => err,
5058 };
5059 assert!(
5060 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5061 "unexpected capture error: {err:?}"
5062 );
5063 }
5064
5065 #[test]
5066 fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5067 let registry = RuntimeOpsLifecycleRegistry::new();
5068
5069 let spec = background_spec("terminal");
5070 let operation_id = spec.id.clone();
5071 registry.register_operation(spec).unwrap();
5072 registry.provisioning_succeeded(&operation_id).unwrap();
5073 registry
5074 .complete_operation(
5075 &operation_id,
5076 OperationResult {
5077 id: operation_id.clone(),
5078 content: "done".into(),
5079 is_error: false,
5080 duration_ms: 1,
5081 tokens_used: 0,
5082 },
5083 )
5084 .unwrap();
5085
5086 let cursor_state = meerkat_core::EpochCursorState::new();
5087 let mut snapshot = registry
5088 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5089 .unwrap();
5090 let phantom_id = OperationId::new();
5091 let phantom_result = OperationResult {
5092 id: phantom_id.clone(),
5093 content: "phantom".into(),
5094 is_error: false,
5095 duration_ms: 1,
5096 tokens_used: 0,
5097 };
5098 let phantom_entry = CompletionFeedCanonicalState {
5099 seq: snapshot.authority_state.next_completion_seq + 1,
5100 kind: OperationKind::BackgroundToolOp,
5101 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5102 };
5103 snapshot
5104 .authority_state
5105 .completion_feed_entries
5106 .insert(phantom_id.clone(), phantom_entry);
5107 snapshot.completion_entries.push(CompletionEntry {
5108 seq: snapshot.authority_state.next_completion_seq + 1,
5109 operation_id: phantom_id.clone(),
5110 kind: OperationKind::BackgroundToolOp,
5111 display_name: "phantom".into(),
5112 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5113 completed_at_ms: None,
5114 });
5115
5116 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5117 Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5118 Err(err) => err,
5119 };
5120 assert!(
5121 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5122 "unexpected recovery error: {err:?}"
5123 );
5124 }
5125
5126 #[test]
5127 fn recovered_completed_order_uses_generated_completion_sequences() {
5128 let registry = RuntimeOpsLifecycleRegistry::new();
5129
5130 let spec_a = background_spec("a");
5131 let id_a = spec_a.id.clone();
5132 registry.register_operation(spec_a).unwrap();
5133 registry.provisioning_succeeded(&id_a).unwrap();
5134 registry
5135 .complete_operation(
5136 &id_a,
5137 OperationResult {
5138 id: id_a.clone(),
5139 content: "a".into(),
5140 is_error: false,
5141 duration_ms: 1,
5142 tokens_used: 0,
5143 },
5144 )
5145 .unwrap();
5146
5147 let spec_b = background_spec("b");
5148 let id_b = spec_b.id.clone();
5149 registry.register_operation(spec_b).unwrap();
5150 registry.provisioning_succeeded(&id_b).unwrap();
5151 registry
5152 .complete_operation(
5153 &id_b,
5154 OperationResult {
5155 id: id_b.clone(),
5156 content: "b".into(),
5157 is_error: false,
5158 duration_ms: 1,
5159 tokens_used: 0,
5160 },
5161 )
5162 .unwrap();
5163
5164 let cursor_state = meerkat_core::EpochCursorState::new();
5165 let mut snapshot = registry
5166 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5167 .unwrap();
5168 snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5169
5170 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5171 let collected = recovered.collect_completed().unwrap();
5172
5173 assert_eq!(collected[0].0, id_a);
5174 assert_eq!(collected[1].0, id_b);
5175 }
5176
5177 #[test]
5178 fn max_concurrent_enforcement() {
5179 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5180 max_completed: DEFAULT_MAX_COMPLETED,
5181 max_concurrent: Some(2),
5182 });
5183
5184 let spec_a = background_spec("a");
5185 let id_a = spec_a.id.clone();
5186 registry.register_operation(spec_a).unwrap();
5187
5188 let spec_b = background_spec("b");
5189 registry.register_operation(spec_b).unwrap();
5190
5191 let spec_c = background_spec("c");
5192 let result = registry.register_operation(spec_c);
5193 assert!(matches!(
5194 result,
5195 Err(OpsLifecycleError::MaxConcurrentExceeded {
5196 limit: 2,
5197 active: 2,
5198 })
5199 ));
5200
5201 registry.provisioning_succeeded(&id_a).unwrap();
5202 registry
5203 .complete_operation(
5204 &id_a,
5205 OperationResult {
5206 id: id_a.clone(),
5207 content: "done".into(),
5208 is_error: false,
5209 duration_ms: 1,
5210 tokens_used: 0,
5211 },
5212 )
5213 .unwrap();
5214
5215 let spec_d = background_spec("d");
5216 assert!(registry.register_operation(spec_d).is_ok());
5217 }
5218
5219 #[test]
5220 fn snapshot_includes_timestamps() {
5221 let registry = RuntimeOpsLifecycleRegistry::new();
5222 let spec = background_spec("timed");
5223 let op_id = spec.id.clone();
5224 registry.register_operation(spec).unwrap();
5225
5226 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5227 assert!(snap1.created_at_ms > 0);
5228 assert!(snap1.started_at_ms.is_none());
5229 assert!(snap1.completed_at_ms.is_none());
5230 assert!(snap1.elapsed_ms.is_none());
5231
5232 registry.provisioning_succeeded(&op_id).unwrap();
5233 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5234 assert!(snap2.started_at_ms.is_some());
5235 assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5236
5237 registry
5238 .complete_operation(
5239 &op_id,
5240 OperationResult {
5241 id: op_id.clone(),
5242 content: "done".into(),
5243 is_error: false,
5244 duration_ms: 1,
5245 tokens_used: 0,
5246 },
5247 )
5248 .unwrap();
5249 let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5250 assert!(snap3.completed_at_ms.is_some());
5251 assert!(snap3.elapsed_ms.is_some());
5252 assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5253 }
5254
5255 #[test]
5256 fn snapshot_includes_peer_handle() {
5257 let registry = RuntimeOpsLifecycleRegistry::new();
5258 let child_session_id = SessionId::new();
5259 let spec = OperationSpec {
5260 id: OperationId::new(),
5261 kind: OperationKind::MobMemberChild,
5262 owner_session_id: SessionId::new(),
5263 display_name: "peer-test".into(),
5264 source_label: "test".into(),
5265 operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5266 child_session_id: Some(child_session_id),
5267 expect_peer_channel: true,
5268 };
5269 let op_id = spec.id.clone();
5270 registry.register_operation(spec).unwrap();
5271 registry.provisioning_succeeded(&op_id).unwrap();
5272
5273 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5274 assert!(snap1.peer_handle.is_none());
5275
5276 let handle = OperationPeerHandle {
5277 peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5278 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5279 "member-x",
5280 PeerId::new(),
5281 "inproc://x",
5282 )
5283 .unwrap(),
5284 };
5285 registry.peer_ready(&op_id, handle).unwrap();
5286
5287 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5288 assert_eq!(
5289 snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5290 "member-x"
5291 );
5292 }
5293}