1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24 CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31 CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32 OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33 OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34 OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35 OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36 WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43fn deserialize_required_operation_source<'de, D>(
54 deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57 D: serde::Deserializer<'de>,
58{
59 <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65 status: OperationStatus,
66 kind: OperationKind,
67 #[serde(deserialize_with = "deserialize_required_operation_source")]
68 operation_source: Option<OperationSource>,
69 peer_ready: bool,
70 progress_count: u32,
71 watcher_count: u32,
72 terminal_outcome: Option<OperationTerminalOutcome>,
73 #[serde(default, skip_serializing_if = "Option::is_none")]
74 completion_sequence: Option<CompletionSeq>,
75 terminal_buffered: bool,
76}
77
78#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81 seq: CompletionSeq,
82 kind: OperationKind,
83 terminal_outcome: OperationTerminalOutcome,
84}
85
86#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89 operations: HashMap<OperationId, OperationCanonicalState>,
90 completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91 completed_order: VecDeque<OperationId>,
92 max_completed: usize,
93 max_concurrent: Option<usize>,
94 active_count: usize,
95 wait_request_id: Option<WaitRequestId>,
96 wait_operation_ids: Vec<OperationId>,
97 next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101 pub fn max_completed(&self) -> usize {
103 self.max_completed
104 }
105
106 pub fn max_concurrent(&self) -> Option<usize> {
108 self.max_concurrent
109 }
110
111 pub fn operation_count(&self) -> usize {
113 self.operations.len()
114 }
115
116 pub fn completion_feed_count(&self) -> usize {
118 self.completion_feed_entries.len()
119 }
120}
121
122#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134 pub epoch_id: meerkat_core::RuntimeEpochId,
136 pub authority_state: RegistryCanonicalState,
138 pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140 pub completion_entries: Vec<CompletionEntry>,
143 pub cursors: meerkat_core::EpochCursorSnapshot,
145}
146
147#[derive(Debug)]
148pub struct OpsLifecyclePersistenceRequest {
149 snapshot: PersistedOpsSnapshot,
150 result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
151}
152
153impl OpsLifecyclePersistenceRequest {
154 pub fn snapshot(&self) -> &PersistedOpsSnapshot {
155 &self.snapshot
156 }
157
158 pub fn complete(self, result: Result<(), OpsLifecycleError>) {
159 let _ = self.result_tx.send(result);
160 }
161}
162
163#[derive(Debug)]
172struct FeedBufferInner {
173 entries: VecDeque<CompletionEntry>,
174 watermark: CompletionSeq,
175 max_retained: usize,
176}
177
178#[derive(Debug)]
183struct FeedBuffer {
184 inner: RwLock<FeedBufferInner>,
185 watermark_atomic: AtomicU64,
187 notify: tokio::sync::Notify,
189}
190
191impl FeedBuffer {
192 fn new(max_retained: usize) -> Self {
193 Self {
194 inner: RwLock::new(FeedBufferInner {
195 entries: VecDeque::new(),
196 watermark: 0,
197 max_retained,
198 }),
199 watermark_atomic: AtomicU64::new(0),
200 notify: tokio::sync::Notify::new(),
201 }
202 }
203
204 fn push(&self, entry: CompletionEntry) {
205 let mut inner = self
206 .inner
207 .write()
208 .unwrap_or_else(std::sync::PoisonError::into_inner);
209 let seq = entry.seq;
210 inner.entries.push_back(entry);
211 inner.watermark = seq;
212
213 while inner.entries.len() > inner.max_retained {
215 inner.entries.pop_front();
216 }
217
218 drop(inner);
219
220 self.watermark_atomic.store(seq, Ordering::Release);
221 self.notify.notify_waiters();
222 }
223}
224
225#[derive(Debug, Clone)]
230pub struct RuntimeCompletionFeed {
231 buffer: Arc<FeedBuffer>,
232}
233
234impl CompletionFeed for RuntimeCompletionFeed {
235 fn watermark(&self) -> CompletionSeq {
236 self.buffer.watermark_atomic.load(Ordering::Acquire)
237 }
238
239 fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
240 let inner = self
241 .buffer
242 .inner
243 .read()
244 .unwrap_or_else(std::sync::PoisonError::into_inner);
245 let entries: Vec<CompletionEntry> = inner
246 .entries
247 .iter()
248 .filter(|e| e.seq > after_seq)
249 .cloned()
250 .collect();
251 let watermark = inner.watermark;
252 CompletionBatch { entries, watermark }
253 }
254
255 fn wait_for_advance(
256 &self,
257 after_seq: CompletionSeq,
258 ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
259 Box::pin(async move {
260 loop {
261 let notified = self.buffer.notify.notified();
266 let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
267 if current > after_seq {
268 return current;
269 }
270 notified.await;
271 }
272 })
273 }
274}
275
276#[derive(Debug)]
281struct OperationCompletionNotifier {
282 tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
283}
284
285impl OperationCompletionNotifier {
286 fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
287 Self { tx }
288 }
289
290 fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
291 let _ = self.tx.send(outcome.clone());
292 }
293}
294
295fn operation_completion_watch_from_receiver(
296 rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
297) -> OperationCompletionWatch {
298 Box::pin(async move {
299 rx.await
300 .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
301 })
302}
303
304fn resolved_operation_completion_watch(
305 outcome: OperationTerminalOutcome,
306) -> OperationCompletionWatch {
307 Box::pin(async move { Ok(outcome) })
308}
309
310#[derive(Debug)]
314struct ShellRecord {
315 spec: OperationSpec,
316 peer_handle: Option<OperationPeerHandle>,
317 watchers: Vec<OperationCompletionNotifier>,
321 created_at: Instant,
323 started_at: Option<Instant>,
324 completed_at: Option<Instant>,
325 created_at_wall: SystemTime,
327}
328
329#[derive(Debug)]
330struct PendingWaitState {
331 wait_request_id: WaitRequestId,
332 sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
333}
334
335enum WaitAllAuthorityPlan {
336 AlreadySatisfied(WaitAllSatisfied),
337 ActivateBarrier,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq)]
341enum RecoveredOperationRecordDisposition {
342 Retain,
343 Discard,
344}
345
346impl ShellRecord {
347 fn new(spec: OperationSpec) -> Self {
348 Self {
349 spec,
350 peer_handle: None,
351 watchers: Vec::new(),
352 created_at: Instant::now(),
353 started_at: None,
354 completed_at: None,
355 created_at_wall: SystemTime::now(),
356 }
357 }
358
359 fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
360 wall_anchor
361 .duration_since(UNIX_EPOCH)
362 .map(|d| d.as_millis() as u64)
363 .unwrap_or(0)
364 }
365
366 fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
367 let offset = instant.saturating_duration_since(self.created_at);
370 let wall = self.created_at_wall + offset;
371 Self::epoch_millis(&wall)
372 }
373
374 fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
376 for watcher in std::mem::take(&mut self.watchers) {
377 watcher.notify_after_generated_terminal(outcome);
378 }
379 }
380
381 fn mark_completed(&mut self) {
383 self.completed_at = Some(Instant::now());
384 }
385}
386
387#[derive(Debug)]
392struct ShellState {
393 dsl: DslAuthority,
395 records: HashMap<OperationId, ShellRecord>,
397 pending_wait: Option<PendingWaitState>,
399 completed_order: VecDeque<OperationId>,
401 max_completed: usize,
403 max_concurrent: Option<usize>,
405 wait_request_id: Option<WaitRequestId>,
411 feed_buffer: Arc<FeedBuffer>,
413 persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
415 persistence_sealed: bool,
417 persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
419 persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
421 owner_retired: bool,
427}
428
429struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
435
436impl std::fmt::Debug for DslAuthority {
437 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
438 f.debug_struct("DslAuthority")
439 .field("state", self.0.state())
440 .finish()
441 }
442}
443
444fn new_ops_dsl_authority() -> DslAuthority {
448 DslAuthority(Box::new(
449 crate::meerkat_machine::dsl_authority::new_initialized_authority(
450 "ops lifecycle DSL Initialize must be accepted",
451 ),
452 ))
453}
454
455impl ShellState {
456 fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
457 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
458 let dsl = new_ops_dsl_authority();
459 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
460 let feed_capacity = max_completed.saturating_mul(4).max(1024);
461 tracing::info!(
462 feed_capacity,
463 "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
464 );
465 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
466 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
467 Self {
468 dsl,
469 records: HashMap::new(),
470 pending_wait: None,
471 completed_order: VecDeque::new(),
472 max_completed,
473 max_concurrent,
474 wait_request_id: None,
475 feed_buffer,
480 persist_tx: None,
481 persistence_sealed: false,
482 persist_epoch_id: None,
483 persist_cursor_state: None,
484 owner_retired: false,
485 }
486 }
487
488 fn ensure_owner_active(&self) -> Result<(), OpsLifecycleError> {
489 if self.owner_retired {
490 Err(OpsLifecycleError::OwnerRetired)
491 } else {
492 Ok(())
493 }
494 }
495
496 fn dsl_apply(
503 &mut self,
504 input: mm_dsl::MeerkatMachineInput,
505 context: &str,
506 ) -> Result<(), OpsLifecycleError> {
507 self.dsl_apply_raw(input).map_err(|err| {
508 OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
509 })
510 }
511
512 fn dsl_apply_raw(
519 &mut self,
520 input: mm_dsl::MeerkatMachineInput,
521 ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
522 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
523 }
524
525 fn dsl_apply_with_effects(
526 &mut self,
527 input: mm_dsl::MeerkatMachineInput,
528 context: &str,
529 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
530 let transition =
531 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
532 OpsLifecycleError::Internal(format!(
533 "DSL rejected ops transition ({context}): {err:?}"
534 ))
535 })?;
536 Ok(transition.into_effects())
537 }
538
539 fn checked_terminal_payload(
544 kind: mm_dsl::OperationTerminalOutcomeKind,
545 payload: &OperationTerminalOutcome,
546 authority: &str,
547 operation_id: &str,
548 ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
549 if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
550 return Err(OpsLifecycleError::Internal(format!(
551 "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
552 )));
553 }
554 Ok(payload.clone())
555 }
556
557 fn status(&self, id: &OperationId) -> Option<OperationStatus> {
559 let id_key = mm_dsl::OperationId::from_domain(id).0;
560 self.dsl
561 .0
562 .state()
563 .op_statuses
564 .get(&id_key)
565 .copied()
566 .map(OperationStatus::from)
567 }
568
569 fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
570 self.status(id).ok_or_else(|| {
571 OpsLifecycleError::Internal(format!(
572 "generated op lifecycle authority missing status for {id}"
573 ))
574 })
575 }
576
577 fn kind(&self, id: &OperationId) -> Option<OperationKind> {
579 let id_key = mm_dsl::OperationId::from_domain(id).0;
580 self.dsl
581 .0
582 .state()
583 .op_kinds
584 .get(&id_key)
585 .copied()
586 .map(OperationKind::from)
587 }
588
589 fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
590 self.kind(id).ok_or_else(|| {
591 OpsLifecycleError::Internal(format!(
592 "generated op lifecycle authority missing kind for {id}"
593 ))
594 })
595 }
596
597 fn operation_source(
598 &self,
599 id: &OperationId,
600 ) -> Result<Option<OperationSource>, OpsLifecycleError> {
601 let id_key = mm_dsl::OperationId::from_domain(id).0;
602 self.dsl
603 .0
604 .state()
605 .op_sources
606 .get(&id_key)
607 .map(|source| {
608 source.to_domain().map_err(|error| {
609 OpsLifecycleError::Internal(format!(
610 "generated operation source authority has invalid source for {id}: {error}"
611 ))
612 })
613 })
614 .transpose()
615 }
616
617 fn child_session_id_from_operation_source(
618 operation_source: Option<&OperationSource>,
619 ) -> Option<SessionId> {
620 match operation_source {
621 Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
622 Some(OperationSource::BackendPeer { .. }) | None => None,
623 }
624 }
625
626 fn align_spec_child_session_id_to_source(
627 spec: &mut OperationSpec,
628 operation_source: Option<&OperationSource>,
629 ) {
630 spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
631 }
632
633 fn peer_ready(&self, id: &OperationId) -> Option<bool> {
635 let id_key = mm_dsl::OperationId::from_domain(id).0;
636 self.dsl.0.state().op_peer_ready.get(&id_key).copied()
637 }
638
639 fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
640 self.peer_ready(id).ok_or_else(|| {
641 OpsLifecycleError::Internal(format!(
642 "generated op peer wiring authority missing peer-ready fact for {id}"
643 ))
644 })
645 }
646
647 fn progress_count(&self, id: &OperationId) -> Option<u32> {
649 let id_key = mm_dsl::OperationId::from_domain(id).0;
650 self.dsl
651 .0
652 .state()
653 .op_progress_counts
654 .get(&id_key)
655 .map(|v| (*v).min(u32::MAX as u64) as u32)
656 }
657
658 fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
659 self.progress_count(id).ok_or_else(|| {
660 OpsLifecycleError::Internal(format!(
661 "generated op progress authority missing progress count for {id}"
662 ))
663 })
664 }
665
666 fn terminal_outcome(
670 &self,
671 id: &OperationId,
672 ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
673 let id_key = mm_dsl::OperationId::from_domain(id).0;
674 let state = self.dsl.0.state();
675 let status = self.status(id);
676 let terminal = match status {
677 Some(status) => Self::operation_status_is_terminal(id, status)?,
678 None => false,
679 };
680 let kind = state.op_terminal_outcomes.get(&id_key).copied();
681 let Some(kind) = kind else {
682 if terminal {
683 return Err(OpsLifecycleError::Internal(format!(
684 "generated op terminal authority missing terminal outcome for {id}"
685 )));
686 }
687 return Ok(None);
688 };
689 if !terminal {
690 return Err(OpsLifecycleError::Internal(format!(
691 "generated op terminal authority has terminal outcome for non-terminal {id}"
692 )));
693 }
694 let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
695 OpsLifecycleError::Internal(format!(
696 "generated op terminal authority missing terminal payload for {id}"
697 ))
698 })?;
699 Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
700 .map(Some)
701 }
702
703 fn contains(&self, id: &OperationId) -> bool {
705 let id_key = mm_dsl::OperationId::from_domain(id).0;
706 self.dsl.0.state().op_statuses.contains_key(&id_key)
707 }
708
709 fn active_count(&self) -> usize {
711 self.dsl.0.state().active_op_count as usize
712 }
713
714 fn operation_count(&self) -> usize {
716 self.dsl.0.state().op_statuses.len()
717 }
718
719 fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
721 let mut ids = BTreeSet::new();
722 let state = self.dsl.0.state();
723 Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
724 Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
725 Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
726 Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
727 Self::collect_operation_id_keys(
728 &mut ids,
729 "op_progress_counts",
730 state.op_progress_counts.keys(),
731 )?;
732 Self::collect_operation_id_keys(
733 &mut ids,
734 "op_terminal_outcomes",
735 state.op_terminal_outcomes.keys(),
736 )?;
737 Self::collect_operation_id_keys(
738 &mut ids,
739 "op_terminal_payload",
740 state.op_terminal_payload.keys(),
741 )?;
742 Self::collect_operation_id_keys(
743 &mut ids,
744 "op_completion_seq",
745 state.op_completion_seq.keys(),
746 )?;
747 ids.extend(self.records.keys().cloned());
748 Ok(ids.into_iter().collect())
749 }
750
751 fn collect_operation_id_keys<'a, I>(
752 ids: &mut BTreeSet<OperationId>,
753 field: &str,
754 keys: I,
755 ) -> Result<(), OpsLifecycleError>
756 where
757 I: IntoIterator<Item = &'a String>,
758 {
759 for key in keys {
760 let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
761 OpsLifecycleError::Internal(format!(
762 "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
763 ))
764 })?;
765 ids.insert(id);
766 }
767 Ok(())
768 }
769
770 fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
771 let id_key = mm_dsl::OperationId::from_domain(id).0;
772 let state = self.dsl.0.state();
773 state.op_statuses.contains_key(&id_key)
774 || state.op_kinds.contains_key(&id_key)
775 || state.op_sources.contains_key(&id_key)
776 || state.op_peer_ready.contains_key(&id_key)
777 || state.op_progress_counts.contains_key(&id_key)
778 || state.op_terminal_outcomes.contains_key(&id_key)
779 || state.op_terminal_payload.contains_key(&id_key)
780 || state.op_completion_seq.contains_key(&id_key)
781 }
782
783 fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
785 let id_key = mm_dsl::OperationId::from_domain(id).0;
786 self.dsl.0.state().op_completion_seq.get(&id_key).copied()
787 }
788
789 fn completion_feed_authority_entries(
790 &self,
791 ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
792 let state = self.dsl.0.state();
793 let sequence_keys: BTreeSet<String> =
794 state.completion_feed_sequences.keys().cloned().collect();
795 let companion_domains: [(&str, BTreeSet<String>); 3] = [
796 (
797 "completion_feed_kinds",
798 state.completion_feed_kinds.keys().cloned().collect(),
799 ),
800 (
801 "completion_feed_terminal_outcomes",
802 state
803 .completion_feed_terminal_outcomes
804 .keys()
805 .cloned()
806 .collect(),
807 ),
808 (
809 "completion_feed_terminal_payload",
810 state
811 .completion_feed_terminal_payload
812 .keys()
813 .cloned()
814 .collect(),
815 ),
816 ];
817 for (field, keys) in companion_domains {
818 if keys != sequence_keys {
819 return Err(OpsLifecycleError::Internal(format!(
820 "generated completion feed authority has mismatched {field} domain"
821 )));
822 }
823 }
824
825 let mut entries = HashMap::new();
826 for (id_key, seq) in &state.completion_feed_sequences {
827 if !state.completion_sequence_claims.contains(seq) {
828 return Err(OpsLifecycleError::Internal(format!(
829 "generated completion feed authority sequence {seq} for {id_key} is not claimed"
830 )));
831 }
832 let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
833 OpsLifecycleError::Internal(format!(
834 "generated completion feed authority used invalid operation id key {id_key}: {error}"
835 ))
836 })?;
837 let kind = state
838 .completion_feed_kinds
839 .get(id_key)
840 .copied()
841 .map(OperationKind::from)
842 .ok_or_else(|| {
843 OpsLifecycleError::Internal(format!(
844 "generated completion feed authority missing kind for {id_key}"
845 ))
846 })?;
847 let outcome_kind = state
848 .completion_feed_terminal_outcomes
849 .get(id_key)
850 .copied()
851 .ok_or_else(|| {
852 OpsLifecycleError::Internal(format!(
853 "generated completion feed authority missing terminal outcome for {id_key}"
854 ))
855 })?;
856 let payload = state
857 .completion_feed_terminal_payload
858 .get(id_key)
859 .ok_or_else(|| {
860 OpsLifecycleError::Internal(format!(
861 "generated completion feed authority missing terminal payload for {id_key}"
862 ))
863 })?;
864 let terminal_outcome = Self::checked_terminal_payload(
865 outcome_kind,
866 payload,
867 "generated completion feed authority",
868 id_key,
869 )?;
870 entries.insert(
871 operation_id,
872 CompletionFeedCanonicalState {
873 seq: *seq,
874 kind,
875 terminal_outcome,
876 },
877 );
878 }
879 Ok(entries)
880 }
881
882 fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
883 let state = self.dsl.0.state();
884 match consumer {
885 CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
886 CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
887 CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
888 }
889 }
890
891 fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
892 meerkat_core::EpochCursorSnapshot {
893 agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
894 runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
895 runtime_last_injected_seq: self
896 .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
897 }
898 }
899
900 fn snapshot(
902 &self,
903 id: &OperationId,
904 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
905 let Some(shell) = self.records.get(id) else {
906 if self.has_generated_operation_record_fact(id) {
907 return Err(OpsLifecycleError::Internal(format!(
908 "generated op lifecycle authority has operation facts without shell projection record for {id}"
909 )));
910 }
911 return Ok(None);
912 };
913 let kind = self.require_kind(id)?;
914 let status = self.require_status(id)?;
915 let terminal = Self::operation_status_is_terminal(id, status)?;
916 let public_result_class = Self::operation_public_result_class(id, status)?;
917 let peer_ready = self.require_peer_ready(id)?;
918 let progress_count = self.require_progress_count(id)?;
919 let operation_source = self.operation_source(id)?;
920 let terminal_outcome = self.terminal_outcome(id)?;
921
922 let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
923 let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
924 let completed_at_ms = shell
925 .completed_at
926 .map(|i| shell.epoch_millis_for_instant(i));
927 let elapsed_ms = shell.completed_at.map(|completed| {
928 completed
929 .saturating_duration_since(shell.created_at)
930 .as_millis() as u64
931 });
932
933 Ok(Some(OperationLifecycleSnapshot {
934 id: shell.spec.id.clone(),
935 kind,
936 owner_session_id: shell.spec.owner_session_id.clone(),
937 display_name: shell.spec.display_name.clone(),
938 source_label: shell.spec.source_label.clone(),
939 child_session_id: Self::child_session_id_from_operation_source(
940 operation_source.as_ref(),
941 ),
942 operation_source,
943 expect_peer_channel: shell.spec.expect_peer_channel,
944 status,
945 terminal,
946 public_result_class,
947 peer_ready,
948 progress_count,
949 watcher_count: shell.watchers.len() as u32,
950 terminal_outcome,
951 peer_handle: shell.peer_handle.clone(),
952 created_at_ms,
953 started_at_ms,
954 completed_at_ms,
955 elapsed_ms,
956 }))
957 }
958
959 fn finalize_terminal(
964 &mut self,
965 id: &OperationId,
966 ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
967 let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
968 OpsLifecycleError::Internal(format!(
969 "generated op terminal transition did not mint terminal outcome for {id}"
970 ))
971 })?;
972 let kind = self.require_kind(id)?;
973
974 if let Some(shell) = self.records.get_mut(id) {
976 shell.notify_watchers(&outcome);
977 shell.mark_completed();
978 }
979
980 if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
981 {
982 self.dsl_apply(
983 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
984 operation_id: mm_dsl::OperationId::from_domain(id).0,
985 },
986 "CollectCompletedOp",
987 )?;
988 self.records.remove(id);
989 self.completed_order.retain(|queued| queued != id);
990 return Ok(None);
991 }
992
993 let mut completion_entry = None;
994 if Self::operation_completion_feed_class(id, kind)?
995 == mm_dsl::OperationCompletionFeedClass::Emit
996 {
997 let feed_authority = self
998 .completion_feed_authority_entries()?
999 .remove(id)
1000 .ok_or_else(|| {
1001 OpsLifecycleError::Internal(format!(
1002 "generated op terminal transition did not mint completion feed authority for {id}"
1003 ))
1004 })?;
1005 if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
1006 return Err(OpsLifecycleError::Internal(format!(
1007 "generated completion feed authority drifted from terminal op authority for {id}"
1008 )));
1009 }
1010 let seq = self.completion_sequence(id).ok_or_else(|| {
1011 OpsLifecycleError::Internal(format!(
1012 "generated op terminal transition did not mint completion sequence for {id}"
1013 ))
1014 })?;
1015 if feed_authority.seq != seq {
1016 return Err(OpsLifecycleError::Internal(format!(
1017 "generated completion feed authority sequence drifted for {id}"
1018 )));
1019 }
1020 let display_name = self
1021 .records
1022 .get(id)
1023 .map(|r| r.spec.display_name.clone())
1024 .unwrap_or_default();
1025 let completed_at_ms = self
1026 .records
1027 .get(id)
1028 .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1029 completion_entry = Some(CompletionEntry {
1030 seq: feed_authority.seq,
1031 operation_id: id.clone(),
1032 kind: feed_authority.kind,
1033 display_name,
1034 terminal_outcome: feed_authority.terminal_outcome,
1035 completed_at_ms,
1036 });
1037 }
1038
1039 self.completed_order.push_back(id.clone());
1041 while self.completed_order.len() > self.max_completed {
1042 if let Some(evicted) = self.completed_order.pop_front() {
1043 self.dsl_apply(
1044 mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1045 operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1046 },
1047 "EvictCompletedOp",
1048 )?;
1049 self.records.remove(&evicted);
1050 }
1051 }
1052
1053 self.maybe_satisfy_wait()?;
1058 Ok(completion_entry)
1059 }
1060
1061 fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1062 if let Some(entry) = entry {
1063 self.feed_buffer.push(entry);
1064 }
1065 }
1066
1067 fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1069 self.dsl
1070 .0
1071 .state()
1072 .wait_operation_ids
1073 .iter()
1074 .map(|key| {
1075 serde_json::from_str::<OperationId>(key).map_err(|error| {
1076 OpsLifecycleError::Internal(format!(
1077 "generated wait operation identity authority used invalid operation id key {key}: {error}"
1078 ))
1079 })
1080 })
1081 .collect()
1082 }
1083
1084 #[cfg(test)]
1086 fn wait_active(&self) -> bool {
1087 self.dsl.0.state().wait_active
1088 }
1089
1090 fn wait_all_satisfied_from_effects(
1091 effects: &[mm_dsl::MeerkatMachineEffect],
1092 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1093 let mut satisfied = None;
1094 for effect in effects {
1095 let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1096 wait_request_id,
1097 run_id,
1098 operation_ids,
1099 } = effect
1100 else {
1101 continue;
1102 };
1103 if satisfied.is_some() {
1104 return Err(OpsLifecycleError::Internal(
1105 "generated wait_all authority emitted multiple satisfaction effects".into(),
1106 ));
1107 }
1108 let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1109 OpsLifecycleError::Internal(format!(
1110 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1111 wait_request_id.0
1112 ))
1113 })?;
1114 let mut ids = Vec::with_capacity(operation_ids.len());
1115 for operation_id in operation_ids {
1116 ids.push(
1117 serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1118 OpsLifecycleError::Internal(format!(
1119 "generated wait_all authority emitted invalid operation id '{}': {err}",
1120 operation_id.0
1121 ))
1122 })?,
1123 );
1124 }
1125 satisfied = Some(WaitAllSatisfied {
1126 wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1127 run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1128 OpsLifecycleError::Internal(format!(
1129 "generated wait_all authority emitted invalid run id '{}': {err}",
1130 run_id.0
1131 ))
1132 })?),
1133 operation_ids: ids,
1134 });
1135 }
1136 Ok(satisfied)
1137 }
1138
1139 fn parse_wait_all_operation_id(
1140 raw: &str,
1141 context: &str,
1142 ) -> Result<OperationId, OpsLifecycleError> {
1143 serde_json::from_str::<OperationId>(raw).map_err(|err| {
1144 OpsLifecycleError::Internal(format!(
1145 "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1146 ))
1147 })
1148 }
1149
1150 fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1151 let mut seen = HashSet::new();
1152 operation_ids
1153 .iter()
1154 .find(|operation_id| !seen.insert((*operation_id).clone()))
1155 .cloned()
1156 }
1157
1158 fn wait_all_admission_error_from_effects(
1159 wait_request_id: &WaitRequestId,
1160 effects: &[mm_dsl::MeerkatMachineEffect],
1161 ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1162 let mut admission = None;
1163 for effect in effects {
1164 let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1165 wait_request_id: resolved_wait_request_id,
1166 result,
1167 reject_reason,
1168 rejected_operation_id,
1169 } = effect
1170 else {
1171 continue;
1172 };
1173 if admission.is_some() {
1174 return Err(OpsLifecycleError::Internal(
1175 "generated wait_all authority emitted multiple admission results".into(),
1176 ));
1177 }
1178 let resolved_uuid =
1179 uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1180 OpsLifecycleError::Internal(format!(
1181 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1182 resolved_wait_request_id.0
1183 ))
1184 })?;
1185 let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1186 if &resolved_wait_request_id != wait_request_id {
1187 return Err(OpsLifecycleError::Internal(format!(
1188 "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1189 )));
1190 }
1191 admission = Some(match result {
1192 mm_dsl::WaitAllAdmissionResultKind::Accept => {
1193 if reject_reason.is_some() || rejected_operation_id.is_some() {
1194 return Err(OpsLifecycleError::Internal(
1195 "generated wait_all authority accepted with rejection payload".into(),
1196 ));
1197 }
1198 None
1199 }
1200 mm_dsl::WaitAllAdmissionResultKind::Reject => {
1201 let reason = reject_reason.ok_or_else(|| {
1202 OpsLifecycleError::Internal(
1203 "generated wait_all authority rejected without reason".into(),
1204 )
1205 })?;
1206 let error = match reason {
1207 mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1208 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1209 OpsLifecycleError::Internal(
1210 "generated wait_all authority rejected duplicate without operation id"
1211 .into(),
1212 )
1213 })?;
1214 OpsLifecycleError::DuplicateWaitOperation(
1215 Self::parse_wait_all_operation_id(raw, "duplicate")?,
1216 )
1217 }
1218 mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1219 if rejected_operation_id.is_some() {
1220 return Err(OpsLifecycleError::Internal(
1221 "generated wait_all authority rejected active wait with operation id"
1222 .into(),
1223 ));
1224 }
1225 OpsLifecycleError::WaitAlreadyActive
1226 }
1227 mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1228 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1229 OpsLifecycleError::Internal(
1230 "generated wait_all authority rejected missing operation without operation id"
1231 .into(),
1232 )
1233 })?;
1234 OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1235 raw, "missing",
1236 )?)
1237 }
1238 };
1239 Some(error)
1240 }
1241 });
1242 }
1243 admission.ok_or_else(|| {
1244 OpsLifecycleError::Internal(
1245 "generated wait_all authority emitted no admission result".into(),
1246 )
1247 })
1248 }
1249
1250 fn resolve_wait_all_admission(
1251 &mut self,
1252 wait_request_id: &WaitRequestId,
1253 operation_ids: &[OperationId],
1254 dsl_ids: &BTreeSet<String>,
1255 dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1256 operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1257 operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1258 ) -> Result<(), OpsLifecycleError> {
1259 let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1260 .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1261 let not_found = operation_ids
1262 .iter()
1263 .find(|operation_id| !self.contains(operation_id))
1264 .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1265 let dsl_id_sequence: Vec<String> = operation_ids
1266 .iter()
1267 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1268 .collect();
1269 let effects = self.dsl_apply_with_effects(
1270 mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1271 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1272 operation_id_sequence: dsl_id_sequence,
1273 operation_ids: dsl_ids.clone(),
1274 operation_id_tokens: dsl_id_tokens.clone(),
1275 operation_token_by_id: operation_token_by_id.clone(),
1276 operation_id_by_token: operation_id_by_token.clone(),
1277 duplicate_operation_id: duplicate,
1278 not_found_operation_id: not_found,
1279 },
1280 "ResolveWaitAllAdmission",
1281 )?;
1282 if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1283 {
1284 return Err(error);
1285 }
1286 Ok(())
1287 }
1288
1289 fn try_satisfy_wait_all_authority(
1290 &mut self,
1291 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1292 let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1293 return Ok(None);
1294 };
1295 let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1296 return Err(OpsLifecycleError::Internal(
1297 "generated wait_all authority has active wait without run id".into(),
1298 ));
1299 };
1300 let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1301 let transition = match mm_dsl::MeerkatMachineMutator::apply(
1302 &mut *self.dsl.0,
1303 mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1304 wait_request_id: dsl_wait_request_id,
1305 run_id: dsl_run_id,
1306 operation_id_tokens: dsl_operation_id_tokens,
1307 },
1308 ) {
1309 Ok(transition) => transition,
1310 Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1311 Err(err) => {
1312 return Err(OpsLifecycleError::Internal(format!(
1313 "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1314 )));
1315 }
1316 };
1317 Self::wait_all_satisfied_from_effects(transition.effects())?
1318 .ok_or_else(|| {
1319 OpsLifecycleError::Internal(
1320 "generated wait_all authority accepted satisfaction without effect".into(),
1321 )
1322 })
1323 .map(Some)
1324 }
1325
1326 fn begin_wait_all_authority(
1327 &mut self,
1328 run_id: &RunId,
1329 wait_request_id: &WaitRequestId,
1330 operation_ids: &[OperationId],
1331 ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1332 let mut dsl_ids = BTreeSet::new();
1333 let mut dsl_id_tokens = BTreeSet::new();
1334 let mut operation_token_by_id = BTreeMap::new();
1335 let mut operation_id_by_token = BTreeMap::new();
1336 for id in operation_ids {
1337 let token = mm_dsl::OperationId::from_domain(id);
1338 let raw_id = token.0.clone();
1339 dsl_ids.insert(raw_id.clone());
1340 dsl_id_tokens.insert(token.clone());
1341 operation_token_by_id.insert(raw_id.clone(), token.clone());
1342 operation_id_by_token.insert(token, raw_id);
1343 }
1344 self.resolve_wait_all_admission(
1345 wait_request_id,
1346 operation_ids,
1347 &dsl_ids,
1348 &dsl_id_tokens,
1349 &operation_token_by_id,
1350 &operation_id_by_token,
1351 )?;
1352 self.dsl_apply(
1353 mm_dsl::MeerkatMachineInput::RequestWaitAll {
1354 run_id: mm_dsl::RunId::from_domain(run_id),
1355 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1356 operation_id_sequence: operation_ids
1357 .iter()
1358 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1359 .collect(),
1360 operation_ids: dsl_ids,
1361 operation_id_tokens: dsl_id_tokens,
1362 operation_token_by_id,
1363 operation_id_by_token,
1364 },
1365 "RequestWaitAll",
1366 )?;
1367 if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1368 return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1369 }
1370 Ok(WaitAllAuthorityPlan::ActivateBarrier)
1371 }
1372
1373 fn owner_termination_targets(
1374 &self,
1375 ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1376 let mut targets = Vec::new();
1377 for id in self.operation_ids()? {
1378 let status = self.require_status(&id)?;
1379 if !Self::operation_status_is_terminal(&id, status)? {
1380 targets.push((id, status));
1381 }
1382 }
1383 Ok(targets)
1384 }
1385
1386 fn operation_status_is_terminal(
1387 operation_id: &OperationId,
1388 status: OperationStatus,
1389 ) -> Result<bool, OpsLifecycleError> {
1390 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1391 let effects = Self::apply_stateless_classifier(
1392 mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1393 operation_id: operation_id_key.clone(),
1394 status: mm_dsl::OperationStatus::from(status),
1395 },
1396 "ClassifyOperationTerminality",
1397 )?;
1398 let mut terminal = None;
1399 for effect in effects {
1400 match effect {
1401 mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1402 if operation_id == operation_id_key =>
1403 {
1404 terminal = Some(true);
1405 }
1406 mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1407 if operation_id == operation_id_key =>
1408 {
1409 terminal = Some(false);
1410 }
1411 other => {
1412 return Err(OpsLifecycleError::Internal(format!(
1413 "unexpected generated operation terminality effect: {other:?}"
1414 )));
1415 }
1416 }
1417 }
1418 terminal.ok_or_else(|| {
1419 OpsLifecycleError::Internal(format!(
1420 "generated operation terminality authority emitted no effect for {operation_id}"
1421 ))
1422 })
1423 }
1424
1425 fn operation_public_result_class(
1426 operation_id: &OperationId,
1427 status: OperationStatus,
1428 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1429 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1430 let effects = Self::apply_stateless_classifier(
1431 mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1432 operation_id: operation_id_key.clone(),
1433 status: mm_dsl::OperationStatus::from(status),
1434 },
1435 "ClassifyOperationPublicResult",
1436 )?;
1437 let mut result = None;
1438 for effect in effects {
1439 match effect {
1440 mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1441 operation_id,
1442 result: classified,
1443 } if operation_id == operation_id_key => {
1444 result = Some(OperationPublicResultClass::from(classified));
1445 }
1446 other => {
1447 return Err(OpsLifecycleError::Internal(format!(
1448 "unexpected generated operation public-result effect: {other:?}"
1449 )));
1450 }
1451 }
1452 }
1453 result.ok_or_else(|| {
1454 OpsLifecycleError::Internal(format!(
1455 "generated operation public-result authority emitted no effect for {operation_id}"
1456 ))
1457 })
1458 }
1459
1460 fn operation_transition_rejection_is_idempotent(
1461 operation_id: &OperationId,
1462 action: OperationLifecycleAction,
1463 status: OperationStatus,
1464 ) -> Result<bool, OpsLifecycleError> {
1465 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1466 let action = mm_dsl::OpLifecycleActionKind::from(action);
1467 let status = mm_dsl::OperationStatus::from(status);
1468 let effects = Self::apply_stateless_classifier(
1469 mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1470 operation_id: operation_id_key.clone(),
1471 action,
1472 status,
1473 },
1474 "ClassifyOperationTransitionIdempotence",
1475 )?;
1476 let mut idempotent = None;
1477 for effect in effects {
1478 match effect {
1479 mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1480 operation_id,
1481 action: effect_action,
1482 status: effect_status,
1483 } if operation_id == operation_id_key
1484 && effect_action == action
1485 && effect_status == status =>
1486 {
1487 idempotent = Some(true);
1488 }
1489 mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1490 operation_id,
1491 action: effect_action,
1492 status: effect_status,
1493 } if operation_id == operation_id_key
1494 && effect_action == action
1495 && effect_status == status =>
1496 {
1497 idempotent = Some(false);
1498 }
1499 other => {
1500 return Err(OpsLifecycleError::Internal(format!(
1501 "unexpected generated operation transition-idempotence effect: {other:?}"
1502 )));
1503 }
1504 }
1505 }
1506 idempotent.ok_or_else(|| {
1507 OpsLifecycleError::Internal(format!(
1508 "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1509 ))
1510 })
1511 }
1512
1513 fn operation_completion_feed_class(
1514 operation_id: &OperationId,
1515 kind: OperationKind,
1516 ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1517 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1518 let kind = mm_dsl::OperationKind::from(kind);
1519 let effects = Self::apply_stateless_classifier(
1520 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1521 operation_id: operation_id_key.clone(),
1522 kind,
1523 },
1524 "ClassifyOperationCompletionFeed",
1525 )?;
1526 let mut class = None;
1527 for effect in effects {
1528 match effect {
1529 mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1530 operation_id,
1531 result,
1532 } if operation_id == operation_id_key => {
1533 class = Some(result);
1534 }
1535 other => {
1536 return Err(OpsLifecycleError::Internal(format!(
1537 "unexpected generated operation completion-feed effect: {other:?}"
1538 )));
1539 }
1540 }
1541 }
1542 class.ok_or_else(|| {
1543 OpsLifecycleError::Internal(format!(
1544 "generated operation completion-feed authority emitted no effect for {operation_id}"
1545 ))
1546 })
1547 }
1548
1549 fn operation_completion_wake_class(
1550 operation_id: &OperationId,
1551 kind: OperationKind,
1552 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1553 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1554 let kind = mm_dsl::OperationKind::from(kind);
1555 let effects = Self::apply_stateless_classifier(
1556 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1557 operation_id: operation_id_key.clone(),
1558 kind,
1559 },
1560 "ClassifyOperationCompletionWake",
1561 )?;
1562 let mut class = None;
1563 for effect in effects {
1564 match effect {
1565 mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1566 operation_id,
1567 result,
1568 } if operation_id == operation_id_key => {
1569 class = Some(OperationCompletionWakeClass::from(result));
1570 }
1571 other => {
1572 return Err(OpsLifecycleError::Internal(format!(
1573 "unexpected generated operation completion-wake effect: {other:?}"
1574 )));
1575 }
1576 }
1577 }
1578 class.ok_or_else(|| {
1579 OpsLifecycleError::Internal(format!(
1580 "generated operation completion-wake authority emitted no effect for {operation_id}"
1581 ))
1582 })
1583 }
1584
1585 fn operation_durability_class(
1586 operation_id: &OperationId,
1587 kind: OperationKind,
1588 ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1589 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1590 let kind = mm_dsl::OperationKind::from(kind);
1591 let effects = Self::apply_stateless_classifier(
1592 mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1593 operation_id: operation_id_key.clone(),
1594 kind,
1595 },
1596 "ClassifyOperationDurability",
1597 )?;
1598 let mut class = None;
1599 for effect in effects {
1600 match effect {
1601 mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1602 operation_id,
1603 result,
1604 } if operation_id == operation_id_key => {
1605 class = Some(result);
1606 }
1607 other => {
1608 return Err(OpsLifecycleError::Internal(format!(
1609 "unexpected generated operation durability effect: {other:?}"
1610 )));
1611 }
1612 }
1613 }
1614 class.ok_or_else(|| {
1615 OpsLifecycleError::Internal(format!(
1616 "generated operation durability authority emitted no effect for {operation_id}"
1617 ))
1618 })
1619 }
1620
1621 fn recovered_operation_record_disposition(
1622 operation_id: &OperationId,
1623 status: OperationStatus,
1624 kind: OperationKind,
1625 terminal_outcome_present: bool,
1626 terminal_payload_present: bool,
1627 completion_sequence_present: bool,
1628 ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1629 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1630 let effects = Self::apply_stateless_classifier(
1631 mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1632 operation_id: operation_id_key.clone(),
1633 status: mm_dsl::OperationStatus::from(status),
1634 kind: mm_dsl::OperationKind::from(kind),
1635 terminal_outcome_present,
1636 terminal_payload_present,
1637 completion_sequence_present,
1638 },
1639 "ClassifyRecoveredOperationRecord",
1640 )?;
1641 let mut disposition = None;
1642 for effect in effects {
1643 match effect {
1644 mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1645 if operation_id == operation_id_key =>
1646 {
1647 disposition = Some(RecoveredOperationRecordDisposition::Retain);
1648 }
1649 mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1650 if operation_id == operation_id_key =>
1651 {
1652 disposition = Some(RecoveredOperationRecordDisposition::Discard);
1653 }
1654 other => {
1655 return Err(OpsLifecycleError::Internal(format!(
1656 "unexpected generated recovered-operation classification effect: {other:?}"
1657 )));
1658 }
1659 }
1660 }
1661 disposition.ok_or_else(|| {
1662 OpsLifecycleError::Internal(format!(
1663 "generated recovered-operation classifier emitted no effect for {operation_id}"
1664 ))
1665 })
1666 }
1667
1668 fn apply_stateless_classifier(
1669 input: mm_dsl::MeerkatMachineInput,
1670 label: &'static str,
1671 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1672 let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1673 "ops stateless classifier Initialize must be accepted",
1674 );
1675 let transition =
1676 mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1677 OpsLifecycleError::Internal(format!(
1678 "DSL rejected ops transition ({label}): {err:?}"
1679 ))
1680 })?;
1681 Ok(transition.into_effects())
1682 }
1683
1684 fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1708 let satisfied = match self.try_satisfy_wait_all_authority() {
1709 Ok(Some(satisfied)) => satisfied,
1710 Ok(None) => return Ok(()),
1711 Err(err) => {
1712 if let Some(pending) = self.pending_wait.take() {
1720 drop(pending.sender);
1721 }
1722 self.wait_request_id = None;
1723 return Err(err);
1724 }
1725 };
1726 let shell_wait_id = self.wait_request_id.take();
1727 if shell_wait_id
1728 .as_ref()
1729 .is_some_and(|id| id != &satisfied.wait_request_id)
1730 {
1731 tracing::error!(
1732 shell_wait_request_id = ?shell_wait_id,
1733 authority_wait_request_id = %satisfied.wait_request_id,
1734 "generated wait_all authority satisfied a different wait request"
1735 );
1736 }
1737 if let Some(pending) = self.pending_wait.take() {
1738 if pending.wait_request_id == satisfied.wait_request_id {
1739 let _ = pending.sender.send(satisfied);
1740 } else if let Some(shell_wait_id) = shell_wait_id {
1741 tracing::error!(
1742 shell_wait_request_id = %shell_wait_id,
1743 pending_wait_request_id = %pending.wait_request_id,
1744 authority_wait_request_id = %satisfied.wait_request_id,
1745 "generated wait_all authority satisfied without a matching pending waiter"
1746 );
1747 }
1748 }
1749 Ok(())
1750 }
1751
1752 fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1758 let (tx, epoch_id, cursor_state) = match (
1759 &self.persist_tx,
1760 &self.persist_epoch_id,
1761 &self.persist_cursor_state,
1762 ) {
1763 (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1764 _ => return Ok(()),
1765 };
1766
1767 let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1768 let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1769 let request = OpsLifecyclePersistenceRequest {
1770 snapshot,
1771 result_tx,
1772 };
1773
1774 tx.send(request).map_err(|_| {
1775 OpsLifecycleError::Internal(
1776 "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1777 .into(),
1778 )
1779 })?;
1780 result_rx.recv().map_err(|_| {
1781 OpsLifecycleError::Internal(
1782 "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1783 .into(),
1784 )
1785 })?
1786 }
1787
1788 fn capture_snapshot(
1790 &self,
1791 epoch_id: meerkat_core::RuntimeEpochId,
1792 _cursor_state: &meerkat_core::EpochCursorState,
1793 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1794 let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1795 for op_id in self.operation_ids()? {
1796 let status = self.require_status(&op_id)?;
1797 let kind = self.require_kind(&op_id)?;
1798 if Self::operation_durability_class(&op_id, kind)?
1799 != mm_dsl::OperationDurabilityClass::Retain
1800 {
1801 continue;
1802 }
1803 let peer_ready = self.require_peer_ready(&op_id)?;
1804 let progress_count = self.require_progress_count(&op_id)?;
1805 let operation_source = self.operation_source(&op_id)?;
1806 let terminal_outcome = self.terminal_outcome(&op_id)?;
1807 let completion_sequence = self.completion_sequence(&op_id);
1808 if terminal_outcome.is_some() && completion_sequence.is_none() {
1809 return Err(OpsLifecycleError::Internal(format!(
1810 "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1811 )));
1812 }
1813 if terminal_outcome.is_none() && completion_sequence.is_some() {
1814 return Err(OpsLifecycleError::Internal(format!(
1815 "generated op terminal authority has completion sequence for non-terminal {op_id}"
1816 )));
1817 }
1818 let terminal_buffered = terminal_outcome.is_some();
1819 let watcher_count = self
1820 .records
1821 .get(&op_id)
1822 .map(|r| r.watchers.len() as u32)
1823 .unwrap_or(0);
1824 operations.insert(
1825 op_id,
1826 OperationCanonicalState {
1827 status,
1828 kind,
1829 operation_source,
1830 peer_ready,
1831 progress_count,
1832 watcher_count,
1833 terminal_outcome,
1834 completion_sequence,
1835 terminal_buffered,
1836 },
1837 );
1838 }
1839 let operation_specs: HashMap<OperationId, OperationSpec> = self
1840 .records
1841 .iter()
1842 .filter(|(id, _)| operations.contains_key(*id))
1843 .map(|(id, record)| {
1844 let mut spec = record.spec.clone();
1845 let operation_source = operations
1846 .get(id)
1847 .and_then(|state| state.operation_source.as_ref());
1848 Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1849 (id.clone(), spec)
1850 })
1851 .collect();
1852 let completed_order: VecDeque<OperationId> = self
1853 .completed_order
1854 .iter()
1855 .filter(|id| operations.contains_key(*id))
1856 .cloned()
1857 .collect();
1858 let active_count = operations
1859 .iter()
1860 .filter(|(id, state)| {
1861 matches!(
1862 Self::operation_status_is_terminal(id, state.status),
1863 Ok(false)
1864 )
1865 })
1866 .count();
1867 let authority_completion_entries = self.completion_feed_authority_entries()?;
1868 let published_completion_entries_by_id: HashMap<OperationId, CompletionEntry> = {
1869 let inner = self
1870 .feed_buffer
1871 .inner
1872 .read()
1873 .unwrap_or_else(std::sync::PoisonError::into_inner);
1874 let mut entries = HashMap::new();
1875 for entry in &inner.entries {
1876 if !authority_completion_entries.contains_key(&entry.operation_id) {
1877 return Err(OpsLifecycleError::Internal(format!(
1878 "public completion feed projection for {} has no generated authority",
1879 entry.operation_id
1880 )));
1881 }
1882 if entries
1883 .insert(entry.operation_id.clone(), entry.clone())
1884 .is_some()
1885 {
1886 return Err(OpsLifecycleError::Internal(format!(
1887 "public completion feed projection for {} appeared more than once",
1888 entry.operation_id
1889 )));
1890 }
1891 }
1892 entries
1893 };
1894 let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
1895 .iter()
1896 .map(|(operation_id, authority_entry)| {
1897 if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
1898 if projection.seq != authority_entry.seq
1899 || projection.kind != authority_entry.kind
1900 || projection.terminal_outcome != authority_entry.terminal_outcome
1901 {
1902 return Err(OpsLifecycleError::Internal(format!(
1903 "public completion feed projection for {operation_id} drifted from generated authority"
1904 )));
1905 }
1906 return Ok(projection.clone());
1907 }
1908
1909 let display_name = self
1910 .records
1911 .get(operation_id)
1912 .map(|record| record.spec.display_name.clone())
1913 .unwrap_or_default();
1914 let completed_at_ms = self.records.get(operation_id).and_then(|record| {
1915 record
1916 .completed_at
1917 .map(|completed_at| record.epoch_millis_for_instant(completed_at))
1918 });
1919
1920 Ok(CompletionEntry {
1921 seq: authority_entry.seq,
1922 operation_id: operation_id.clone(),
1923 kind: authority_entry.kind,
1924 display_name,
1925 terminal_outcome: authority_entry.terminal_outcome.clone(),
1926 completed_at_ms,
1927 })
1928 })
1929 .collect::<Result<_, _>>()?;
1930 completion_entries.sort_by_key(|entry| entry.seq);
1931
1932 let authority_state = RegistryCanonicalState {
1933 operations,
1934 completion_feed_entries: authority_completion_entries,
1935 completed_order,
1936 max_completed: self.max_completed,
1937 max_concurrent: self.max_concurrent,
1938 active_count,
1939 wait_request_id: self.wait_request_id.clone(),
1940 wait_operation_ids: self.wait_operation_ids()?,
1941 next_completion_seq: self.dsl.0.state().next_completion_seq,
1942 };
1943
1944 Ok(PersistedOpsSnapshot {
1945 epoch_id,
1946 authority_state,
1947 operation_specs,
1948 completion_entries,
1949 cursors: self.completion_cursor_snapshot(),
1950 })
1951 }
1952
1953 fn shell_record_mut(
1954 &mut self,
1955 id: &OperationId,
1956 ) -> Result<&mut ShellRecord, OpsLifecycleError> {
1957 self.records
1958 .get_mut(id)
1959 .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
1960 }
1961
1962 fn collect_wait_outcomes(
1963 &self,
1964 operation_ids: &[OperationId],
1965 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
1966 operation_ids
1967 .iter()
1968 .map(|operation_id| {
1969 let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
1970 OpsLifecycleError::Internal(format!(
1971 "wait_all completed without terminal outcome for {operation_id}"
1972 ))
1973 })?;
1974 Ok((operation_id.clone(), outcome))
1975 })
1976 .collect()
1977 }
1978}
1979
1980impl Default for ShellState {
1981 fn default() -> Self {
1982 Self::new(DEFAULT_MAX_COMPLETED, None)
1983 }
1984}
1985
1986#[derive(Debug, Clone)]
1992pub struct OpsLifecycleConfig {
1993 pub max_completed: usize,
1995 pub max_concurrent: Option<usize>,
1997}
1998
1999impl Default for OpsLifecycleConfig {
2000 fn default() -> Self {
2001 Self {
2002 max_completed: DEFAULT_MAX_COMPLETED,
2003 max_concurrent: None,
2004 }
2005 }
2006}
2007
2008#[derive(Debug)]
2015pub struct RuntimeOpsLifecycleRegistry {
2016 state: RwLock<ShellState>,
2017}
2018
2019#[derive(Debug, Clone)]
2020pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2021 pub operation_count: usize,
2022 pub active_count: usize,
2023 pub wait_request_id: Option<WaitRequestId>,
2024 pub pending_wait_present: bool,
2025 pub pending_wait_request_id: Option<WaitRequestId>,
2026 pub wait_operation_ids: Vec<OperationId>,
2027 pub operations: Vec<OperationLifecycleSnapshot>,
2028}
2029
2030impl Default for RuntimeOpsLifecycleRegistry {
2031 fn default() -> Self {
2032 Self {
2033 state: RwLock::new(ShellState::default()),
2034 }
2035 }
2036}
2037
2038impl RuntimeOpsLifecycleRegistry {
2039 pub fn new() -> Self {
2040 let dsl = new_ops_dsl_authority();
2041 let feed_capacity = DEFAULT_MAX_COMPLETED.saturating_mul(4).max(1024);
2042 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2043 Self {
2044 state: RwLock::new(ShellState {
2045 dsl,
2046 records: HashMap::new(),
2047 pending_wait: None,
2048 completed_order: VecDeque::new(),
2049 max_completed: DEFAULT_MAX_COMPLETED,
2050 max_concurrent: None,
2051 wait_request_id: None,
2052 feed_buffer,
2053 persist_tx: None,
2054 persistence_sealed: false,
2055 persist_epoch_id: None,
2056 persist_cursor_state: None,
2057 owner_retired: false,
2058 }),
2059 }
2060 }
2061
2062 pub fn with_config(config: OpsLifecycleConfig) -> Self {
2063 Self {
2064 state: RwLock::new(ShellState::new(config.max_completed, config.max_concurrent)),
2065 }
2066 }
2067
2068 fn recover_completion_feed_entry(
2069 shell: &mut ShellState,
2070 operation_id: &OperationId,
2071 entry: &CompletionFeedCanonicalState,
2072 ) -> Result<(), OpsLifecycleError> {
2073 let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2074 let terminal_outcome_kind =
2075 mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2076 let effects = shell.dsl_apply_with_effects(
2077 mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2078 operation_id: expected_operation_id.clone(),
2079 kind: mm_dsl::OperationKind::from(entry.kind),
2080 terminal_outcome: terminal_outcome_kind,
2081 terminal_payload: entry.terminal_outcome.clone(),
2082 completion_sequence: entry.seq,
2083 },
2084 "RecoverCompletionFeedEntry",
2085 )?;
2086 let recovered = effects.iter().find_map(|effect| match effect {
2087 mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2088 operation_id,
2089 seq,
2090 kind,
2091 terminal_outcome,
2092 terminal_payload,
2093 } => Some((
2094 operation_id,
2095 *seq,
2096 OperationKind::from(*kind),
2097 *terminal_outcome,
2098 terminal_payload,
2099 )),
2100 _ => None,
2101 });
2102 let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2103 return Err(OpsLifecycleError::Internal(
2104 "generated completion-feed recovery emitted no recovered entry".into(),
2105 ));
2106 };
2107 if operation_id != &expected_operation_id
2108 || seq != entry.seq
2109 || kind != entry.kind
2110 || terminal_outcome != terminal_outcome_kind
2111 || terminal_payload != &entry.terminal_outcome
2112 {
2113 return Err(OpsLifecycleError::Internal(format!(
2114 "generated completion-feed recovery drifted for {operation_id}"
2115 )));
2116 }
2117 Ok(())
2118 }
2119
2120 pub fn set_persistence_channel(
2126 &self,
2127 tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2128 epoch_id: meerkat_core::RuntimeEpochId,
2129 cursor_state: Arc<meerkat_core::EpochCursorState>,
2130 ) {
2131 if let Ok(mut state) = self.state.write() {
2132 if state.owner_retired || state.persistence_sealed {
2133 tracing::warn!(
2134 "ignored attempt to rewire a terminally sealed ops persistence channel"
2135 );
2136 return;
2137 }
2138 state.persist_tx = Some(tx);
2139 state.persist_epoch_id = Some(epoch_id);
2140 state.persist_cursor_state = Some(cursor_state);
2141 }
2142 }
2143
2144 pub(crate) fn retire_owner_for_unregister(
2153 &self,
2154 reason: String,
2155 ) -> Result<(), OpsLifecycleError> {
2156 let closed_sender = {
2157 let mut state = self.write_state()?;
2158 if state.owner_retired {
2159 return Ok(());
2160 }
2161 terminate_owner_locked(&mut state, &reason)?;
2162 state.owner_retired = true;
2163 state.persistence_sealed = true;
2164 state.persist_epoch_id = None;
2165 state.persist_cursor_state = None;
2166 state.persist_tx.take()
2167 };
2168 drop(closed_sender);
2169 Ok(())
2170 }
2171
2172 pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2179 let PersistedOpsSnapshot {
2180 authority_state,
2181 operation_specs,
2182 completion_entries,
2183 cursors,
2184 ..
2185 } = snapshot;
2186 let max_completed = authority_state.max_completed;
2187 let max_concurrent = authority_state.max_concurrent;
2188 let next_completion_seq = authority_state.next_completion_seq;
2189 let authority_completion_entries = authority_state.completion_feed_entries;
2190 let authority_operations = authority_state.operations;
2191 let mut shell = ShellState::new(max_completed, max_concurrent);
2192
2193 let mut retained_ids: HashSet<OperationId> = HashSet::new();
2199 for (op_id, op_state) in authority_operations {
2200 let terminal_outcome = op_state
2201 .terminal_outcome
2202 .as_ref()
2203 .map(mm_dsl::OperationTerminalOutcomeKind::from);
2204 let terminal_payload = op_state.terminal_outcome.clone();
2205 let disposition = ShellState::recovered_operation_record_disposition(
2206 &op_id,
2207 op_state.status,
2208 op_state.kind,
2209 terminal_outcome.is_some(),
2210 terminal_payload.is_some(),
2211 op_state.completion_sequence.is_some(),
2212 )?;
2213 if disposition == RecoveredOperationRecordDisposition::Discard {
2214 continue;
2215 }
2216 if let Some(spec_source) = operation_specs
2217 .get(&op_id)
2218 .and_then(|spec| spec.operation_source.as_ref())
2219 && op_state.operation_source.as_ref() != Some(spec_source)
2220 {
2221 return Err(OpsLifecycleError::Internal(format!(
2222 "persisted operation source mirror for {op_id} drifted from generated authority"
2223 )));
2224 }
2225 let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2226 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2227 status: mm_dsl::OperationStatus::from(op_state.status),
2228 kind: mm_dsl::OperationKind::from(op_state.kind),
2229 source: op_state
2230 .operation_source
2231 .as_ref()
2232 .map(mm_dsl::OperationSource::from_domain),
2233 peer_ready: op_state.peer_ready,
2234 progress_count: u64::from(op_state.progress_count),
2235 terminal_outcome,
2236 terminal_payload,
2237 completion_sequence: op_state.completion_sequence,
2238 };
2239 shell.dsl_apply(recovery, "RecoverOpRecord")?;
2240 let recovered_seq = shell.completion_sequence(&op_id).ok_or_else(|| {
2241 OpsLifecycleError::Internal(format!(
2242 "generated op recovery accepted {op_id} without completion sequence"
2243 ))
2244 })?;
2245 if op_state.completion_sequence != Some(recovered_seq) {
2246 return Err(OpsLifecycleError::Internal(format!(
2247 "generated op recovery completion sequence mismatch for {op_id}"
2248 )));
2249 }
2250 retained_ids.insert(op_id);
2251 }
2252 shell.dsl_apply(
2253 mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2254 next_completion_seq,
2255 },
2256 "RecoverOpsCompletionCursor",
2257 )?;
2258 shell.dsl_apply(
2259 mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2260 agent_applied_cursor: cursors.agent_applied_cursor,
2261 runtime_observed_cursor: cursors.runtime_observed_seq,
2262 runtime_injected_cursor: cursors.runtime_last_injected_seq,
2263 },
2264 "RecoverCompletionConsumerCursors",
2265 )?;
2266
2267 let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2270 .iter()
2271 .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2272 .collect();
2273 recovered_completed.sort_by_key(|(seq, _)| *seq);
2274 shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2275
2276 for (operation_id, entry) in &authority_completion_entries {
2280 if !retained_ids.contains(operation_id) {
2281 Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2282 }
2283 }
2284
2285 let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2286 for (operation_id, entry) in &authority_completion_entries {
2287 let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2288 return Err(OpsLifecycleError::Internal(format!(
2289 "persisted completion feed authority for {operation_id} was not recovered"
2290 )));
2291 };
2292 if recovered_entry != entry {
2293 return Err(OpsLifecycleError::Internal(format!(
2294 "persisted completion feed authority drifted from generated recovery for {operation_id}"
2295 )));
2296 }
2297 }
2298
2299 let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2303 for entry in completion_entries {
2304 let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2305 return Err(OpsLifecycleError::Internal(format!(
2306 "persisted completion feed projection for {} has no generated feed authority",
2307 entry.operation_id
2308 )));
2309 };
2310 if authority_entry.seq != entry.seq
2311 || authority_entry.kind != entry.kind
2312 || authority_entry.terminal_outcome != entry.terminal_outcome
2313 {
2314 return Err(OpsLifecycleError::Internal(format!(
2315 "persisted completion feed projection for {} drifted from generated feed authority",
2316 entry.operation_id
2317 )));
2318 }
2319 projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2320 }
2321
2322 let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2323 canonical_feed_entries.into_iter().collect();
2324 recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2325 for (operation_id, entry) in recovered_entries {
2326 let projection = projection_entries_by_id.get(&operation_id);
2327 let display_name = operation_specs
2328 .get(&operation_id)
2329 .map(|spec| spec.display_name.clone())
2330 .or_else(|| projection.map(|entry| entry.display_name.clone()))
2331 .unwrap_or_default();
2332 let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2333 shell.feed_buffer.push(CompletionEntry {
2334 seq: entry.seq,
2335 operation_id,
2336 kind: entry.kind,
2337 display_name,
2338 terminal_outcome: entry.terminal_outcome,
2339 completed_at_ms,
2340 });
2341 }
2342
2343 for (op_id, spec) in operation_specs {
2346 if retained_ids.contains(&op_id) {
2347 let mut spec = spec;
2348 let operation_source = shell.operation_source(&op_id)?;
2349 ShellState::align_spec_child_session_id_to_source(
2350 &mut spec,
2351 operation_source.as_ref(),
2352 );
2353 shell.records.insert(
2354 op_id,
2355 ShellRecord {
2356 spec,
2357 peer_handle: None,
2358 watchers: Vec::new(),
2359 created_at: Instant::now(),
2360 started_at: None,
2361 completed_at: None,
2362 created_at_wall: SystemTime::now(),
2363 },
2364 );
2365 }
2366 }
2367
2368 Ok(Self {
2369 state: RwLock::new(shell),
2370 })
2371 }
2372
2373 pub fn capture_persistence_snapshot(
2378 &self,
2379 epoch_id: meerkat_core::RuntimeEpochId,
2380 cursor_state: &meerkat_core::EpochCursorState,
2381 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2382 let state = self
2383 .state
2384 .read()
2385 .unwrap_or_else(std::sync::PoisonError::into_inner);
2386 state.capture_snapshot(epoch_id, cursor_state)
2387 }
2388
2389 pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2391 let state = self
2392 .state
2393 .read()
2394 .unwrap_or_else(std::sync::PoisonError::into_inner);
2395 state.completion_cursor_snapshot()
2396 }
2397
2398 pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2400 let state = self
2401 .state
2402 .read()
2403 .unwrap_or_else(std::sync::PoisonError::into_inner);
2404 Arc::new(RuntimeCompletionFeed {
2405 buffer: Arc::clone(&state.feed_buffer),
2406 })
2407 }
2408
2409 pub(crate) fn diagnostic_snapshot(
2411 &self,
2412 ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2413 let state = self
2414 .state
2415 .read()
2416 .unwrap_or_else(std::sync::PoisonError::into_inner);
2417 let mut operations = state
2418 .operation_ids()?
2419 .into_iter()
2420 .map(|id| state.snapshot(&id))
2421 .collect::<Result<Vec<_>, _>>()?
2422 .into_iter()
2423 .flatten()
2424 .collect::<Vec<_>>();
2425 operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2426 Ok(RuntimeOpsDiagnosticSnapshot {
2427 operation_count: state.operation_count(),
2428 active_count: state.active_count(),
2429 wait_request_id: state.wait_request_id.clone(),
2430 pending_wait_present: state.pending_wait.is_some(),
2431 pending_wait_request_id: state
2432 .pending_wait
2433 .as_ref()
2434 .map(|pending_wait| pending_wait.wait_request_id.clone()),
2435 wait_operation_ids: state.wait_operation_ids()?,
2436 operations,
2437 })
2438 }
2439
2440 fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2441 self.state
2442 .read()
2443 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2444 }
2445
2446 fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2447 self.state
2448 .write()
2449 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2450 }
2451
2452 fn cancel_wait_all_internal(
2453 &self,
2454 wait_request_id: &WaitRequestId,
2455 ) -> Result<(), OpsLifecycleError> {
2456 let mut state = self.write_state()?;
2457 match state.wait_request_id.as_ref() {
2458 Some(active) if active == wait_request_id => {
2459 state.dsl_apply(
2464 mm_dsl::MeerkatMachineInput::CancelWaitAll,
2465 "CancelWaitAll(cancel)",
2466 )?;
2467 state.wait_request_id = None;
2468 if state
2469 .pending_wait
2470 .as_ref()
2471 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2472 {
2473 state.pending_wait = None;
2474 }
2475 Ok(())
2476 }
2477 _ => {
2478 if state
2479 .pending_wait
2480 .as_ref()
2481 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2482 {
2483 state.pending_wait = None;
2484 }
2485 Ok(())
2486 }
2487 }
2488 }
2489}
2490
2491enum WaitAllFutureState {
2492 Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2493 Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2494 Done,
2495}
2496
2497struct WaitAllFuture<'a> {
2498 registry: &'a RuntimeOpsLifecycleRegistry,
2499 wait_request_id: WaitRequestId,
2500 state: WaitAllFutureState,
2501}
2502
2503impl Future for WaitAllFuture<'_> {
2504 type Output = Result<WaitAllResult, OpsLifecycleError>;
2505
2506 fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2507 match &mut self.state {
2508 WaitAllFutureState::Ready(result) => {
2509 let ready = result.take().unwrap_or_else(|| {
2510 Err(OpsLifecycleError::Internal(
2511 "wait_all future polled after completion".into(),
2512 ))
2513 });
2514 self.state = WaitAllFutureState::Done;
2515 Poll::Ready(ready)
2516 }
2517 WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2518 Poll::Pending => Poll::Pending,
2519 Poll::Ready(Ok(satisfied)) => {
2520 let outcomes = match self.registry.read_state() {
2521 Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2522 Err(err) => Err(err),
2523 };
2524 self.state = WaitAllFutureState::Done;
2525 Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2526 outcomes,
2527 satisfied,
2528 }))
2529 }
2530 Poll::Ready(Err(_)) => {
2531 self.state = WaitAllFutureState::Done;
2532 Poll::Ready(Err(OpsLifecycleError::Internal(
2533 "wait_all completion channel dropped".into(),
2534 )))
2535 }
2536 },
2537 WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2538 "wait_all future polled after completion".into(),
2539 ))),
2540 }
2541 }
2542}
2543
2544impl Drop for WaitAllFuture<'_> {
2545 fn drop(&mut self) {
2546 if matches!(self.state, WaitAllFutureState::Waiting(_))
2547 && let Err(err) = self
2548 .registry
2549 .cancel_wait_all_internal(&self.wait_request_id)
2550 {
2551 tracing::error!(
2552 wait_request_id = %self.wait_request_id,
2553 error = %err,
2554 "generated wait_all authority rejected cancellation during drop"
2555 );
2556 }
2557 }
2558}
2559
2560fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2565 match action {
2566 mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2567 mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2568 mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2569 mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2570 mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2571 mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2572 mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2573 mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2574 mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2575 mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2576 }
2577}
2578
2579fn op_lifecycle_rejection_error_from_effects(
2580 id: &OperationId,
2581 requested_action: mm_dsl::OpLifecycleActionKind,
2582 effects: &[mm_dsl::MeerkatMachineEffect],
2583) -> Result<OpsLifecycleError, OpsLifecycleError> {
2584 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2585 let mut rejection = None;
2586 for effect in effects {
2587 let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2588 operation_id,
2589 action,
2590 reason,
2591 status,
2592 } = effect
2593 else {
2594 continue;
2595 };
2596 if rejection.is_some() {
2597 return Err(OpsLifecycleError::Internal(
2598 "generated op lifecycle authority emitted multiple rejection results".into(),
2599 ));
2600 }
2601 if operation_id != &expected_id || *action != requested_action {
2602 return Err(OpsLifecycleError::Internal(format!(
2603 "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2604 )));
2605 }
2606 let error = match reason {
2607 mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2608 if status.is_some() {
2609 return Err(OpsLifecycleError::Internal(
2610 "generated op lifecycle authority emitted not-found with status".into(),
2611 ));
2612 }
2613 OpsLifecycleError::NotFound(id.clone())
2614 }
2615 mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2616 let status = status.ok_or_else(|| {
2617 OpsLifecycleError::Internal(
2618 "generated op lifecycle authority emitted invalid-transition without status"
2619 .into(),
2620 )
2621 })?;
2622 OpsLifecycleError::InvalidTransition {
2623 id: id.clone(),
2624 status: OperationStatus::from(status),
2625 action: op_lifecycle_action_label(requested_action),
2626 }
2627 }
2628 mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2629 if status.is_none() {
2630 return Err(OpsLifecycleError::Internal(
2631 "generated op lifecycle authority emitted peer-not-expected without status"
2632 .into(),
2633 ));
2634 }
2635 OpsLifecycleError::PeerNotExpected(id.clone())
2636 }
2637 mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2638 if status.is_none() {
2639 return Err(OpsLifecycleError::Internal(
2640 "generated op lifecycle authority emitted already-peer-ready without status"
2641 .into(),
2642 ));
2643 }
2644 OpsLifecycleError::AlreadyPeerReady(id.clone())
2645 }
2646 };
2647 rejection = Some(error);
2648 }
2649 rejection.ok_or_else(|| {
2650 OpsLifecycleError::Internal(
2651 "generated op lifecycle authority emitted no rejection result".into(),
2652 )
2653 })
2654}
2655
2656fn classify_generated_op_rejection(
2657 state: &mut ShellState,
2658 err: mm_dsl::MeerkatMachineTransitionError,
2659 id: &OperationId,
2660 action: mm_dsl::OpLifecycleActionKind,
2661) -> OpsLifecycleError {
2662 match err {
2663 mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2664 match state.dsl_apply_with_effects(
2665 mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2666 operation_id: mm_dsl::OperationId::from_domain(id).0,
2667 action,
2668 },
2669 "ResolveOpLifecycleTransitionRejection",
2670 ) {
2671 Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2672 .unwrap_or_else(|err| err),
2673 Err(err) => err,
2674 }
2675 }
2676 other => OpsLifecycleError::Internal(format!(
2677 "DSL rejected ops transition ({}): {other:?}",
2678 op_lifecycle_action_label(action)
2679 )),
2680 }
2681}
2682
2683fn apply_op_transition(
2684 state: &mut ShellState,
2685 id: &OperationId,
2686 input: mm_dsl::MeerkatMachineInput,
2687 action: mm_dsl::OpLifecycleActionKind,
2688) -> Result<(), OpsLifecycleError> {
2689 state.ensure_owner_active()?;
2690 state
2691 .dsl_apply_raw(input)
2692 .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2693}
2694
2695fn apply_terminal_op_transition_and_persist(
2696 state: &mut ShellState,
2697 id: &OperationId,
2698 input: mm_dsl::MeerkatMachineInput,
2699 action: mm_dsl::OpLifecycleActionKind,
2700) -> Result<(), OpsLifecycleError> {
2701 let previous_snapshot = state.dsl.0.snapshot();
2702 apply_op_transition(state, id, input, action)?;
2703 if let Err(err) = state.maybe_persist() {
2704 state.dsl.0.restore_snapshot(previous_snapshot);
2705 return Err(err);
2706 }
2707 Ok(())
2708}
2709
2710fn terminate_owner_locked(state: &mut ShellState, reason: &str) -> Result<(), OpsLifecycleError> {
2711 state.ensure_owner_active()?;
2712 let to_terminate = state.owner_termination_targets()?;
2713
2714 for (op_id, _status) in &to_terminate {
2715 let terminal_outcome = OperationTerminalOutcome::Terminated {
2716 reason: reason.to_owned(),
2717 };
2718 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2719
2720 apply_terminal_op_transition_and_persist(
2721 state,
2722 op_id,
2723 mm_dsl::MeerkatMachineInput::TerminateOp {
2724 operation_id: mm_dsl::OperationId::from_domain(op_id).0,
2725 outcome: outcome_kind,
2726 payload: terminal_outcome,
2727 },
2728 mm_dsl::OpLifecycleActionKind::Terminate,
2729 )?;
2730
2731 if let Some(entry) = state.finalize_terminal(op_id)? {
2732 state.publish_completion_entry(Some(entry));
2733 }
2734 }
2735 Ok(())
2736}
2737
2738fn op_registration_error_from_effects(
2739 id: &OperationId,
2740 effects: &[mm_dsl::MeerkatMachineEffect],
2741) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2742 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2743 let mut admission = None;
2744 for effect in effects {
2745 let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2746 operation_id,
2747 result,
2748 reject_reason,
2749 max_concurrent_limit,
2750 active_op_count,
2751 } = effect
2752 else {
2753 continue;
2754 };
2755 if admission.is_some() {
2756 return Err(OpsLifecycleError::Internal(
2757 "generated op registration authority emitted multiple admission results".into(),
2758 ));
2759 }
2760 if operation_id != &expected_id {
2761 return Err(OpsLifecycleError::Internal(format!(
2762 "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2763 )));
2764 }
2765 admission = Some(match result {
2766 mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2767 if reject_reason.is_some() {
2768 return Err(OpsLifecycleError::Internal(
2769 "generated op registration authority accepted with rejection reason".into(),
2770 ));
2771 }
2772 None
2773 }
2774 mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2775 let reason = reject_reason.ok_or_else(|| {
2776 OpsLifecycleError::Internal(
2777 "generated op registration authority rejected without reason".into(),
2778 )
2779 })?;
2780 let error = match reason {
2781 mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2782 OpsLifecycleError::AlreadyRegistered(id.clone())
2783 }
2784 mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2785 let limit = max_concurrent_limit.ok_or_else(|| {
2786 OpsLifecycleError::Internal(
2787 "generated op registration authority rejected capacity without limit"
2788 .into(),
2789 )
2790 })?;
2791 OpsLifecycleError::MaxConcurrentExceeded {
2792 limit: limit as usize,
2793 active: *active_op_count as usize,
2794 }
2795 }
2796 };
2797 Some(error)
2798 }
2799 });
2800 }
2801 admission.ok_or_else(|| {
2802 OpsLifecycleError::Internal(
2803 "generated op registration authority emitted no admission result".into(),
2804 )
2805 })
2806}
2807
2808impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2809 fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2810 self.register_operation_with_admission_limit(spec, None)
2811 }
2812
2813 fn register_operation_with_admission_limit(
2814 &self,
2815 mut spec: OperationSpec,
2816 max_concurrent: Option<usize>,
2817 ) -> Result<(), OpsLifecycleError> {
2818 let mut state = self.write_state()?;
2819 state.ensure_owner_active()?;
2820 let operation_id = spec.id.clone();
2821 let kind = spec.kind;
2822 let max_concurrent = max_concurrent
2823 .or(state.max_concurrent)
2824 .map(|limit| limit as u64);
2825
2826 let effects = state.dsl_apply_with_effects(
2827 mm_dsl::MeerkatMachineInput::RegisterOp {
2828 operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2829 kind: mm_dsl::OperationKind::from_domain(&kind),
2830 source: spec
2831 .operation_source
2832 .as_ref()
2833 .map(mm_dsl::OperationSource::from_domain),
2834 max_concurrent,
2835 },
2836 "RegisterOp",
2837 )?;
2838 if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
2839 return Err(error);
2840 }
2841
2842 let authority_operation_source = state.operation_source(&operation_id)?;
2843 ShellState::align_spec_child_session_id_to_source(
2844 &mut spec,
2845 authority_operation_source.as_ref(),
2846 );
2847
2848 state.records.insert(operation_id, ShellRecord::new(spec));
2850 Ok(())
2851 }
2852
2853 fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2854 let mut state = self.write_state()?;
2855
2856 apply_op_transition(
2857 &mut state,
2858 id,
2859 mm_dsl::MeerkatMachineInput::StartOp {
2860 operation_id: mm_dsl::OperationId::from_domain(id).0,
2861 },
2862 mm_dsl::OpLifecycleActionKind::Start,
2863 )?;
2864
2865 if let Some(shell) = state.records.get_mut(id) {
2867 shell.started_at = Some(Instant::now());
2868 }
2869 Ok(())
2870 }
2871
2872 fn provisioning_failed(
2873 &self,
2874 id: &OperationId,
2875 error: String,
2876 ) -> Result<(), OpsLifecycleError> {
2877 let mut state = self.write_state()?;
2878
2879 let terminal_outcome = OperationTerminalOutcome::Failed { error };
2880 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2881
2882 apply_terminal_op_transition_and_persist(
2883 &mut state,
2884 id,
2885 mm_dsl::MeerkatMachineInput::FailOp {
2886 operation_id: mm_dsl::OperationId::from_domain(id).0,
2887 outcome: outcome_kind,
2888 payload: terminal_outcome,
2889 },
2890 mm_dsl::OpLifecycleActionKind::Fail,
2891 )?;
2892
2893 let completion_entry = state.finalize_terminal(id)?;
2894 state.publish_completion_entry(completion_entry);
2895 Ok(())
2896 }
2897
2898 fn peer_ready(
2899 &self,
2900 id: &OperationId,
2901 peer: OperationPeerHandle,
2902 ) -> Result<(), OpsLifecycleError> {
2903 let mut state = self.write_state()?;
2904
2905 apply_op_transition(
2906 &mut state,
2907 id,
2908 mm_dsl::MeerkatMachineInput::PeerReadyOp {
2909 operation_id: mm_dsl::OperationId::from_domain(id).0,
2910 },
2911 mm_dsl::OpLifecycleActionKind::PeerReady,
2912 )?;
2913
2914 if let Some(shell) = state.records.get_mut(id) {
2916 shell.peer_handle = Some(peer);
2917 }
2918 Ok(())
2919 }
2920
2921 fn register_watcher(
2922 &self,
2923 id: &OperationId,
2924 ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
2925 let mut state = self.write_state()?;
2926
2927 if !state.contains(id) {
2928 return Err(OpsLifecycleError::NotFound(id.clone()));
2929 }
2930
2931 if let Some(outcome) = state.terminal_outcome(id)? {
2933 return Ok(resolved_operation_completion_watch(outcome));
2934 }
2935
2936 let shell = state.shell_record_mut(id)?;
2938 let (tx, rx) = tokio::sync::oneshot::channel();
2939 let watch = operation_completion_watch_from_receiver(rx);
2940 shell.watchers.push(OperationCompletionNotifier::new(tx));
2941 Ok(watch)
2942 }
2943
2944 fn report_progress(
2945 &self,
2946 id: &OperationId,
2947 _update: OperationProgressUpdate,
2948 ) -> Result<(), OpsLifecycleError> {
2949 let mut state = self.write_state()?;
2950
2951 apply_op_transition(
2952 &mut state,
2953 id,
2954 mm_dsl::MeerkatMachineInput::ProgressReportedOp {
2955 operation_id: mm_dsl::OperationId::from_domain(id).0,
2956 },
2957 mm_dsl::OpLifecycleActionKind::ProgressReported,
2958 )?;
2959 Ok(())
2960 }
2961
2962 fn complete_operation(
2963 &self,
2964 id: &OperationId,
2965 result: OperationResult,
2966 ) -> Result<(), OpsLifecycleError> {
2967 let mut state = self.write_state()?;
2968
2969 let terminal_outcome = OperationTerminalOutcome::Completed(result);
2970 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2971
2972 apply_terminal_op_transition_and_persist(
2973 &mut state,
2974 id,
2975 mm_dsl::MeerkatMachineInput::CompleteOp {
2976 operation_id: mm_dsl::OperationId::from_domain(id).0,
2977 outcome: outcome_kind,
2978 payload: terminal_outcome,
2979 },
2980 mm_dsl::OpLifecycleActionKind::Complete,
2981 )?;
2982
2983 let completion_entry = state.finalize_terminal(id)?;
2984 state.publish_completion_entry(completion_entry);
2985 Ok(())
2986 }
2987
2988 fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
2989 let mut state = self.write_state()?;
2990
2991 let terminal_outcome = OperationTerminalOutcome::Failed { error };
2992 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2993
2994 apply_terminal_op_transition_and_persist(
2995 &mut state,
2996 id,
2997 mm_dsl::MeerkatMachineInput::FailOp {
2998 operation_id: mm_dsl::OperationId::from_domain(id).0,
2999 outcome: outcome_kind,
3000 payload: terminal_outcome,
3001 },
3002 mm_dsl::OpLifecycleActionKind::Fail,
3003 )?;
3004
3005 let completion_entry = state.finalize_terminal(id)?;
3006 state.publish_completion_entry(completion_entry);
3007 Ok(())
3008 }
3009
3010 fn abort_provisioning(
3011 &self,
3012 id: &OperationId,
3013 reason: Option<String>,
3014 ) -> Result<(), OpsLifecycleError> {
3015 let mut state = self.write_state()?;
3016
3017 let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
3018 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3019
3020 apply_terminal_op_transition_and_persist(
3021 &mut state,
3022 id,
3023 mm_dsl::MeerkatMachineInput::AbortOp {
3024 operation_id: mm_dsl::OperationId::from_domain(id).0,
3025 outcome: outcome_kind,
3026 payload: terminal_outcome,
3027 },
3028 mm_dsl::OpLifecycleActionKind::Abort,
3029 )?;
3030
3031 let completion_entry = state.finalize_terminal(id)?;
3032 state.publish_completion_entry(completion_entry);
3033 Ok(())
3034 }
3035
3036 fn cancel_operation(
3037 &self,
3038 id: &OperationId,
3039 reason: Option<String>,
3040 ) -> Result<(), OpsLifecycleError> {
3041 let mut state = self.write_state()?;
3042
3043 let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
3044 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3045
3046 apply_terminal_op_transition_and_persist(
3047 &mut state,
3048 id,
3049 mm_dsl::MeerkatMachineInput::CancelOp {
3050 operation_id: mm_dsl::OperationId::from_domain(id).0,
3051 outcome: outcome_kind,
3052 payload: terminal_outcome,
3053 },
3054 mm_dsl::OpLifecycleActionKind::Cancel,
3055 )?;
3056
3057 let completion_entry = state.finalize_terminal(id)?;
3058 state.publish_completion_entry(completion_entry);
3059 Ok(())
3060 }
3061
3062 fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3063 let mut state = self.write_state()?;
3064
3065 apply_op_transition(
3066 &mut state,
3067 id,
3068 mm_dsl::MeerkatMachineInput::RetireRequestedOp {
3069 operation_id: mm_dsl::OperationId::from_domain(id).0,
3070 },
3071 mm_dsl::OpLifecycleActionKind::RetireRequested,
3072 )?;
3073 Ok(())
3074 }
3075
3076 fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3077 let mut state = self.write_state()?;
3078
3079 let terminal_outcome = OperationTerminalOutcome::Retired;
3080 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3081
3082 apply_terminal_op_transition_and_persist(
3083 &mut state,
3084 id,
3085 mm_dsl::MeerkatMachineInput::RetireCompletedOp {
3086 operation_id: mm_dsl::OperationId::from_domain(id).0,
3087 outcome: outcome_kind,
3088 payload: terminal_outcome,
3089 },
3090 mm_dsl::OpLifecycleActionKind::RetireCompleted,
3091 )?;
3092
3093 let completion_entry = state.finalize_terminal(id)?;
3094 state.publish_completion_entry(completion_entry);
3095 Ok(())
3096 }
3097
3098 fn snapshot(
3099 &self,
3100 id: &OperationId,
3101 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3102 let state = self.read_state()?;
3103 state.snapshot(id)
3104 }
3105
3106 fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3107 let state = self.read_state()?;
3108 let mut snapshots = Vec::new();
3109 for id in state.operation_ids()? {
3110 let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3111 OpsLifecycleError::Internal(format!(
3112 "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3113 ))
3114 })?;
3115 snapshots.push(snapshot);
3116 }
3117 snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3118 Ok(snapshots)
3119 }
3120
3121 fn classify_operation_terminality(
3122 &self,
3123 id: &OperationId,
3124 status: OperationStatus,
3125 ) -> Result<bool, OpsLifecycleError> {
3126 ShellState::operation_status_is_terminal(id, status)
3127 }
3128
3129 fn classify_operation_public_result(
3130 &self,
3131 id: &OperationId,
3132 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3133 let state = self.read_state()?;
3134 let status = match state.status(id) {
3135 Some(status) => status,
3136 None if state.records.contains_key(id)
3137 || state.has_generated_operation_record_fact(id) =>
3138 {
3139 return Err(OpsLifecycleError::Internal(format!(
3140 "generated op lifecycle authority missing status for {id}"
3141 )));
3142 }
3143 None => OperationStatus::Absent,
3144 };
3145 ShellState::operation_public_result_class(id, status)
3146 }
3147
3148 fn classify_operation_completion_wake(
3149 &self,
3150 id: &OperationId,
3151 kind: OperationKind,
3152 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3153 ShellState::operation_completion_wake_class(id, kind)
3154 }
3155
3156 fn classify_operation_transition_idempotence(
3157 &self,
3158 id: &OperationId,
3159 action: OperationLifecycleAction,
3160 ) -> Result<bool, OpsLifecycleError> {
3161 let state = self.read_state()?;
3162 let status = match state.status(id) {
3163 Some(status) => status,
3164 None if state.records.contains_key(id)
3165 || state.has_generated_operation_record_fact(id) =>
3166 {
3167 return Err(OpsLifecycleError::Internal(format!(
3168 "generated op lifecycle authority missing status for {id}"
3169 )));
3170 }
3171 None => OperationStatus::Absent,
3172 };
3173 ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3174 }
3175
3176 fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3177 let mut state = self.write_state()?;
3178 if state.owner_retired {
3179 return Ok(());
3180 }
3181 terminate_owner_locked(&mut state, &reason)
3182 }
3183
3184 fn collect_completed(
3185 &self,
3186 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3187 let mut state = self.write_state()?;
3188 state.ensure_owner_active()?;
3189
3190 let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3191 let mut collected = Vec::with_capacity(ids.len());
3192 for id in ids {
3193 let outcome = state.terminal_outcome(&id)?;
3194 state.dsl_apply(
3195 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3196 operation_id: mm_dsl::OperationId::from_domain(&id).0,
3197 },
3198 "CollectCompletedOp",
3199 )?;
3200 state.completed_order.retain(|queued| queued != &id);
3201 state.records.remove(&id);
3202 if let Some(outcome) = outcome {
3203 collected.push((id, outcome));
3204 }
3205 }
3206 Ok(collected)
3207 }
3208
3209 fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3210 Some(self.completion_feed_handle())
3211 }
3212
3213 fn completion_cursor(
3214 &self,
3215 consumer: CompletionCursorConsumer,
3216 ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3217 let state = self.read_state()?;
3218 Ok(Some(state.completion_cursor(consumer)))
3219 }
3220
3221 fn advance_completion_cursor(
3222 &self,
3223 consumer: CompletionCursorConsumer,
3224 cursor: CompletionSeq,
3225 projection: Option<&meerkat_core::EpochCursorState>,
3226 ) -> Result<CompletionSeq, OpsLifecycleError> {
3227 let mut state = self.write_state()?;
3228 state.ensure_owner_active()?;
3229 let input = match consumer {
3230 CompletionCursorConsumer::AgentApplied => {
3231 mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3232 }
3233 CompletionCursorConsumer::RuntimeObserved => {
3234 mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3235 }
3236 CompletionCursorConsumer::RuntimeInjected => {
3237 mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3238 }
3239 };
3240 let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3241 let advanced = effects
3242 .iter()
3243 .find_map(|effect| match (consumer, effect) {
3244 (
3245 CompletionCursorConsumer::AgentApplied,
3246 mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3247 ) => Some(*cursor),
3248 (
3249 CompletionCursorConsumer::RuntimeObserved,
3250 mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3251 cursor,
3252 },
3253 ) => Some(*cursor),
3254 (
3255 CompletionCursorConsumer::RuntimeInjected,
3256 mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3257 cursor,
3258 },
3259 ) => Some(*cursor),
3260 _ => None,
3261 })
3262 .ok_or_else(|| {
3263 OpsLifecycleError::Internal(format!(
3264 "generated completion cursor transition emitted no feedback for {consumer:?}"
3265 ))
3266 })?;
3267 if let Some(projection) = projection {
3268 projection.project_authorized_completion_cursor(consumer, advanced);
3269 }
3270 Ok(advanced)
3271 }
3272
3273 fn wait_all(
3274 &self,
3275 run_id: &RunId,
3276 ids: &[OperationId],
3277 ) -> std::pin::Pin<
3278 Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3279 > {
3280 let wait_request_id = WaitRequestId::new();
3281 let owned_ids = ids.to_vec();
3282
3283 let state = match self.write_state() {
3284 Ok(mut state) => {
3285 if let Err(error) = state.ensure_owner_active() {
3286 return Box::pin(WaitAllFuture {
3287 registry: self,
3288 wait_request_id,
3289 state: WaitAllFutureState::Ready(Some(Err(error))),
3290 });
3291 }
3292 match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3293 Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3294 let outcomes =
3295 state
3296 .collect_wait_outcomes(&satisfied.operation_ids)
3297 .map(|outcomes| WaitAllResult {
3298 outcomes,
3299 satisfied,
3300 });
3301 WaitAllFutureState::Ready(Some(outcomes))
3302 }
3303 Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3304 if state.pending_wait.is_some() {
3305 let rollback = state.dsl_apply(
3311 mm_dsl::MeerkatMachineInput::CancelWaitAll,
3312 "CancelWaitAll(rollback)",
3313 );
3314 return Box::pin(WaitAllFuture {
3315 registry: self,
3316 wait_request_id,
3317 state: WaitAllFutureState::Ready(Some(Err(match rollback {
3318 Ok(()) => OpsLifecycleError::Internal(
3319 "wait_all started while a pending wait sender already existed"
3320 .into(),
3321 ),
3322 Err(err) => err,
3323 }))),
3324 });
3325 }
3326 state.wait_request_id = Some(wait_request_id.clone());
3327 let (sender, receiver) = tokio::sync::oneshot::channel();
3328 state.pending_wait = Some(PendingWaitState {
3329 wait_request_id: wait_request_id.clone(),
3330 sender,
3331 });
3332 WaitAllFutureState::Waiting(receiver)
3333 }
3334 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3335 }
3336 }
3337 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3338 };
3339
3340 Box::pin(WaitAllFuture {
3341 registry: self,
3342 wait_request_id,
3343 state,
3344 })
3345 }
3346}
3347
3348#[cfg(test)]
3349#[allow(clippy::unwrap_used, clippy::panic)]
3350mod tests {
3351 use super::*;
3352 use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3353 use meerkat_core::lifecycle::RunId;
3354 use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3355 use meerkat_core::types::SessionId;
3356 use std::sync::atomic::Ordering;
3357 use uuid::Uuid;
3358
3359 fn test_run_id() -> RunId {
3360 RunId(Uuid::from_u128(1))
3361 }
3362
3363 fn background_spec(name: &str) -> OperationSpec {
3364 OperationSpec {
3365 id: OperationId::new(),
3366 kind: OperationKind::BackgroundToolOp,
3367 owner_session_id: SessionId::new(),
3368 display_name: name.into(),
3369 source_label: "test".into(),
3370 operation_source: None,
3371 child_session_id: None,
3372 expect_peer_channel: false,
3373 }
3374 }
3375
3376 #[tokio::test]
3377 async fn late_watchers_resolve_immediately() {
3378 let registry = RuntimeOpsLifecycleRegistry::new();
3379 let spec = background_spec("late");
3380 let op_id = spec.id.clone();
3381 registry.register_operation(spec).unwrap();
3382 registry.provisioning_succeeded(&op_id).unwrap();
3383 registry
3384 .complete_operation(
3385 &op_id,
3386 OperationResult {
3387 id: op_id.clone(),
3388 content: "done".into(),
3389 is_error: false,
3390 duration_ms: 1,
3391 tokens_used: 0,
3392 },
3393 )
3394 .unwrap();
3395
3396 let watch = registry.register_watcher(&op_id).unwrap();
3397 match watch
3398 .await
3399 .expect("operation completion watch should resolve")
3400 {
3401 OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3402 other => panic!("expected completed outcome, got {other:?}"),
3403 }
3404 }
3405
3406 #[tokio::test]
3407 async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3408 let (tx, rx) = tokio::sync::oneshot::channel();
3409 let watch = operation_completion_watch_from_receiver(rx);
3410 drop(tx);
3411
3412 assert_eq!(
3413 watch.await,
3414 Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3415 );
3416 }
3417
3418 #[test]
3419 fn peer_ready_requires_peer_expectation() {
3420 let registry = RuntimeOpsLifecycleRegistry::new();
3421 let spec = background_spec("no-peer");
3422 let op_id = spec.id.clone();
3423 registry.register_operation(spec).unwrap();
3424 registry.provisioning_succeeded(&op_id).unwrap();
3425
3426 let result = registry.peer_ready(
3427 &op_id,
3428 OperationPeerHandle {
3429 peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3430 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3431 "peer",
3432 PeerId::new(),
3433 "inproc://peer",
3434 )
3435 .unwrap(),
3436 },
3437 );
3438 assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3439 }
3440
3441 #[test]
3447 fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3448 let op_id = OperationId::new();
3449 let outcomes = vec![
3450 (
3451 OperationTerminalOutcome::Completed(OperationResult {
3452 id: op_id.clone(),
3453 content: "done".into(),
3454 is_error: false,
3455 duration_ms: 7,
3456 tokens_used: 42,
3457 }),
3458 mm_dsl::OperationTerminalOutcomeKind::Completed,
3459 ),
3460 (
3461 OperationTerminalOutcome::Failed {
3462 error: "boom".into(),
3463 },
3464 mm_dsl::OperationTerminalOutcomeKind::Failed,
3465 ),
3466 (
3467 OperationTerminalOutcome::Aborted {
3468 reason: Some("user aborted".into()),
3469 },
3470 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3471 ),
3472 (
3473 OperationTerminalOutcome::Aborted { reason: None },
3474 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3475 ),
3476 (
3477 OperationTerminalOutcome::Cancelled {
3478 reason: Some("cancelled".into()),
3479 },
3480 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3481 ),
3482 (
3483 OperationTerminalOutcome::Cancelled { reason: None },
3484 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3485 ),
3486 (
3487 OperationTerminalOutcome::Retired,
3488 mm_dsl::OperationTerminalOutcomeKind::Retired,
3489 ),
3490 (
3491 OperationTerminalOutcome::Terminated {
3492 reason: "owner stopped".into(),
3493 },
3494 mm_dsl::OperationTerminalOutcomeKind::Terminated,
3495 ),
3496 ];
3497
3498 for (outcome, expected_kind) in &outcomes {
3499 assert_eq!(
3500 mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3501 *expected_kind,
3502 "typed payload {outcome:?} must classify to {expected_kind:?}"
3503 );
3504 let read = ShellState::checked_terminal_payload(
3505 *expected_kind,
3506 outcome,
3507 "test authority",
3508 "test-op",
3509 )
3510 .expect("matching discriminant must read back the exact payload");
3511 assert_eq!(&read, outcome);
3512 }
3513
3514 let err = ShellState::checked_terminal_payload(
3516 mm_dsl::OperationTerminalOutcomeKind::Completed,
3517 &OperationTerminalOutcome::Retired,
3518 "test authority",
3519 "test-op",
3520 )
3521 .expect_err("variant mismatch must be rejected");
3522 assert!(matches!(err, OpsLifecycleError::Internal(_)));
3523 }
3524
3525 #[test]
3531 fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3532 let registry = RuntimeOpsLifecycleRegistry::new();
3533 let spec = background_spec("variant-mismatch");
3534 let op_id = spec.id.clone();
3535 registry.register_operation(spec).unwrap();
3536 registry.provisioning_succeeded(&op_id).unwrap();
3537
3538 let mut state = registry.write_state().unwrap();
3539 let err = state
3540 .dsl_apply(
3541 mm_dsl::MeerkatMachineInput::CompleteOp {
3542 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3543 outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3544 payload: OperationTerminalOutcome::Retired,
3546 },
3547 "CompleteOp",
3548 )
3549 .expect_err("machine must reject payload variant mismatch");
3550 assert!(
3557 matches!(
3558 &err,
3559 OpsLifecycleError::Internal(message)
3560 if message.contains("GuardRejected") && message.contains("CompleteOp")
3561 ),
3562 "expected generated guard rejection for CompleteOp, got: {err:?}"
3563 );
3564 drop(state);
3565
3566 registry
3569 .complete_operation(
3570 &op_id,
3571 OperationResult {
3572 id: op_id.clone(),
3573 content: "done".into(),
3574 is_error: false,
3575 duration_ms: 1,
3576 tokens_used: 0,
3577 },
3578 )
3579 .expect("matching variant must complete");
3580 }
3581
3582 #[test]
3583 fn duplicate_registration_rejection_is_generated() {
3584 let registry = RuntimeOpsLifecycleRegistry::new();
3585 let spec = background_spec("duplicate");
3586 let op_id = spec.id.clone();
3587
3588 registry.register_operation(spec.clone()).unwrap();
3589 let result = registry.register_operation(spec);
3590
3591 assert!(matches!(
3592 result,
3593 Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3594 ));
3595 }
3596
3597 #[test]
3598 fn invalid_transition_rejection_is_generated() {
3599 let registry = RuntimeOpsLifecycleRegistry::new();
3600 let spec = background_spec("invalid-transition");
3601 let op_id = spec.id.clone();
3602 registry.register_operation(spec).unwrap();
3603
3604 let result = registry.complete_operation(
3605 &op_id,
3606 OperationResult {
3607 id: op_id.clone(),
3608 content: "too-early".into(),
3609 is_error: false,
3610 duration_ms: 1,
3611 tokens_used: 0,
3612 },
3613 );
3614
3615 assert!(matches!(
3616 result,
3617 Err(OpsLifecycleError::InvalidTransition {
3618 id,
3619 status: OperationStatus::Provisioning,
3620 action: "complete_operation",
3621 }) if id == op_id
3622 ));
3623 }
3624
3625 #[tokio::test]
3626 async fn multi_listener_completion() {
3627 let registry = RuntimeOpsLifecycleRegistry::new();
3628 let spec = background_spec("multi");
3629 let op_id = spec.id.clone();
3630 registry.register_operation(spec).unwrap();
3631 registry.provisioning_succeeded(&op_id).unwrap();
3632
3633 let watch1 = registry.register_watcher(&op_id).unwrap();
3634 let watch2 = registry.register_watcher(&op_id).unwrap();
3635 let watch3 = registry.register_watcher(&op_id).unwrap();
3636
3637 registry
3638 .complete_operation(
3639 &op_id,
3640 OperationResult {
3641 id: op_id.clone(),
3642 content: "multi-done".into(),
3643 is_error: false,
3644 duration_ms: 1,
3645 tokens_used: 0,
3646 },
3647 )
3648 .unwrap();
3649
3650 for watch in [watch1, watch2, watch3] {
3651 match watch
3652 .await
3653 .expect("operation completion watch should resolve")
3654 {
3655 OperationTerminalOutcome::Completed(result) => {
3656 assert_eq!(result.content, "multi-done");
3657 }
3658 other => panic!("expected completed, got {other:?}"),
3659 }
3660 }
3661 }
3662
3663 #[tokio::test]
3664 async fn wait_all_returns_all_outcomes() {
3665 let registry = RuntimeOpsLifecycleRegistry::new();
3666
3667 let spec_a = background_spec("a");
3668 let id_a = spec_a.id.clone();
3669 registry.register_operation(spec_a).unwrap();
3670 registry.provisioning_succeeded(&id_a).unwrap();
3671
3672 let spec_b = background_spec("b");
3673 let id_b = spec_b.id.clone();
3674 registry.register_operation(spec_b).unwrap();
3675 registry.provisioning_succeeded(&id_b).unwrap();
3676
3677 registry
3678 .complete_operation(
3679 &id_a,
3680 OperationResult {
3681 id: id_a.clone(),
3682 content: "a-done".into(),
3683 is_error: false,
3684 duration_ms: 1,
3685 tokens_used: 0,
3686 },
3687 )
3688 .unwrap();
3689 registry.fail_operation(&id_b, "b-error".into()).unwrap();
3690
3691 let wait_result = registry
3692 .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3693 .await
3694 .unwrap();
3695 assert_eq!(wait_result.outcomes.len(), 2);
3696 assert_eq!(wait_result.outcomes[0].0, id_a);
3697 assert!(matches!(
3698 wait_result.outcomes[0].1,
3699 OperationTerminalOutcome::Completed(_)
3700 ));
3701 assert_eq!(wait_result.outcomes[1].0, id_b);
3702 assert!(matches!(
3703 wait_result.outcomes[1].1,
3704 OperationTerminalOutcome::Failed { .. }
3705 ));
3706 assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3708 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3709 }
3710
3711 #[tokio::test]
3714 async fn wait_all_trait_path_submits_through_authority() {
3715 let registry = RuntimeOpsLifecycleRegistry::new();
3716 let spec = background_spec("trait-wait");
3717 let op_id = spec.id.clone();
3718 registry.register_operation(spec).unwrap();
3719 registry.provisioning_succeeded(&op_id).unwrap();
3720 registry
3721 .complete_operation(
3722 &op_id,
3723 OperationResult {
3724 id: op_id.clone(),
3725 content: "done".into(),
3726 is_error: false,
3727 duration_ms: 1,
3728 tokens_used: 0,
3729 },
3730 )
3731 .unwrap();
3732
3733 let trait_ref: &dyn OpsLifecycleRegistry = ®istry;
3735 let wait_result = trait_ref
3736 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3737 .await
3738 .unwrap();
3739 assert_eq!(wait_result.outcomes.len(), 1);
3740 assert!(matches!(
3741 wait_result.outcomes[0].1,
3742 OperationTerminalOutcome::Completed(_)
3743 ));
3744 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3746 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3747 let state = registry.read_state().unwrap();
3748 assert!(
3749 !state.wait_active(),
3750 "already-satisfied wait_all must be cleared by generated satisfaction authority"
3751 );
3752 assert!(state.wait_operation_ids().unwrap().is_empty());
3753 }
3754
3755 #[tokio::test]
3756 async fn wait_all_duplicate_rejection_is_generated() {
3757 let registry = RuntimeOpsLifecycleRegistry::new();
3758 let spec = background_spec("duplicate-wait");
3759 let op_id = spec.id.clone();
3760 registry.register_operation(spec).unwrap();
3761 registry.provisioning_succeeded(&op_id).unwrap();
3762
3763 let result = registry
3764 .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3765 .await;
3766
3767 assert!(matches!(
3768 result,
3769 Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3770 ));
3771 let state = registry.read_state().unwrap();
3772 assert!(
3773 !state.wait_active(),
3774 "duplicate wait rejection must not create a shell or machine barrier"
3775 );
3776 assert!(state.wait_operation_ids().unwrap().is_empty());
3777 }
3778
3779 #[tokio::test]
3780 async fn wait_all_active_rejection_is_generated() {
3781 let registry = RuntimeOpsLifecycleRegistry::new();
3782 let spec = background_spec("active-wait");
3783 let op_id = spec.id.clone();
3784 registry.register_operation(spec).unwrap();
3785 registry.provisioning_succeeded(&op_id).unwrap();
3786
3787 let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3788 let result = registry
3789 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3790 .await;
3791
3792 assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3793 drop(active_wait);
3794 let state = registry.read_state().unwrap();
3795 assert!(!state.wait_active());
3796 assert!(state.wait_operation_ids().unwrap().is_empty());
3797 }
3798
3799 #[tokio::test]
3800 async fn wait_all_unknown_operation_rejection_is_generated() {
3801 let registry = RuntimeOpsLifecycleRegistry::new();
3802 let op_id = OperationId::new();
3803
3804 let result = registry
3805 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3806 .await;
3807
3808 assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3809 let state = registry.read_state().unwrap();
3810 assert!(!state.wait_active());
3811 assert!(state.wait_operation_ids().unwrap().is_empty());
3812 }
3813
3814 #[tokio::test]
3815 async fn wait_all_resolves_from_authority_owned_wait_request() {
3816 let registry = RuntimeOpsLifecycleRegistry::new();
3817 let run_id = test_run_id();
3818
3819 let spec = background_spec("pending");
3820 let op_id = spec.id.clone();
3821 registry.register_operation(spec).unwrap();
3822 registry.provisioning_succeeded(&op_id).unwrap();
3823
3824 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3825 tokio::pin!(wait_fut);
3826 assert!(
3827 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3828 .await
3829 .is_err()
3830 );
3831
3832 let active_wait_request_id = {
3833 let state = registry.read_state().unwrap();
3834 let wait_request_id = match state.wait_request_id.clone() {
3835 Some(wait_request_id) => wait_request_id,
3836 None => panic!("wait request should be active"),
3837 };
3838 assert_eq!(
3839 state.wait_operation_ids().unwrap().as_slice(),
3840 std::slice::from_ref(&op_id)
3841 );
3842 wait_request_id
3843 };
3844
3845 registry
3846 .complete_operation(
3847 &op_id,
3848 OperationResult {
3849 id: op_id.clone(),
3850 content: "done".into(),
3851 is_error: false,
3852 duration_ms: 1,
3853 tokens_used: 0,
3854 },
3855 )
3856 .unwrap();
3857
3858 let wait_result = wait_fut.await.unwrap();
3859 assert_eq!(
3860 wait_result.satisfied.wait_request_id,
3861 active_wait_request_id
3862 );
3863 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
3864 assert!(matches!(
3865 wait_result.outcomes.as_slice(),
3866 [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
3867 ));
3868 assert!(registry.read_state().unwrap().wait_request_id.is_none());
3869 }
3870
3871 #[tokio::test]
3872 async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
3873 let registry = RuntimeOpsLifecycleRegistry::new();
3874 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3875 registry.set_persistence_channel(
3876 tx,
3877 meerkat_core::RuntimeEpochId::new(),
3878 Arc::new(meerkat_core::EpochCursorState::new()),
3879 );
3880 let worker = std::thread::spawn(move || {
3881 let request = rx.blocking_recv().expect("persistence request");
3882 let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
3883 "injected persistence failure".to_string(),
3884 )));
3885 });
3886
3887 let spec = background_spec("feed-after-persist");
3888 let op_id = spec.id.clone();
3889 registry.register_operation(spec).unwrap();
3890 registry.provisioning_succeeded(&op_id).unwrap();
3891 let watch = registry.register_watcher(&op_id).unwrap();
3892
3893 let err = registry
3894 .complete_operation(
3895 &op_id,
3896 OperationResult {
3897 id: op_id.clone(),
3898 content: "done".into(),
3899 is_error: false,
3900 duration_ms: 1,
3901 tokens_used: 0,
3902 },
3903 )
3904 .expect_err("persistence failure must fail the terminal transition");
3905 assert!(
3906 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
3907 "unexpected persistence error: {err:?}"
3908 );
3909 worker.join().expect("persistence worker");
3910
3911 let feed = registry.completion_feed_handle();
3912 let batch = feed.list_since(0);
3913 assert!(
3914 batch.entries.is_empty(),
3915 "completion feed must not publish entries before durable snapshot success"
3916 );
3917 assert_eq!(batch.watermark, 0);
3918 assert!(
3919 tokio::time::timeout(std::time::Duration::from_millis(10), watch)
3920 .await
3921 .is_err(),
3922 "watchers must not resolve before durable terminal snapshot success"
3923 );
3924 assert_eq!(
3925 registry.snapshot(&op_id).unwrap().unwrap().status,
3926 OperationStatus::Running,
3927 "failed persistence must roll the generated terminal transition back"
3928 );
3929
3930 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3931 registry.set_persistence_channel(
3932 tx,
3933 meerkat_core::RuntimeEpochId::new(),
3934 Arc::new(meerkat_core::EpochCursorState::new()),
3935 );
3936 let worker = std::thread::spawn(move || {
3937 let request = rx.blocking_recv().expect("second persistence request");
3938 let _ = request.result_tx.send(Ok(()));
3939 });
3940 registry
3941 .complete_operation(
3942 &op_id,
3943 OperationResult {
3944 id: op_id.clone(),
3945 content: "done".into(),
3946 is_error: false,
3947 duration_ms: 1,
3948 tokens_used: 0,
3949 },
3950 )
3951 .expect("rolled-back terminal transition should remain retryable");
3952 worker.join().expect("second persistence worker");
3953
3954 let batch = feed.list_since(0);
3955 assert_eq!(batch.entries.len(), 1);
3956 assert_eq!(batch.entries[0].operation_id, op_id);
3957 }
3958
3959 #[tokio::test]
3960 async fn dropping_wait_all_future_cancels_active_wait_request() {
3961 let registry = RuntimeOpsLifecycleRegistry::new();
3962 let run_id = test_run_id();
3963
3964 let spec = background_spec("cancelled-wait");
3965 let op_id = spec.id.clone();
3966 registry.register_operation(spec).unwrap();
3967 registry.provisioning_succeeded(&op_id).unwrap();
3968
3969 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3970 drop(wait_fut);
3971
3972 let state = registry.read_state().unwrap();
3973 assert!(state.wait_request_id.is_none());
3974 assert!(state.wait_operation_ids().unwrap().is_empty());
3975 assert!(!state.wait_active());
3976 }
3977
3978 #[tokio::test]
3984 async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
3985 let registry = RuntimeOpsLifecycleRegistry::new();
3986 let run_id = test_run_id();
3987
3988 let spec = background_spec("corrupt-barrier");
3989 let op_id = spec.id.clone();
3990 registry.register_operation(spec).unwrap();
3991 registry.provisioning_succeeded(&op_id).unwrap();
3992
3993 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3995 tokio::pin!(wait_fut);
3996 assert!(
3997 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3998 .await
3999 .is_err(),
4000 "barrier waiter must still be pending before corruption"
4001 );
4002 assert!(registry.read_state().unwrap().wait_request_id.is_some());
4003
4004 {
4008 let mut state = registry.write_state().unwrap();
4009 let mut machine_state = state.dsl.0.state().clone();
4010 machine_state.wait_run_id = None;
4011 state.dsl = DslAuthority(Box::new(
4012 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4013 ));
4014 }
4015
4016 let err = registry
4019 .complete_operation(
4020 &op_id,
4021 OperationResult {
4022 id: op_id.clone(),
4023 content: "done".into(),
4024 is_error: false,
4025 duration_ms: 1,
4026 tokens_used: 0,
4027 },
4028 )
4029 .expect_err("corrupt wait authority must fail the terminal transition");
4030 assert!(
4031 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
4032 "unexpected terminal error: {err:?}"
4033 );
4034
4035 let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
4037 .await
4038 .expect("waiter must resolve, not hang, after authority corruption");
4039 match waiter_result {
4040 Err(OpsLifecycleError::Internal(message)) => assert!(
4041 message.contains("wait_all completion channel dropped"),
4042 "unexpected waiter error message: {message}"
4043 ),
4044 other => panic!("expected dropped-channel Internal error, got {other:?}"),
4045 }
4046 }
4047
4048 #[test]
4052 fn completion_cursor_propagates_poison_not_none() {
4053 let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
4054
4055 let poison_registry = std::sync::Arc::clone(®istry);
4057 let join = std::thread::spawn(move || {
4058 let _guard = poison_registry.write_state().unwrap();
4059 panic!("intentional panic to poison ops lifecycle registry lock");
4060 });
4061 assert!(
4062 join.join().is_err(),
4063 "poisoning thread must have panicked while holding the write guard"
4064 );
4065
4066 let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
4067 let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
4068 match result {
4069 Err(OpsLifecycleError::Internal(message)) => assert!(
4070 message.contains("ops lifecycle registry poisoned"),
4071 "unexpected cursor error message: {message}"
4072 ),
4073 other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
4074 }
4075 }
4076
4077 #[test]
4078 fn terminate_owner_only_targets_non_terminal_operations() {
4079 let registry = RuntimeOpsLifecycleRegistry::new();
4080
4081 let running_spec = background_spec("running");
4082 let running_id = running_spec.id.clone();
4083 registry.register_operation(running_spec).unwrap();
4084 registry.provisioning_succeeded(&running_id).unwrap();
4085
4086 let completed_spec = background_spec("completed");
4087 let completed_id = completed_spec.id.clone();
4088 registry.register_operation(completed_spec).unwrap();
4089 registry.provisioning_succeeded(&completed_id).unwrap();
4090 registry
4091 .complete_operation(
4092 &completed_id,
4093 OperationResult {
4094 id: completed_id.clone(),
4095 content: "done".into(),
4096 is_error: false,
4097 duration_ms: 1,
4098 tokens_used: 0,
4099 },
4100 )
4101 .unwrap();
4102
4103 registry.terminate_owner("shutdown".into()).unwrap();
4104
4105 assert!(matches!(
4106 registry.snapshot(&running_id).unwrap().unwrap().status,
4107 OperationStatus::Terminated
4108 ));
4109 assert!(matches!(
4110 registry.snapshot(&completed_id).unwrap().unwrap().status,
4111 OperationStatus::Completed
4112 ));
4113 }
4114
4115 #[test]
4116 fn unregister_retirement_fences_detached_late_callbacks() {
4117 let registry = RuntimeOpsLifecycleRegistry::new();
4118 let running_spec = background_spec("detached-running");
4119 let running_id = running_spec.id.clone();
4120 registry.register_operation(running_spec).unwrap();
4121 registry.provisioning_succeeded(&running_id).unwrap();
4122
4123 registry
4124 .retire_owner_for_unregister("owner unregistered".into())
4125 .unwrap();
4126 assert!(matches!(
4127 registry.snapshot(&running_id).unwrap().unwrap().status,
4128 OperationStatus::Terminated
4129 ));
4130
4131 let late_result = OperationResult {
4132 id: running_id.clone(),
4133 content: "late detached completion".into(),
4134 is_error: false,
4135 duration_ms: 1,
4136 tokens_used: 0,
4137 };
4138 assert_eq!(
4139 registry.complete_operation(&running_id, late_result),
4140 Err(OpsLifecycleError::OwnerRetired)
4141 );
4142 assert_eq!(
4143 registry.register_operation(background_spec("late-new-op")),
4144 Err(OpsLifecycleError::OwnerRetired)
4145 );
4146 assert_eq!(
4147 registry.report_progress(
4148 &running_id,
4149 OperationProgressUpdate {
4150 message: "late progress".into(),
4151 percent: None,
4152 },
4153 ),
4154 Err(OpsLifecycleError::OwnerRetired)
4155 );
4156 }
4157
4158 #[test]
4159 fn collect_completed_drains_terminal_operations() {
4160 let registry = RuntimeOpsLifecycleRegistry::new();
4161
4162 let spec_a = background_spec("a");
4163 let id_a = spec_a.id.clone();
4164 registry.register_operation(spec_a).unwrap();
4165 registry.provisioning_succeeded(&id_a).unwrap();
4166 registry
4167 .complete_operation(
4168 &id_a,
4169 OperationResult {
4170 id: id_a.clone(),
4171 content: "done".into(),
4172 is_error: false,
4173 duration_ms: 1,
4174 tokens_used: 0,
4175 },
4176 )
4177 .unwrap();
4178
4179 let spec_b = background_spec("b");
4180 let id_b = spec_b.id.clone();
4181 registry.register_operation(spec_b).unwrap();
4182
4183 let collected = registry.collect_completed().unwrap();
4184 assert_eq!(collected.len(), 1);
4185 assert_eq!(collected[0].0, id_a);
4186
4187 assert!(registry.snapshot(&id_a).unwrap().is_none());
4188 assert!(registry.snapshot(&id_b).unwrap().is_some());
4189
4190 let collected2 = registry.collect_completed().unwrap();
4191 assert!(collected2.is_empty());
4192 }
4193
4194 #[test]
4195 fn bounded_completed_retention_evicts_oldest() {
4196 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4197 max_completed: 3,
4198 max_concurrent: None,
4199 });
4200
4201 let mut ids = Vec::new();
4202 for i in 0..5 {
4203 let spec = background_spec(&format!("op-{i}"));
4204 let id = spec.id.clone();
4205 registry.register_operation(spec).unwrap();
4206 registry.provisioning_succeeded(&id).unwrap();
4207 registry
4208 .complete_operation(
4209 &id,
4210 OperationResult {
4211 id: id.clone(),
4212 content: format!("done-{i}"),
4213 is_error: false,
4214 duration_ms: 1,
4215 tokens_used: 0,
4216 },
4217 )
4218 .unwrap();
4219 ids.push(id);
4220 }
4221
4222 assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4223 assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4224 assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4225 assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4226 assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4227 }
4228
4229 #[test]
4230 fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4231 let registry = RuntimeOpsLifecycleRegistry::new();
4232
4233 let completed_spec = background_spec("completed");
4234 let completed_id = completed_spec.id.clone();
4235 registry.register_operation(completed_spec).unwrap();
4236 registry.provisioning_succeeded(&completed_id).unwrap();
4237 registry
4238 .complete_operation(
4239 &completed_id,
4240 OperationResult {
4241 id: completed_id.clone(),
4242 content: "done".into(),
4243 is_error: false,
4244 duration_ms: 1,
4245 tokens_used: 0,
4246 },
4247 )
4248 .unwrap();
4249
4250 let running_spec = background_spec("running");
4251 let running_id = running_spec.id.clone();
4252 registry.register_operation(running_spec).unwrap();
4253 registry.provisioning_succeeded(&running_id).unwrap();
4254
4255 let cursor_state = meerkat_core::EpochCursorState::new();
4256 let snapshot = registry
4257 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4258 .unwrap();
4259 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4260
4261 assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4262 assert!(recovered.snapshot(&running_id).unwrap().is_none());
4263
4264 let collected = recovered.collect_completed().unwrap();
4265 assert_eq!(collected.len(), 1);
4266 assert_eq!(collected[0].0, completed_id);
4267 }
4268
4269 #[test]
4270 fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4271 let registry = RuntimeOpsLifecycleRegistry::new();
4272
4273 let mut spec = background_spec("capacity");
4274 spec.kind = OperationKind::BackgroundToolCapacitySlot;
4275 let operation_id = spec.id.clone();
4276 registry.register_operation(spec).unwrap();
4277 registry.provisioning_succeeded(&operation_id).unwrap();
4278 registry.mark_retired(&operation_id).unwrap();
4279
4280 assert!(registry.snapshot(&operation_id).unwrap().is_none());
4281
4282 let cursor_state = meerkat_core::EpochCursorState::new();
4283 let snapshot = registry
4284 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4285 .unwrap();
4286 assert!(
4287 !snapshot
4288 .authority_state
4289 .operations
4290 .contains_key(&operation_id)
4291 );
4292 assert!(!snapshot.operation_specs.contains_key(&operation_id));
4293 assert!(snapshot.completion_entries.is_empty());
4294
4295 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4296 assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4297 }
4298
4299 #[test]
4300 fn recovered_snapshot_uses_authority_operation_source() {
4301 let registry = RuntimeOpsLifecycleRegistry::new();
4302 let child_session_id = SessionId::new();
4303 let operation_source = OperationSource::session_child(child_session_id.clone());
4304 let spec = OperationSpec {
4305 id: OperationId::new(),
4306 kind: OperationKind::MobMemberChild,
4307 owner_session_id: SessionId::new(),
4308 display_name: "source-recovery".into(),
4309 source_label: "test".into(),
4310 operation_source: Some(operation_source.clone()),
4311 child_session_id: Some(child_session_id),
4312 expect_peer_channel: true,
4313 };
4314 let operation_id = spec.id.clone();
4315
4316 registry.register_operation(spec).unwrap();
4317 registry.provisioning_succeeded(&operation_id).unwrap();
4318 registry.mark_retired(&operation_id).unwrap();
4319
4320 let cursor_state = meerkat_core::EpochCursorState::new();
4321 let mut snapshot = registry
4322 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4323 .unwrap();
4324 assert_eq!(
4325 snapshot
4326 .authority_state
4327 .operations
4328 .get(&operation_id)
4329 .and_then(|state| state.operation_source.as_ref()),
4330 Some(&operation_source)
4331 );
4332
4333 snapshot
4334 .operation_specs
4335 .get_mut(&operation_id)
4336 .expect("persisted spec")
4337 .operation_source = None;
4338 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4339 assert_eq!(
4340 recovered
4341 .snapshot(&operation_id)
4342 .unwrap()
4343 .unwrap()
4344 .operation_source,
4345 Some(operation_source)
4346 );
4347 }
4348
4349 #[test]
4350 fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4351 let registry = RuntimeOpsLifecycleRegistry::new();
4352 let child_session_id = SessionId::new();
4353 let operation_source = OperationSource::session_child(child_session_id.clone());
4354 let spec = OperationSpec {
4355 id: OperationId::new(),
4356 kind: OperationKind::MobMemberChild,
4357 owner_session_id: SessionId::new(),
4358 display_name: "source-drift".into(),
4359 source_label: "test".into(),
4360 operation_source: Some(operation_source),
4361 child_session_id: Some(child_session_id),
4362 expect_peer_channel: true,
4363 };
4364 let operation_id = spec.id.clone();
4365
4366 registry.register_operation(spec).unwrap();
4367 registry.provisioning_succeeded(&operation_id).unwrap();
4368 registry.mark_retired(&operation_id).unwrap();
4369
4370 let cursor_state = meerkat_core::EpochCursorState::new();
4371 let mut snapshot = registry
4372 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4373 .unwrap();
4374 snapshot
4375 .operation_specs
4376 .get_mut(&operation_id)
4377 .expect("persisted spec")
4378 .operation_source = Some(OperationSource::session_child(SessionId::new()));
4379
4380 let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4381 .expect_err("source mirror drift must fail recovery");
4382 assert!(
4383 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4384 "unexpected recovery error: {err:?}"
4385 );
4386 }
4387
4388 #[test]
4389 fn persisted_authority_state_serializes_explicit_no_operation_source() {
4390 let registry = RuntimeOpsLifecycleRegistry::new();
4391
4392 let spec = background_spec("explicit-no-source");
4393 let operation_id = spec.id.clone();
4394 registry.register_operation(spec).unwrap();
4395 registry.provisioning_succeeded(&operation_id).unwrap();
4396
4397 let cursor_state = meerkat_core::EpochCursorState::new();
4398 let snapshot = registry
4399 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4400 .unwrap();
4401 let value = serde_json::to_value(&snapshot).unwrap();
4402 let operations = value
4403 .get("authority_state")
4404 .and_then(|state| state.get("operations"))
4405 .and_then(serde_json::Value::as_object)
4406 .expect("serialized authority operations");
4407 let persisted_state = operations
4408 .values()
4409 .next()
4410 .and_then(serde_json::Value::as_object)
4411 .expect("serialized operation state");
4412
4413 assert!(
4414 persisted_state
4415 .get("operation_source")
4416 .is_some_and(serde_json::Value::is_null),
4417 "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4418 );
4419
4420 let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4421 assert_eq!(
4422 recovered_snapshot
4423 .authority_state
4424 .operations
4425 .get(&operation_id)
4426 .expect("round-tripped operation")
4427 .operation_source,
4428 None
4429 );
4430 }
4431
4432 #[test]
4433 fn persisted_authority_state_rejects_missing_operation_source_fact() {
4434 let registry = RuntimeOpsLifecycleRegistry::new();
4435
4436 let spec = background_spec("missing-source-fact");
4437 registry.register_operation(spec).unwrap();
4438
4439 let cursor_state = meerkat_core::EpochCursorState::new();
4440 let snapshot = registry
4441 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4442 .unwrap();
4443 let mut value = serde_json::to_value(&snapshot).unwrap();
4444 let operations = value
4445 .get_mut("authority_state")
4446 .and_then(|state| state.get_mut("operations"))
4447 .and_then(serde_json::Value::as_object_mut)
4448 .expect("serialized authority operations");
4449 let operation_state = operations
4450 .values_mut()
4451 .next()
4452 .and_then(serde_json::Value::as_object_mut)
4453 .expect("serialized operation state");
4454 assert!(operation_state.remove("operation_source").is_some());
4455
4456 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4457 .expect_err("missing generated source fact must fail recovery snapshot decoding");
4458 assert!(
4459 err.to_string().contains("operation_source"),
4460 "unexpected decode error: {err}"
4461 );
4462 }
4463
4464 #[test]
4465 fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4466 let registry = RuntimeOpsLifecycleRegistry::new();
4467
4468 let spec = background_spec("missing-feed-authority");
4469 let operation_id = spec.id.clone();
4470 registry.register_operation(spec).unwrap();
4471 registry.provisioning_succeeded(&operation_id).unwrap();
4472 registry
4473 .complete_operation(
4474 &operation_id,
4475 OperationResult {
4476 id: operation_id.clone(),
4477 content: "done".into(),
4478 is_error: false,
4479 duration_ms: 1,
4480 tokens_used: 0,
4481 },
4482 )
4483 .unwrap();
4484
4485 let cursor_state = meerkat_core::EpochCursorState::new();
4486 let snapshot = registry
4487 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4488 .unwrap();
4489 let mut value = serde_json::to_value(&snapshot).unwrap();
4490 let authority_state = value
4491 .get_mut("authority_state")
4492 .and_then(serde_json::Value::as_object_mut)
4493 .expect("serialized authority state");
4494 assert!(authority_state.remove("completion_feed_entries").is_some());
4495
4496 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4497 .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4498 assert!(
4499 err.to_string().contains("completion_feed_entries"),
4500 "unexpected decode error: {err}"
4501 );
4502 }
4503
4504 #[test]
4505 fn public_child_session_projection_uses_authority_operation_source() {
4506 let registry = RuntimeOpsLifecycleRegistry::new();
4507 let authority_child_session_id = SessionId::new();
4508 let stale_shell_child_session_id = SessionId::new();
4509 let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4510 let spec = OperationSpec {
4511 id: OperationId::new(),
4512 kind: OperationKind::MobMemberChild,
4513 owner_session_id: SessionId::new(),
4514 display_name: "child-projection".into(),
4515 source_label: "test".into(),
4516 operation_source: Some(operation_source),
4517 child_session_id: Some(stale_shell_child_session_id),
4518 expect_peer_channel: true,
4519 };
4520 let operation_id = spec.id.clone();
4521
4522 registry.register_operation(spec).unwrap();
4523
4524 assert_eq!(
4525 registry
4526 .snapshot(&operation_id)
4527 .unwrap()
4528 .unwrap()
4529 .child_session_id,
4530 Some(authority_child_session_id.clone())
4531 );
4532
4533 let cursor_state = meerkat_core::EpochCursorState::new();
4534 let snapshot = registry
4535 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4536 .unwrap();
4537 assert_eq!(
4538 snapshot
4539 .operation_specs
4540 .get(&operation_id)
4541 .expect("persisted spec")
4542 .child_session_id,
4543 Some(authority_child_session_id)
4544 );
4545 }
4546
4547 #[test]
4548 fn generated_terminal_payload_projection_fails_closed() {
4549 let registry = RuntimeOpsLifecycleRegistry::new();
4550
4551 let spec = background_spec("terminal-payload-drift");
4552 let operation_id = spec.id.clone();
4553 registry.register_operation(spec).unwrap();
4554 registry.provisioning_succeeded(&operation_id).unwrap();
4555 registry
4556 .complete_operation(
4557 &operation_id,
4558 OperationResult {
4559 id: operation_id.clone(),
4560 content: "done".into(),
4561 is_error: false,
4562 duration_ms: 1,
4563 tokens_used: 0,
4564 },
4565 )
4566 .unwrap();
4567
4568 {
4569 let mut state = registry.write_state().unwrap();
4570 let mut machine_state = state.dsl.0.state().clone();
4571 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4572 machine_state
4573 .op_terminal_payload
4574 .insert(operation_id_key, OperationTerminalOutcome::Retired);
4575 state.dsl = DslAuthority(Box::new(
4576 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4577 ));
4578 }
4579
4580 let err = match registry.register_watcher(&operation_id) {
4581 Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4582 Err(err) => err,
4583 };
4584 assert!(
4585 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4586 "unexpected watcher error: {err:?}"
4587 );
4588 let err = registry
4589 .snapshot(&operation_id)
4590 .expect_err("invalid generated terminal payload must reject public snapshot");
4591 assert!(
4592 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
4593 "unexpected public snapshot error: {err:?}"
4594 );
4595
4596 let cursor_state = meerkat_core::EpochCursorState::new();
4597 let err = match registry
4598 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4599 {
4600 Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
4601 Err(err) => err,
4602 };
4603 assert!(
4604 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4605 "unexpected snapshot error: {err:?}"
4606 );
4607
4608 let err = match registry.collect_completed() {
4609 Ok(_) => panic!("invalid generated terminal payload must reject collection"),
4610 Err(err) => err,
4611 };
4612 assert!(
4613 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4614 "unexpected collection error: {err:?}"
4615 );
4616 }
4617
4618 #[test]
4619 fn generated_terminal_payload_missing_projection_fails_closed() {
4620 let registry = RuntimeOpsLifecycleRegistry::new();
4621
4622 let spec = background_spec("terminal-payload-missing");
4623 let operation_id = spec.id.clone();
4624 registry.register_operation(spec).unwrap();
4625 registry.provisioning_succeeded(&operation_id).unwrap();
4626 registry
4627 .fail_operation(&operation_id, "boom".into())
4628 .unwrap();
4629
4630 {
4631 let mut state = registry.write_state().unwrap();
4632 let mut machine_state = state.dsl.0.state().clone();
4633 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4634 machine_state.op_terminal_payload.remove(&operation_id_key);
4635 state.dsl = DslAuthority(Box::new(
4636 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4637 ));
4638 }
4639
4640 let err = match registry.register_watcher(&operation_id) {
4641 Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
4642 Err(err) => err,
4643 };
4644 assert!(
4645 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4646 "unexpected watcher error: {err:?}"
4647 );
4648 let err = registry
4649 .snapshot(&operation_id)
4650 .expect_err("missing generated terminal payload must reject public snapshot");
4651 assert!(
4652 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4653 "unexpected public snapshot error: {err:?}"
4654 );
4655 }
4656
4657 #[test]
4658 fn generated_terminal_status_without_outcome_fails_closed() {
4659 let registry = RuntimeOpsLifecycleRegistry::new();
4660
4661 let spec = background_spec("terminal-outcome-missing");
4662 let operation_id = spec.id.clone();
4663 registry.register_operation(spec).unwrap();
4664 registry.provisioning_succeeded(&operation_id).unwrap();
4665 registry
4666 .fail_operation(&operation_id, "boom".into())
4667 .unwrap();
4668
4669 {
4670 let mut state = registry.write_state().unwrap();
4671 let mut machine_state = state.dsl.0.state().clone();
4672 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4673 machine_state.op_terminal_outcomes.remove(&operation_id_key);
4674 state.dsl = DslAuthority(Box::new(
4675 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4676 ));
4677 }
4678
4679 let err = registry
4680 .snapshot(&operation_id)
4681 .expect_err("terminal status without outcome must reject public snapshot");
4682 assert!(
4683 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4684 "unexpected public snapshot error: {err:?}"
4685 );
4686
4687 let err = match registry.collect_completed() {
4688 Ok(_) => panic!("terminal status without outcome must reject collection"),
4689 Err(err) => err,
4690 };
4691 assert!(
4692 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4693 "unexpected collection error: {err:?}"
4694 );
4695 }
4696
4697 #[test]
4698 fn generated_operation_source_projection_fails_closed() {
4699 let registry = RuntimeOpsLifecycleRegistry::new();
4700 let child_session_id = SessionId::new();
4701 let operation_source = OperationSource::session_child(child_session_id.clone());
4702 let spec = OperationSpec {
4703 id: OperationId::new(),
4704 kind: OperationKind::MobMemberChild,
4705 owner_session_id: SessionId::new(),
4706 display_name: "source-authority-drift".into(),
4707 source_label: "test".into(),
4708 operation_source: Some(operation_source),
4709 child_session_id: Some(child_session_id),
4710 expect_peer_channel: true,
4711 };
4712 let operation_id = spec.id.clone();
4713
4714 registry.register_operation(spec).unwrap();
4715
4716 {
4717 let mut state = registry.write_state().unwrap();
4718 let mut machine_state = state.dsl.0.state().clone();
4719 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4720 machine_state
4721 .op_sources
4722 .get_mut(&operation_id_key)
4723 .expect("generated operation source")
4724 .session_id = None;
4725 state.dsl = DslAuthority(Box::new(
4726 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4727 ));
4728 }
4729
4730 let err = registry
4731 .snapshot(&operation_id)
4732 .expect_err("invalid generated operation source must reject public snapshot");
4733 assert!(
4734 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4735 "unexpected public snapshot error: {err:?}"
4736 );
4737 let err = registry
4738 .list_operations()
4739 .expect_err("invalid generated operation source must reject public operation list");
4740 assert!(
4741 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4742 "unexpected operation list error: {err:?}"
4743 );
4744
4745 let cursor_state = meerkat_core::EpochCursorState::new();
4746 let err = match registry
4747 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4748 {
4749 Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
4750 Err(err) => err,
4751 };
4752 assert!(
4753 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4754 "unexpected snapshot error: {err:?}"
4755 );
4756 }
4757
4758 #[test]
4759 fn generated_operation_id_projection_fails_closed() {
4760 let registry = RuntimeOpsLifecycleRegistry::new();
4761
4762 {
4763 let mut state = registry.write_state().unwrap();
4764 let mut machine_state = state.dsl.0.state().clone();
4765 machine_state.op_statuses.insert(
4766 "not-json-operation-id".into(),
4767 mm_dsl::OperationStatus::Running,
4768 );
4769 state.dsl = DslAuthority(Box::new(
4770 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4771 ));
4772 }
4773
4774 let err = registry
4775 .list_operations()
4776 .expect_err("invalid generated operation id must reject public operation list");
4777 assert!(
4778 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4779 "unexpected operation list error: {err:?}"
4780 );
4781
4782 let cursor_state = meerkat_core::EpochCursorState::new();
4783 let err = match registry
4784 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4785 {
4786 Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
4787 Err(err) => err,
4788 };
4789 assert!(
4790 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4791 "unexpected persistence snapshot error: {err:?}"
4792 );
4793 }
4794
4795 #[test]
4796 fn generated_missing_kind_projection_fails_closed() {
4797 let registry = RuntimeOpsLifecycleRegistry::new();
4798 let spec = background_spec("missing-kind");
4799 let operation_id = spec.id.clone();
4800 registry.register_operation(spec).unwrap();
4801
4802 {
4803 let mut state = registry.write_state().unwrap();
4804 let mut machine_state = state.dsl.0.state().clone();
4805 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4806 machine_state.op_kinds.remove(&operation_id_key);
4807 state.dsl = DslAuthority(Box::new(
4808 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4809 ));
4810 }
4811
4812 let err = registry
4813 .snapshot(&operation_id)
4814 .expect_err("missing generated kind must reject public snapshot");
4815 assert!(
4816 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4817 "unexpected public snapshot error: {err:?}"
4818 );
4819 let err = registry
4820 .list_operations()
4821 .expect_err("missing generated kind must reject public list");
4822 assert!(
4823 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4824 "unexpected public list error: {err:?}"
4825 );
4826
4827 let cursor_state = meerkat_core::EpochCursorState::new();
4828 let err = registry
4829 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4830 .expect_err("missing generated kind must reject persistence snapshot");
4831 assert!(
4832 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4833 "unexpected persistence snapshot error: {err:?}"
4834 );
4835 }
4836
4837 #[test]
4838 fn generated_missing_status_projection_fails_closed() {
4839 let registry = RuntimeOpsLifecycleRegistry::new();
4840 let spec = background_spec("missing-status");
4841 let operation_id = spec.id.clone();
4842 registry.register_operation(spec).unwrap();
4843
4844 {
4845 let mut state = registry.write_state().unwrap();
4846 let mut machine_state = state.dsl.0.state().clone();
4847 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4848 machine_state.op_statuses.remove(&operation_id_key);
4849 state.dsl = DslAuthority(Box::new(
4850 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4851 ));
4852 }
4853
4854 let err = registry
4855 .snapshot(&operation_id)
4856 .expect_err("missing generated status must reject public snapshot");
4857 assert!(
4858 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4859 "unexpected public snapshot error: {err:?}"
4860 );
4861 let err = registry
4862 .list_operations()
4863 .expect_err("missing generated status must reject public list");
4864 assert!(
4865 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4866 "unexpected public list error: {err:?}"
4867 );
4868 let err = registry
4869 .classify_operation_public_result(&operation_id)
4870 .expect_err("missing generated status must reject public-result classification");
4871 assert!(
4872 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4873 "unexpected public-result error: {err:?}"
4874 );
4875 }
4876
4877 #[test]
4878 fn generated_retiring_public_result_remains_running_until_terminal() {
4879 let registry = RuntimeOpsLifecycleRegistry::new();
4880 let spec = background_spec("retiring-public-result");
4881 let operation_id = spec.id.clone();
4882 registry.register_operation(spec).unwrap();
4883 registry.provisioning_succeeded(&operation_id).unwrap();
4884 registry.request_retire(&operation_id).unwrap();
4885
4886 let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
4887 assert_eq!(snapshot.status, OperationStatus::Retiring);
4888 assert!(snapshot.terminal_outcome.is_none());
4889 assert!(!snapshot.terminal);
4890 assert_eq!(
4891 snapshot.public_result_class,
4892 OperationPublicResultClass::Running
4893 );
4894 assert_eq!(
4895 registry
4896 .classify_operation_public_result(&operation_id)
4897 .unwrap(),
4898 OperationPublicResultClass::Running
4899 );
4900 }
4901
4902 #[test]
4903 fn generated_missing_peer_ready_projection_fails_closed() {
4904 let registry = RuntimeOpsLifecycleRegistry::new();
4905 let spec = background_spec("missing-peer-ready");
4906 let operation_id = spec.id.clone();
4907 registry.register_operation(spec).unwrap();
4908
4909 {
4910 let mut state = registry.write_state().unwrap();
4911 let mut machine_state = state.dsl.0.state().clone();
4912 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4913 machine_state.op_peer_ready.remove(&operation_id_key);
4914 state.dsl = DslAuthority(Box::new(
4915 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4916 ));
4917 }
4918
4919 let err = registry
4920 .snapshot(&operation_id)
4921 .expect_err("missing generated peer-ready fact must reject public snapshot");
4922 assert!(
4923 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4924 "unexpected public snapshot error: {err:?}"
4925 );
4926
4927 let cursor_state = meerkat_core::EpochCursorState::new();
4928 let err = registry
4929 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4930 .expect_err("missing generated peer-ready fact must reject persistence snapshot");
4931 assert!(
4932 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4933 "unexpected persistence snapshot error: {err:?}"
4934 );
4935 }
4936
4937 #[test]
4938 fn generated_missing_progress_count_projection_fails_closed() {
4939 let registry = RuntimeOpsLifecycleRegistry::new();
4940 let spec = background_spec("missing-progress-count");
4941 let operation_id = spec.id.clone();
4942 registry.register_operation(spec).unwrap();
4943
4944 {
4945 let mut state = registry.write_state().unwrap();
4946 let mut machine_state = state.dsl.0.state().clone();
4947 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4948 machine_state.op_progress_counts.remove(&operation_id_key);
4949 state.dsl = DslAuthority(Box::new(
4950 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4951 ));
4952 }
4953
4954 let err = registry
4955 .snapshot(&operation_id)
4956 .expect_err("missing generated progress count must reject public snapshot");
4957 assert!(
4958 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4959 "unexpected public snapshot error: {err:?}"
4960 );
4961
4962 let cursor_state = meerkat_core::EpochCursorState::new();
4963 let err = registry
4964 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4965 .expect_err("missing generated progress count must reject persistence snapshot");
4966 assert!(
4967 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4968 "unexpected persistence snapshot error: {err:?}"
4969 );
4970 }
4971
4972 #[test]
4973 fn generated_terminal_sequence_missing_persistence_fails_closed() {
4974 let registry = RuntimeOpsLifecycleRegistry::new();
4975 let spec = background_spec("terminal-sequence-missing");
4976 let operation_id = spec.id.clone();
4977 registry.register_operation(spec).unwrap();
4978 registry.provisioning_succeeded(&operation_id).unwrap();
4979 registry
4980 .complete_operation(
4981 &operation_id,
4982 OperationResult {
4983 id: operation_id.clone(),
4984 content: "done".into(),
4985 is_error: false,
4986 duration_ms: 1,
4987 tokens_used: 0,
4988 },
4989 )
4990 .unwrap();
4991
4992 {
4993 let mut state = registry.write_state().unwrap();
4994 let mut machine_state = state.dsl.0.state().clone();
4995 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4996 machine_state.op_completion_seq.remove(&operation_id_key);
4997 state.dsl = DslAuthority(Box::new(
4998 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4999 ));
5000 }
5001
5002 let cursor_state = meerkat_core::EpochCursorState::new();
5003 let err = registry
5004 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5005 .expect_err("missing generated terminal sequence must reject persistence snapshot");
5006 assert!(
5007 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
5008 "unexpected persistence snapshot error: {err:?}"
5009 );
5010 }
5011
5012 #[test]
5013 fn generated_record_without_shell_projection_fails_closed() {
5014 let registry = RuntimeOpsLifecycleRegistry::new();
5015 let spec = background_spec("missing-shell-record");
5016 let operation_id = spec.id.clone();
5017 registry.register_operation(spec).unwrap();
5018
5019 {
5020 let mut state = registry.write_state().unwrap();
5021 state.records.remove(&operation_id);
5022 }
5023
5024 let err = registry
5025 .snapshot(&operation_id)
5026 .expect_err("generated operation without shell record must reject public snapshot");
5027 assert!(
5028 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5029 "unexpected public snapshot error: {err:?}"
5030 );
5031 let err = registry
5032 .list_operations()
5033 .expect_err("generated operation without shell record must reject public list");
5034 assert!(
5035 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5036 "unexpected public list error: {err:?}"
5037 );
5038 }
5039
5040 #[test]
5041 fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
5042 let registry = RuntimeOpsLifecycleRegistry::new();
5043 let child_session_id = SessionId::new();
5044 let operation_source = OperationSource::session_child(child_session_id.clone());
5045 let spec = OperationSpec {
5046 id: OperationId::new(),
5047 kind: OperationKind::MobMemberChild,
5048 owner_session_id: SessionId::new(),
5049 display_name: "child-drift".into(),
5050 source_label: "test".into(),
5051 operation_source: Some(operation_source),
5052 child_session_id: Some(child_session_id.clone()),
5053 expect_peer_channel: true,
5054 };
5055 let operation_id = spec.id.clone();
5056
5057 registry.register_operation(spec).unwrap();
5058 registry.provisioning_succeeded(&operation_id).unwrap();
5059 registry.mark_retired(&operation_id).unwrap();
5060
5061 let cursor_state = meerkat_core::EpochCursorState::new();
5062 let mut snapshot = registry
5063 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5064 .unwrap();
5065 snapshot
5066 .operation_specs
5067 .get_mut(&operation_id)
5068 .expect("persisted spec")
5069 .child_session_id = Some(SessionId::new());
5070
5071 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5072 assert_eq!(
5073 recovered
5074 .snapshot(&operation_id)
5075 .unwrap()
5076 .unwrap()
5077 .child_session_id,
5078 Some(child_session_id)
5079 );
5080 }
5081
5082 #[test]
5083 fn completion_wake_class_is_generated_by_operation_kind() {
5084 let registry = RuntimeOpsLifecycleRegistry::new();
5085 let operation_id = OperationId::new();
5086
5087 assert_eq!(
5088 registry
5089 .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
5090 .unwrap(),
5091 OperationCompletionWakeClass::Wake
5092 );
5093 assert_eq!(
5094 registry
5095 .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
5096 .unwrap(),
5097 OperationCompletionWakeClass::Ignore
5098 );
5099 assert_eq!(
5100 registry
5101 .classify_operation_completion_wake(
5102 &operation_id,
5103 OperationKind::BackgroundToolCapacitySlot,
5104 )
5105 .unwrap(),
5106 OperationCompletionWakeClass::Ignore
5107 );
5108 }
5109
5110 #[test]
5111 fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
5112 let registry = RuntimeOpsLifecycleRegistry::new();
5113
5114 let running_spec = background_spec("running");
5115 let running_id = running_spec.id.clone();
5116 registry.register_operation(running_spec.clone()).unwrap();
5117 registry.provisioning_succeeded(&running_id).unwrap();
5118
5119 let cursor_state = meerkat_core::EpochCursorState::new();
5120 let mut snapshot = registry
5121 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5122 .unwrap();
5123 snapshot.completion_entries.push(CompletionEntry {
5124 seq: 1,
5125 operation_id: running_id.clone(),
5126 kind: running_spec.kind,
5127 display_name: running_spec.display_name,
5128 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5129 id: running_id,
5130 content: "phantom".into(),
5131 is_error: false,
5132 duration_ms: 1,
5133 tokens_used: 0,
5134 }),
5135 completed_at_ms: None,
5136 });
5137
5138 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5139 Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5140 Err(err) => err,
5141 };
5142 assert!(
5143 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5144 "unexpected recovery error: {err:?}"
5145 );
5146 }
5147
5148 #[test]
5149 fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5150 let registry = RuntimeOpsLifecycleRegistry::new();
5151 let phantom_id = OperationId::new();
5152 {
5153 let state = registry.read_state().unwrap();
5154 state.feed_buffer.push(CompletionEntry {
5155 seq: 1,
5156 operation_id: phantom_id.clone(),
5157 kind: OperationKind::BackgroundToolOp,
5158 display_name: "phantom".into(),
5159 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5160 id: phantom_id,
5161 content: "phantom".into(),
5162 is_error: false,
5163 duration_ms: 1,
5164 tokens_used: 0,
5165 }),
5166 completed_at_ms: None,
5167 });
5168 }
5169
5170 let cursor_state = meerkat_core::EpochCursorState::new();
5171 let err = match registry
5172 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5173 {
5174 Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5175 Err(err) => err,
5176 };
5177 assert!(
5178 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5179 "unexpected capture error: {err:?}"
5180 );
5181 }
5182
5183 #[test]
5184 fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5185 let registry = RuntimeOpsLifecycleRegistry::new();
5186
5187 let spec = background_spec("terminal");
5188 let operation_id = spec.id.clone();
5189 registry.register_operation(spec).unwrap();
5190 registry.provisioning_succeeded(&operation_id).unwrap();
5191 registry
5192 .complete_operation(
5193 &operation_id,
5194 OperationResult {
5195 id: operation_id.clone(),
5196 content: "done".into(),
5197 is_error: false,
5198 duration_ms: 1,
5199 tokens_used: 0,
5200 },
5201 )
5202 .unwrap();
5203
5204 let cursor_state = meerkat_core::EpochCursorState::new();
5205 let mut snapshot = registry
5206 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5207 .unwrap();
5208 let phantom_id = OperationId::new();
5209 let phantom_result = OperationResult {
5210 id: phantom_id.clone(),
5211 content: "phantom".into(),
5212 is_error: false,
5213 duration_ms: 1,
5214 tokens_used: 0,
5215 };
5216 let phantom_entry = CompletionFeedCanonicalState {
5217 seq: snapshot.authority_state.next_completion_seq + 1,
5218 kind: OperationKind::BackgroundToolOp,
5219 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5220 };
5221 snapshot
5222 .authority_state
5223 .completion_feed_entries
5224 .insert(phantom_id.clone(), phantom_entry);
5225 snapshot.completion_entries.push(CompletionEntry {
5226 seq: snapshot.authority_state.next_completion_seq + 1,
5227 operation_id: phantom_id.clone(),
5228 kind: OperationKind::BackgroundToolOp,
5229 display_name: "phantom".into(),
5230 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5231 completed_at_ms: None,
5232 });
5233
5234 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5235 Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5236 Err(err) => err,
5237 };
5238 assert!(
5239 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5240 "unexpected recovery error: {err:?}"
5241 );
5242 }
5243
5244 #[test]
5245 fn recovered_completed_order_uses_generated_completion_sequences() {
5246 let registry = RuntimeOpsLifecycleRegistry::new();
5247
5248 let spec_a = background_spec("a");
5249 let id_a = spec_a.id.clone();
5250 registry.register_operation(spec_a).unwrap();
5251 registry.provisioning_succeeded(&id_a).unwrap();
5252 registry
5253 .complete_operation(
5254 &id_a,
5255 OperationResult {
5256 id: id_a.clone(),
5257 content: "a".into(),
5258 is_error: false,
5259 duration_ms: 1,
5260 tokens_used: 0,
5261 },
5262 )
5263 .unwrap();
5264
5265 let spec_b = background_spec("b");
5266 let id_b = spec_b.id.clone();
5267 registry.register_operation(spec_b).unwrap();
5268 registry.provisioning_succeeded(&id_b).unwrap();
5269 registry
5270 .complete_operation(
5271 &id_b,
5272 OperationResult {
5273 id: id_b.clone(),
5274 content: "b".into(),
5275 is_error: false,
5276 duration_ms: 1,
5277 tokens_used: 0,
5278 },
5279 )
5280 .unwrap();
5281
5282 let cursor_state = meerkat_core::EpochCursorState::new();
5283 let mut snapshot = registry
5284 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5285 .unwrap();
5286 snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5287
5288 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5289 let collected = recovered.collect_completed().unwrap();
5290
5291 assert_eq!(collected[0].0, id_a);
5292 assert_eq!(collected[1].0, id_b);
5293 }
5294
5295 #[test]
5296 fn max_concurrent_enforcement() {
5297 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5298 max_completed: DEFAULT_MAX_COMPLETED,
5299 max_concurrent: Some(2),
5300 });
5301
5302 let spec_a = background_spec("a");
5303 let id_a = spec_a.id.clone();
5304 registry.register_operation(spec_a).unwrap();
5305
5306 let spec_b = background_spec("b");
5307 registry.register_operation(spec_b).unwrap();
5308
5309 let spec_c = background_spec("c");
5310 let result = registry.register_operation(spec_c);
5311 assert!(matches!(
5312 result,
5313 Err(OpsLifecycleError::MaxConcurrentExceeded {
5314 limit: 2,
5315 active: 2,
5316 })
5317 ));
5318
5319 registry.provisioning_succeeded(&id_a).unwrap();
5320 registry
5321 .complete_operation(
5322 &id_a,
5323 OperationResult {
5324 id: id_a.clone(),
5325 content: "done".into(),
5326 is_error: false,
5327 duration_ms: 1,
5328 tokens_used: 0,
5329 },
5330 )
5331 .unwrap();
5332
5333 let spec_d = background_spec("d");
5334 assert!(registry.register_operation(spec_d).is_ok());
5335 }
5336
5337 #[test]
5338 fn snapshot_includes_timestamps() {
5339 let registry = RuntimeOpsLifecycleRegistry::new();
5340 let spec = background_spec("timed");
5341 let op_id = spec.id.clone();
5342 registry.register_operation(spec).unwrap();
5343
5344 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5345 assert!(snap1.created_at_ms > 0);
5346 assert!(snap1.started_at_ms.is_none());
5347 assert!(snap1.completed_at_ms.is_none());
5348 assert!(snap1.elapsed_ms.is_none());
5349
5350 registry.provisioning_succeeded(&op_id).unwrap();
5351 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5352 assert!(snap2.started_at_ms.is_some());
5353 assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5354
5355 registry
5356 .complete_operation(
5357 &op_id,
5358 OperationResult {
5359 id: op_id.clone(),
5360 content: "done".into(),
5361 is_error: false,
5362 duration_ms: 1,
5363 tokens_used: 0,
5364 },
5365 )
5366 .unwrap();
5367 let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5368 assert!(snap3.completed_at_ms.is_some());
5369 assert!(snap3.elapsed_ms.is_some());
5370 assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5371 }
5372
5373 #[test]
5374 fn snapshot_includes_peer_handle() {
5375 let registry = RuntimeOpsLifecycleRegistry::new();
5376 let child_session_id = SessionId::new();
5377 let spec = OperationSpec {
5378 id: OperationId::new(),
5379 kind: OperationKind::MobMemberChild,
5380 owner_session_id: SessionId::new(),
5381 display_name: "peer-test".into(),
5382 source_label: "test".into(),
5383 operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5384 child_session_id: Some(child_session_id),
5385 expect_peer_channel: true,
5386 };
5387 let op_id = spec.id.clone();
5388 registry.register_operation(spec).unwrap();
5389 registry.provisioning_succeeded(&op_id).unwrap();
5390
5391 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5392 assert!(snap1.peer_handle.is_none());
5393
5394 let handle = OperationPeerHandle {
5395 peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5396 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5397 "member-x",
5398 PeerId::new(),
5399 "inproc://x",
5400 )
5401 .unwrap(),
5402 };
5403 registry.peer_ready(&op_id, handle).unwrap();
5404
5405 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5406 assert_eq!(
5407 snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5408 "member-x"
5409 );
5410 }
5411}