1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24 CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31 CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32 OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33 OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34 OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35 OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36 WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43fn deserialize_required_operation_source<'de, D>(
54 deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57 D: serde::Deserializer<'de>,
58{
59 <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65 status: OperationStatus,
66 kind: OperationKind,
67 #[serde(deserialize_with = "deserialize_required_operation_source")]
68 operation_source: Option<OperationSource>,
69 peer_ready: bool,
70 progress_count: u32,
71 watcher_count: u32,
72 terminal_outcome: Option<OperationTerminalOutcome>,
73 #[serde(default, skip_serializing_if = "Option::is_none")]
74 completion_sequence: Option<CompletionSeq>,
75 terminal_buffered: bool,
76}
77
78#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81 seq: CompletionSeq,
82 kind: OperationKind,
83 terminal_outcome: OperationTerminalOutcome,
84}
85
86#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89 operations: HashMap<OperationId, OperationCanonicalState>,
90 completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91 completed_order: VecDeque<OperationId>,
92 max_completed: usize,
93 max_concurrent: Option<usize>,
94 active_count: usize,
95 wait_request_id: Option<WaitRequestId>,
96 wait_operation_ids: Vec<OperationId>,
97 next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101 pub fn max_completed(&self) -> usize {
103 self.max_completed
104 }
105
106 pub fn max_concurrent(&self) -> Option<usize> {
108 self.max_concurrent
109 }
110
111 pub fn operation_count(&self) -> usize {
113 self.operations.len()
114 }
115
116 pub fn completion_feed_count(&self) -> usize {
118 self.completion_feed_entries.len()
119 }
120}
121
122#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134 pub epoch_id: meerkat_core::RuntimeEpochId,
136 pub authority_state: RegistryCanonicalState,
138 pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140 pub completion_entries: Vec<CompletionEntry>,
143 #[serde(default, skip_serializing_if = "Option::is_none")]
149 completion_feed_retention: Option<PersistedCompletionFeedRetention>,
150 pub cursors: meerkat_core::EpochCursorSnapshot,
152}
153
154#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
157struct PersistedCompletionFeedRetention {
158 dropped_through: CompletionSeq,
159 dropped_count: u64,
160}
161
162#[derive(Debug)]
163pub struct OpsLifecyclePersistenceRequest {
164 snapshot: PersistedOpsSnapshot,
165 result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
166}
167
168impl OpsLifecyclePersistenceRequest {
169 pub fn snapshot(&self) -> &PersistedOpsSnapshot {
170 &self.snapshot
171 }
172
173 pub fn complete(self, result: Result<(), OpsLifecycleError>) {
174 let _ = self.result_tx.send(result);
175 }
176}
177
178#[derive(Debug)]
187struct FeedBufferInner {
188 entries: VecDeque<CompletionEntry>,
189 watermark: CompletionSeq,
190 dropped_through: CompletionSeq,
191 dropped_count: u64,
192 max_retained: usize,
193}
194
195#[derive(Debug)]
200struct FeedBuffer {
201 inner: RwLock<FeedBufferInner>,
202 watermark_atomic: AtomicU64,
204 notify: tokio::sync::Notify,
206}
207
208impl FeedBuffer {
209 fn new(max_retained: usize) -> Self {
210 Self {
211 inner: RwLock::new(FeedBufferInner {
212 entries: VecDeque::new(),
213 watermark: 0,
214 dropped_through: 0,
215 dropped_count: 0,
216 max_retained,
217 }),
218 watermark_atomic: AtomicU64::new(0),
219 notify: tokio::sync::Notify::new(),
220 }
221 }
222
223 fn push(&self, entry: CompletionEntry) {
224 let mut inner = self
225 .inner
226 .write()
227 .unwrap_or_else(std::sync::PoisonError::into_inner);
228 let seq = entry.seq;
229 inner.entries.push_back(entry);
230 inner.watermark = seq;
231
232 while inner.entries.len() > inner.max_retained {
234 if let Some(dropped) = inner.entries.pop_front() {
235 inner.dropped_through = inner.dropped_through.max(dropped.seq);
236 inner.dropped_count = inner.dropped_count.saturating_add(1);
237 }
238 }
239
240 drop(inner);
241
242 self.watermark_atomic.store(seq, Ordering::Release);
243 self.notify.notify_waiters();
244 }
245
246 fn recover_evicted_prefix(
247 &self,
248 retention: PersistedCompletionFeedRetention,
249 ) -> Result<(), OpsLifecycleError> {
250 let PersistedCompletionFeedRetention {
251 dropped_through,
252 dropped_count,
253 } = retention;
254 if (dropped_through == 0) != (dropped_count == 0) || dropped_count > dropped_through {
255 return Err(OpsLifecycleError::Internal(
256 "persisted completion-feed retention carrier is inconsistent".to_string(),
257 ));
258 }
259 if dropped_count == 0 {
260 return Ok(());
261 }
262 let mut inner = self
263 .inner
264 .write()
265 .unwrap_or_else(std::sync::PoisonError::into_inner);
266 inner.dropped_through = dropped_through;
267 inner.dropped_count = dropped_count;
268 Ok(())
269 }
270}
271
272#[derive(Debug, Clone)]
277pub struct RuntimeCompletionFeed {
278 buffer: Arc<FeedBuffer>,
279 authority: std::sync::Weak<RwLock<ShellState>>,
283}
284
285impl CompletionFeed for RuntimeCompletionFeed {
286 fn watermark(&self) -> CompletionSeq {
287 self.buffer.watermark_atomic.load(Ordering::Acquire)
288 }
289
290 fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
291 let inner = self
292 .buffer
293 .inner
294 .read()
295 .unwrap_or_else(std::sync::PoisonError::into_inner);
296 let entries: Vec<CompletionEntry> = inner
297 .entries
298 .iter()
299 .filter(|e| e.seq > after_seq)
300 .cloned()
301 .collect();
302 let watermark = inner.watermark;
303 let dropped_count = inner.dropped_count;
304 let batch = CompletionBatch {
305 entries,
306 watermark,
307 dropped_through: inner.dropped_through,
308 };
309 drop(inner);
310
311 if batch.gap_after(after_seq).is_none() {
312 return batch;
313 }
314
315 let Some(authority) = self.authority.upgrade() else {
316 return batch;
317 };
318 let state = authority
319 .read()
320 .unwrap_or_else(std::sync::PoisonError::into_inner);
321 let Ok(authority_entries) = state.completion_feed_authority_entries() else {
322 return batch;
326 };
327 let reconstructed_dropped_count = authority_entries
328 .values()
329 .filter(|entry| entry.seq <= batch.dropped_through)
330 .count() as u64;
331 if reconstructed_dropped_count != dropped_count {
332 return batch;
336 }
337 let buffered_projection_by_id: HashMap<OperationId, CompletionEntry> = batch
338 .entries
339 .iter()
340 .cloned()
341 .map(|entry| (entry.operation_id.clone(), entry))
342 .collect();
343 let mut entries = authority_entries
344 .into_iter()
345 .filter(|(_, entry)| entry.seq > after_seq && entry.seq <= batch.watermark)
346 .map(|(operation_id, entry)| {
347 if let Some(buffered) = buffered_projection_by_id.get(&operation_id) {
348 return buffered.clone();
349 }
350 let display_name = state
351 .records
352 .get(&operation_id)
353 .map(|record| record.spec.display_name.clone())
354 .unwrap_or_default();
355 let completed_at_ms = state.records.get(&operation_id).and_then(|record| {
356 record
357 .completed_at
358 .map(|completed_at| record.epoch_millis_for_instant(completed_at))
359 });
360 CompletionEntry {
361 seq: entry.seq,
362 operation_id,
363 kind: entry.kind,
364 display_name,
365 terminal_outcome: entry.terminal_outcome,
366 completed_at_ms,
367 }
368 })
369 .collect::<Vec<_>>();
370 entries.sort_by_key(|entry| entry.seq);
371 CompletionBatch {
372 entries,
373 watermark: batch.watermark,
374 dropped_through: 0,
377 }
378 }
379
380 fn wait_for_advance(
381 &self,
382 after_seq: CompletionSeq,
383 ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
384 Box::pin(async move {
385 loop {
386 let notified = self.buffer.notify.notified();
391 let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
392 if current > after_seq {
393 return current;
394 }
395 notified.await;
396 }
397 })
398 }
399}
400
401#[derive(Debug)]
406struct OperationCompletionNotifier {
407 tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
408}
409
410impl OperationCompletionNotifier {
411 fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
412 Self { tx }
413 }
414
415 fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
416 let _ = self.tx.send(outcome.clone());
417 }
418}
419
420fn operation_completion_watch_from_receiver(
421 rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
422) -> OperationCompletionWatch {
423 Box::pin(async move {
424 rx.await
425 .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
426 })
427}
428
429fn resolved_operation_completion_watch(
430 outcome: OperationTerminalOutcome,
431) -> OperationCompletionWatch {
432 Box::pin(async move { Ok(outcome) })
433}
434
435#[derive(Debug)]
439struct ShellRecord {
440 spec: OperationSpec,
441 peer_handle: Option<OperationPeerHandle>,
442 watchers: Vec<OperationCompletionNotifier>,
446 created_at: Instant,
448 started_at: Option<Instant>,
449 completed_at: Option<Instant>,
450 created_at_wall: SystemTime,
452}
453
454#[derive(Debug)]
455struct PendingWaitState {
456 wait_request_id: WaitRequestId,
457 sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
458}
459
460enum WaitAllAuthorityPlan {
461 AlreadySatisfied(WaitAllSatisfied),
462 ActivateBarrier,
463}
464
465#[derive(Debug, Clone, Copy, PartialEq, Eq)]
466enum RecoveredOperationRecordDisposition {
467 Retain,
468 Discard,
469}
470
471impl ShellRecord {
472 fn new(spec: OperationSpec) -> Self {
473 Self {
474 spec,
475 peer_handle: None,
476 watchers: Vec::new(),
477 created_at: Instant::now(),
478 started_at: None,
479 completed_at: None,
480 created_at_wall: SystemTime::now(),
481 }
482 }
483
484 fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
485 wall_anchor
486 .duration_since(UNIX_EPOCH)
487 .map(|d| d.as_millis() as u64)
488 .unwrap_or(0)
489 }
490
491 fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
492 let offset = instant.saturating_duration_since(self.created_at);
495 let wall = self.created_at_wall + offset;
496 Self::epoch_millis(&wall)
497 }
498
499 fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
501 for watcher in std::mem::take(&mut self.watchers) {
502 watcher.notify_after_generated_terminal(outcome);
503 }
504 }
505
506 fn mark_completed(&mut self) {
508 self.completed_at = Some(Instant::now());
509 }
510}
511
512#[derive(Debug)]
517struct ShellState {
518 dsl: DslAuthority,
520 records: HashMap<OperationId, ShellRecord>,
522 pending_wait: Option<PendingWaitState>,
524 completed_order: VecDeque<OperationId>,
526 max_completed: usize,
528 max_concurrent: Option<usize>,
530 wait_request_id: Option<WaitRequestId>,
536 feed_buffer: Arc<FeedBuffer>,
538 persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
540 persistence_sealed: bool,
542 persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
544 persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
546 owner_retired: bool,
552}
553
554struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
560
561impl std::fmt::Debug for DslAuthority {
562 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
563 f.debug_struct("DslAuthority")
564 .field("state", self.0.state())
565 .finish()
566 }
567}
568
569fn new_ops_dsl_authority() -> DslAuthority {
573 DslAuthority(Box::new(
574 crate::meerkat_machine::dsl_authority::new_initialized_authority(
575 "ops lifecycle DSL Initialize must be accepted",
576 ),
577 ))
578}
579
580impl ShellState {
581 fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
582 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
583 let dsl = new_ops_dsl_authority();
584 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
585 let max_completed = max_completed.max(1);
589 let feed_capacity = max_completed.max(1);
594 tracing::info!(
595 feed_capacity,
596 "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
597 );
598 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
599 tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
600 Self {
601 dsl,
602 records: HashMap::new(),
603 pending_wait: None,
604 completed_order: VecDeque::new(),
605 max_completed,
606 max_concurrent,
607 wait_request_id: None,
608 feed_buffer,
613 persist_tx: None,
614 persistence_sealed: false,
615 persist_epoch_id: None,
616 persist_cursor_state: None,
617 owner_retired: false,
618 }
619 }
620
621 fn ensure_owner_active(&self) -> Result<(), OpsLifecycleError> {
622 if self.owner_retired {
623 Err(OpsLifecycleError::OwnerRetired)
624 } else {
625 Ok(())
626 }
627 }
628
629 fn dsl_apply(
636 &mut self,
637 input: mm_dsl::MeerkatMachineInput,
638 context: &str,
639 ) -> Result<(), OpsLifecycleError> {
640 self.dsl_apply_raw(input).map_err(|err| {
641 OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
642 })
643 }
644
645 fn dsl_apply_raw(
652 &mut self,
653 input: mm_dsl::MeerkatMachineInput,
654 ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
655 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
656 }
657
658 fn dsl_apply_with_effects(
659 &mut self,
660 input: mm_dsl::MeerkatMachineInput,
661 context: &str,
662 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
663 let transition =
664 mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
665 OpsLifecycleError::Internal(format!(
666 "DSL rejected ops transition ({context}): {err:?}"
667 ))
668 })?;
669 Ok(transition.into_effects())
670 }
671
672 fn checked_terminal_payload(
677 kind: mm_dsl::OperationTerminalOutcomeKind,
678 payload: &OperationTerminalOutcome,
679 authority: &str,
680 operation_id: &str,
681 ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
682 if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
683 return Err(OpsLifecycleError::Internal(format!(
684 "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
685 )));
686 }
687 Ok(payload.clone())
688 }
689
690 fn status(&self, id: &OperationId) -> Option<OperationStatus> {
692 let id_key = mm_dsl::OperationId::from_domain(id).0;
693 self.dsl
694 .0
695 .state()
696 .op_statuses
697 .get(&id_key)
698 .copied()
699 .map(OperationStatus::from)
700 }
701
702 fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
703 self.status(id).ok_or_else(|| {
704 OpsLifecycleError::Internal(format!(
705 "generated op lifecycle authority missing status for {id}"
706 ))
707 })
708 }
709
710 fn kind(&self, id: &OperationId) -> Option<OperationKind> {
712 let id_key = mm_dsl::OperationId::from_domain(id).0;
713 self.dsl
714 .0
715 .state()
716 .op_kinds
717 .get(&id_key)
718 .copied()
719 .map(OperationKind::from)
720 }
721
722 fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
723 self.kind(id).ok_or_else(|| {
724 OpsLifecycleError::Internal(format!(
725 "generated op lifecycle authority missing kind for {id}"
726 ))
727 })
728 }
729
730 fn operation_source(
731 &self,
732 id: &OperationId,
733 ) -> Result<Option<OperationSource>, OpsLifecycleError> {
734 let id_key = mm_dsl::OperationId::from_domain(id).0;
735 self.dsl
736 .0
737 .state()
738 .op_sources
739 .get(&id_key)
740 .map(|source| {
741 source.to_domain().map_err(|error| {
742 OpsLifecycleError::Internal(format!(
743 "generated operation source authority has invalid source for {id}: {error}"
744 ))
745 })
746 })
747 .transpose()
748 }
749
750 fn child_session_id_from_operation_source(
751 operation_source: Option<&OperationSource>,
752 ) -> Option<SessionId> {
753 match operation_source {
754 Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
755 Some(OperationSource::BackendPeer { .. } | OperationSource::DetachedJob { .. })
756 | None => None,
757 }
758 }
759
760 fn align_spec_child_session_id_to_source(
761 spec: &mut OperationSpec,
762 operation_source: Option<&OperationSource>,
763 ) {
764 spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
765 }
766
767 fn peer_ready(&self, id: &OperationId) -> Option<bool> {
769 let id_key = mm_dsl::OperationId::from_domain(id).0;
770 self.dsl.0.state().op_peer_ready.get(&id_key).copied()
771 }
772
773 fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
774 self.peer_ready(id).ok_or_else(|| {
775 OpsLifecycleError::Internal(format!(
776 "generated op peer wiring authority missing peer-ready fact for {id}"
777 ))
778 })
779 }
780
781 fn progress_count(&self, id: &OperationId) -> Option<u32> {
783 let id_key = mm_dsl::OperationId::from_domain(id).0;
784 self.dsl
785 .0
786 .state()
787 .op_progress_counts
788 .get(&id_key)
789 .map(|v| (*v).min(u32::MAX as u64) as u32)
790 }
791
792 fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
793 self.progress_count(id).ok_or_else(|| {
794 OpsLifecycleError::Internal(format!(
795 "generated op progress authority missing progress count for {id}"
796 ))
797 })
798 }
799
800 fn terminal_outcome(
804 &self,
805 id: &OperationId,
806 ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
807 let id_key = mm_dsl::OperationId::from_domain(id).0;
808 let state = self.dsl.0.state();
809 let status = self.status(id);
810 let terminal = match status {
811 Some(status) => Self::operation_status_is_terminal(id, status)?,
812 None => false,
813 };
814 let kind = state.op_terminal_outcomes.get(&id_key).copied();
815 let Some(kind) = kind else {
816 if terminal {
817 return Err(OpsLifecycleError::Internal(format!(
818 "generated op terminal authority missing terminal outcome for {id}"
819 )));
820 }
821 return Ok(None);
822 };
823 if !terminal {
824 return Err(OpsLifecycleError::Internal(format!(
825 "generated op terminal authority has terminal outcome for non-terminal {id}"
826 )));
827 }
828 let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
829 OpsLifecycleError::Internal(format!(
830 "generated op terminal authority missing terminal payload for {id}"
831 ))
832 })?;
833 Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
834 .map(Some)
835 }
836
837 fn contains(&self, id: &OperationId) -> bool {
839 let id_key = mm_dsl::OperationId::from_domain(id).0;
840 self.dsl.0.state().op_statuses.contains_key(&id_key)
841 }
842
843 fn active_count(&self) -> usize {
845 self.dsl.0.state().active_op_count as usize
846 }
847
848 fn operation_count(&self) -> usize {
850 self.dsl.0.state().op_statuses.len()
851 }
852
853 fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
855 let mut ids = BTreeSet::new();
856 let state = self.dsl.0.state();
857 Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
858 Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
859 Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
860 Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
861 Self::collect_operation_id_keys(
862 &mut ids,
863 "op_progress_counts",
864 state.op_progress_counts.keys(),
865 )?;
866 Self::collect_operation_id_keys(
867 &mut ids,
868 "op_terminal_outcomes",
869 state.op_terminal_outcomes.keys(),
870 )?;
871 Self::collect_operation_id_keys(
872 &mut ids,
873 "op_terminal_payload",
874 state.op_terminal_payload.keys(),
875 )?;
876 Self::collect_operation_id_keys(
877 &mut ids,
878 "op_completion_seq",
879 state.op_completion_seq.keys(),
880 )?;
881 ids.extend(self.records.keys().cloned());
882 Ok(ids.into_iter().collect())
883 }
884
885 fn collect_operation_id_keys<'a, I>(
886 ids: &mut BTreeSet<OperationId>,
887 field: &str,
888 keys: I,
889 ) -> Result<(), OpsLifecycleError>
890 where
891 I: IntoIterator<Item = &'a String>,
892 {
893 for key in keys {
894 let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
895 OpsLifecycleError::Internal(format!(
896 "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
897 ))
898 })?;
899 ids.insert(id);
900 }
901 Ok(())
902 }
903
904 fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
905 let id_key = mm_dsl::OperationId::from_domain(id).0;
906 let state = self.dsl.0.state();
907 state.op_statuses.contains_key(&id_key)
908 || state.op_kinds.contains_key(&id_key)
909 || state.op_sources.contains_key(&id_key)
910 || state.op_peer_ready.contains_key(&id_key)
911 || state.op_progress_counts.contains_key(&id_key)
912 || state.op_terminal_outcomes.contains_key(&id_key)
913 || state.op_terminal_payload.contains_key(&id_key)
914 || state.op_completion_seq.contains_key(&id_key)
915 }
916
917 fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
919 let id_key = mm_dsl::OperationId::from_domain(id).0;
920 self.dsl.0.state().op_completion_seq.get(&id_key).copied()
921 }
922
923 fn completion_feed_authority_entries(
924 &self,
925 ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
926 let state = self.dsl.0.state();
927 let sequence_keys: BTreeSet<String> =
928 state.completion_feed_sequences.keys().cloned().collect();
929 let companion_domains: [(&str, BTreeSet<String>); 3] = [
930 (
931 "completion_feed_kinds",
932 state.completion_feed_kinds.keys().cloned().collect(),
933 ),
934 (
935 "completion_feed_terminal_outcomes",
936 state
937 .completion_feed_terminal_outcomes
938 .keys()
939 .cloned()
940 .collect(),
941 ),
942 (
943 "completion_feed_terminal_payload",
944 state
945 .completion_feed_terminal_payload
946 .keys()
947 .cloned()
948 .collect(),
949 ),
950 ];
951 for (field, keys) in companion_domains {
952 if keys != sequence_keys {
953 return Err(OpsLifecycleError::Internal(format!(
954 "generated completion feed authority has mismatched {field} domain"
955 )));
956 }
957 }
958
959 let mut entries = HashMap::new();
960 for (id_key, seq) in &state.completion_feed_sequences {
961 if !state.completion_sequence_claims.contains(seq) {
962 return Err(OpsLifecycleError::Internal(format!(
963 "generated completion feed authority sequence {seq} for {id_key} is not claimed"
964 )));
965 }
966 let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
967 OpsLifecycleError::Internal(format!(
968 "generated completion feed authority used invalid operation id key {id_key}: {error}"
969 ))
970 })?;
971 let kind = state
972 .completion_feed_kinds
973 .get(id_key)
974 .copied()
975 .map(OperationKind::from)
976 .ok_or_else(|| {
977 OpsLifecycleError::Internal(format!(
978 "generated completion feed authority missing kind for {id_key}"
979 ))
980 })?;
981 let outcome_kind = state
982 .completion_feed_terminal_outcomes
983 .get(id_key)
984 .copied()
985 .ok_or_else(|| {
986 OpsLifecycleError::Internal(format!(
987 "generated completion feed authority missing terminal outcome for {id_key}"
988 ))
989 })?;
990 let payload = state
991 .completion_feed_terminal_payload
992 .get(id_key)
993 .ok_or_else(|| {
994 OpsLifecycleError::Internal(format!(
995 "generated completion feed authority missing terminal payload for {id_key}"
996 ))
997 })?;
998 let terminal_outcome = Self::checked_terminal_payload(
999 outcome_kind,
1000 payload,
1001 "generated completion feed authority",
1002 id_key,
1003 )?;
1004 entries.insert(
1005 operation_id,
1006 CompletionFeedCanonicalState {
1007 seq: *seq,
1008 kind,
1009 terminal_outcome,
1010 },
1011 );
1012 }
1013 Ok(entries)
1014 }
1015
1016 fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
1017 let state = self.dsl.0.state();
1018 match consumer {
1019 CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
1020 CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
1021 CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
1022 }
1023 }
1024
1025 fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
1026 meerkat_core::EpochCursorSnapshot {
1027 agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
1028 runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
1029 runtime_last_injected_seq: self
1030 .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
1031 }
1032 }
1033
1034 fn snapshot(
1036 &self,
1037 id: &OperationId,
1038 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
1039 let Some(shell) = self.records.get(id) else {
1040 if self.has_generated_operation_record_fact(id) {
1041 return Err(OpsLifecycleError::Internal(format!(
1042 "generated op lifecycle authority has operation facts without shell projection record for {id}"
1043 )));
1044 }
1045 return Ok(None);
1046 };
1047 let kind = self.require_kind(id)?;
1048 let status = self.require_status(id)?;
1049 let terminal = Self::operation_status_is_terminal(id, status)?;
1050 let public_result_class = Self::operation_public_result_class(id, status)?;
1051 let peer_ready = self.require_peer_ready(id)?;
1052 let progress_count = self.require_progress_count(id)?;
1053 let operation_source = self.operation_source(id)?;
1054 let terminal_outcome = self.terminal_outcome(id)?;
1055
1056 let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
1057 let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
1058 let completed_at_ms = shell
1059 .completed_at
1060 .map(|i| shell.epoch_millis_for_instant(i));
1061 let elapsed_ms = shell.completed_at.map(|completed| {
1062 completed
1063 .saturating_duration_since(shell.created_at)
1064 .as_millis() as u64
1065 });
1066
1067 Ok(Some(OperationLifecycleSnapshot {
1068 id: shell.spec.id.clone(),
1069 kind,
1070 owner_session_id: shell.spec.owner_session_id.clone(),
1071 display_name: shell.spec.display_name.clone(),
1072 source_label: shell.spec.source_label.clone(),
1073 child_session_id: Self::child_session_id_from_operation_source(
1074 operation_source.as_ref(),
1075 ),
1076 operation_source,
1077 expect_peer_channel: shell.spec.expect_peer_channel,
1078 status,
1079 terminal,
1080 public_result_class,
1081 peer_ready,
1082 progress_count,
1083 watcher_count: shell.watchers.len() as u32,
1084 terminal_outcome,
1085 peer_handle: shell.peer_handle.clone(),
1086 created_at_ms,
1087 started_at_ms,
1088 completed_at_ms,
1089 elapsed_ms,
1090 }))
1091 }
1092
1093 fn finalize_terminal(
1098 &mut self,
1099 id: &OperationId,
1100 ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
1101 let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
1102 OpsLifecycleError::Internal(format!(
1103 "generated op terminal transition did not mint terminal outcome for {id}"
1104 ))
1105 })?;
1106 let kind = self.require_kind(id)?;
1107
1108 if let Some(shell) = self.records.get_mut(id) {
1110 shell.notify_watchers(&outcome);
1111 shell.mark_completed();
1112 }
1113
1114 if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
1115 {
1116 self.dsl_apply(
1117 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
1118 operation_id: mm_dsl::OperationId::from_domain(id).0,
1119 },
1120 "CollectCompletedOp",
1121 )?;
1122 self.records.remove(id);
1123 self.completed_order.retain(|queued| queued != id);
1124 return Ok(None);
1125 }
1126
1127 let mut completion_entry = None;
1128 if Self::operation_completion_feed_class(id, kind)?
1129 == mm_dsl::OperationCompletionFeedClass::Emit
1130 {
1131 let feed_authority = self
1132 .completion_feed_authority_entries()?
1133 .remove(id)
1134 .ok_or_else(|| {
1135 OpsLifecycleError::Internal(format!(
1136 "generated op terminal transition did not mint completion feed authority for {id}"
1137 ))
1138 })?;
1139 if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
1140 return Err(OpsLifecycleError::Internal(format!(
1141 "generated completion feed authority drifted from terminal op authority for {id}"
1142 )));
1143 }
1144 let seq = self.completion_sequence(id).ok_or_else(|| {
1145 OpsLifecycleError::Internal(format!(
1146 "generated op terminal transition did not mint completion sequence for {id}"
1147 ))
1148 })?;
1149 if feed_authority.seq != seq {
1150 return Err(OpsLifecycleError::Internal(format!(
1151 "generated completion feed authority sequence drifted for {id}"
1152 )));
1153 }
1154 let display_name = self
1155 .records
1156 .get(id)
1157 .map(|r| r.spec.display_name.clone())
1158 .unwrap_or_default();
1159 let completed_at_ms = self
1160 .records
1161 .get(id)
1162 .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1163 completion_entry = Some(CompletionEntry {
1164 seq: feed_authority.seq,
1165 operation_id: id.clone(),
1166 kind: feed_authority.kind,
1167 display_name,
1168 terminal_outcome: feed_authority.terminal_outcome,
1169 completed_at_ms,
1170 });
1171 }
1172
1173 self.completed_order.push_back(id.clone());
1175 while self.completed_order.len() > self.max_completed {
1176 if let Some(evicted) = self.completed_order.pop_front() {
1177 self.dsl_apply(
1178 mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1179 operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1180 },
1181 "EvictCompletedOp",
1182 )?;
1183 self.records.remove(&evicted);
1184 }
1185 }
1186
1187 self.maybe_satisfy_wait()?;
1192 Ok(completion_entry)
1193 }
1194
1195 fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1196 if let Some(entry) = entry {
1197 self.feed_buffer.push(entry);
1198 }
1199 }
1200
1201 fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1203 self.dsl
1204 .0
1205 .state()
1206 .wait_operation_ids
1207 .iter()
1208 .map(|key| {
1209 serde_json::from_str::<OperationId>(key).map_err(|error| {
1210 OpsLifecycleError::Internal(format!(
1211 "generated wait operation identity authority used invalid operation id key {key}: {error}"
1212 ))
1213 })
1214 })
1215 .collect()
1216 }
1217
1218 #[cfg(test)]
1220 fn wait_active(&self) -> bool {
1221 self.dsl.0.state().wait_active
1222 }
1223
1224 fn wait_all_satisfied_from_effects(
1225 effects: &[mm_dsl::MeerkatMachineEffect],
1226 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1227 let mut satisfied = None;
1228 for effect in effects {
1229 let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1230 wait_request_id,
1231 run_id,
1232 operation_ids,
1233 } = effect
1234 else {
1235 continue;
1236 };
1237 if satisfied.is_some() {
1238 return Err(OpsLifecycleError::Internal(
1239 "generated wait_all authority emitted multiple satisfaction effects".into(),
1240 ));
1241 }
1242 let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1243 OpsLifecycleError::Internal(format!(
1244 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1245 wait_request_id.0
1246 ))
1247 })?;
1248 let mut ids = Vec::with_capacity(operation_ids.len());
1249 for operation_id in operation_ids {
1250 ids.push(
1251 serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1252 OpsLifecycleError::Internal(format!(
1253 "generated wait_all authority emitted invalid operation id '{}': {err}",
1254 operation_id.0
1255 ))
1256 })?,
1257 );
1258 }
1259 satisfied = Some(WaitAllSatisfied {
1260 wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1261 run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1262 OpsLifecycleError::Internal(format!(
1263 "generated wait_all authority emitted invalid run id '{}': {err}",
1264 run_id.0
1265 ))
1266 })?),
1267 operation_ids: ids,
1268 });
1269 }
1270 Ok(satisfied)
1271 }
1272
1273 fn parse_wait_all_operation_id(
1274 raw: &str,
1275 context: &str,
1276 ) -> Result<OperationId, OpsLifecycleError> {
1277 serde_json::from_str::<OperationId>(raw).map_err(|err| {
1278 OpsLifecycleError::Internal(format!(
1279 "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1280 ))
1281 })
1282 }
1283
1284 fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1285 let mut seen = HashSet::new();
1286 operation_ids
1287 .iter()
1288 .find(|operation_id| !seen.insert((*operation_id).clone()))
1289 .cloned()
1290 }
1291
1292 fn wait_all_admission_error_from_effects(
1293 wait_request_id: &WaitRequestId,
1294 effects: &[mm_dsl::MeerkatMachineEffect],
1295 ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1296 let mut admission = None;
1297 for effect in effects {
1298 let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1299 wait_request_id: resolved_wait_request_id,
1300 result,
1301 reject_reason,
1302 rejected_operation_id,
1303 } = effect
1304 else {
1305 continue;
1306 };
1307 if admission.is_some() {
1308 return Err(OpsLifecycleError::Internal(
1309 "generated wait_all authority emitted multiple admission results".into(),
1310 ));
1311 }
1312 let resolved_uuid =
1313 uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1314 OpsLifecycleError::Internal(format!(
1315 "generated wait_all authority emitted invalid wait request id '{}': {err}",
1316 resolved_wait_request_id.0
1317 ))
1318 })?;
1319 let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1320 if &resolved_wait_request_id != wait_request_id {
1321 return Err(OpsLifecycleError::Internal(format!(
1322 "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1323 )));
1324 }
1325 admission = Some(match result {
1326 mm_dsl::WaitAllAdmissionResultKind::Accept => {
1327 if reject_reason.is_some() || rejected_operation_id.is_some() {
1328 return Err(OpsLifecycleError::Internal(
1329 "generated wait_all authority accepted with rejection payload".into(),
1330 ));
1331 }
1332 None
1333 }
1334 mm_dsl::WaitAllAdmissionResultKind::Reject => {
1335 let reason = reject_reason.ok_or_else(|| {
1336 OpsLifecycleError::Internal(
1337 "generated wait_all authority rejected without reason".into(),
1338 )
1339 })?;
1340 let error = match reason {
1341 mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1342 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1343 OpsLifecycleError::Internal(
1344 "generated wait_all authority rejected duplicate without operation id"
1345 .into(),
1346 )
1347 })?;
1348 OpsLifecycleError::DuplicateWaitOperation(
1349 Self::parse_wait_all_operation_id(raw, "duplicate")?,
1350 )
1351 }
1352 mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1353 if rejected_operation_id.is_some() {
1354 return Err(OpsLifecycleError::Internal(
1355 "generated wait_all authority rejected active wait with operation id"
1356 .into(),
1357 ));
1358 }
1359 OpsLifecycleError::WaitAlreadyActive
1360 }
1361 mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1362 let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1363 OpsLifecycleError::Internal(
1364 "generated wait_all authority rejected missing operation without operation id"
1365 .into(),
1366 )
1367 })?;
1368 OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1369 raw, "missing",
1370 )?)
1371 }
1372 };
1373 Some(error)
1374 }
1375 });
1376 }
1377 admission.ok_or_else(|| {
1378 OpsLifecycleError::Internal(
1379 "generated wait_all authority emitted no admission result".into(),
1380 )
1381 })
1382 }
1383
1384 fn resolve_wait_all_admission(
1385 &mut self,
1386 wait_request_id: &WaitRequestId,
1387 operation_ids: &[OperationId],
1388 dsl_ids: &BTreeSet<String>,
1389 dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1390 operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1391 operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1392 ) -> Result<(), OpsLifecycleError> {
1393 let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1394 .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1395 let not_found = operation_ids
1396 .iter()
1397 .find(|operation_id| !self.contains(operation_id))
1398 .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1399 let dsl_id_sequence: Vec<String> = operation_ids
1400 .iter()
1401 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1402 .collect();
1403 let effects = self.dsl_apply_with_effects(
1404 mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1405 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1406 operation_id_sequence: dsl_id_sequence,
1407 operation_ids: dsl_ids.clone(),
1408 operation_id_tokens: dsl_id_tokens.clone(),
1409 operation_token_by_id: operation_token_by_id.clone(),
1410 operation_id_by_token: operation_id_by_token.clone(),
1411 duplicate_operation_id: duplicate,
1412 not_found_operation_id: not_found,
1413 },
1414 "ResolveWaitAllAdmission",
1415 )?;
1416 if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1417 {
1418 return Err(error);
1419 }
1420 Ok(())
1421 }
1422
1423 fn try_satisfy_wait_all_authority(
1424 &mut self,
1425 ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1426 let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1427 return Ok(None);
1428 };
1429 let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1430 return Err(OpsLifecycleError::Internal(
1431 "generated wait_all authority has active wait without run id".into(),
1432 ));
1433 };
1434 let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1435 let transition = match mm_dsl::MeerkatMachineMutator::apply(
1436 &mut *self.dsl.0,
1437 mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1438 wait_request_id: dsl_wait_request_id,
1439 run_id: dsl_run_id,
1440 operation_id_tokens: dsl_operation_id_tokens,
1441 },
1442 ) {
1443 Ok(transition) => transition,
1444 Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1445 Err(err) => {
1446 return Err(OpsLifecycleError::Internal(format!(
1447 "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1448 )));
1449 }
1450 };
1451 Self::wait_all_satisfied_from_effects(transition.effects())?
1452 .ok_or_else(|| {
1453 OpsLifecycleError::Internal(
1454 "generated wait_all authority accepted satisfaction without effect".into(),
1455 )
1456 })
1457 .map(Some)
1458 }
1459
1460 fn begin_wait_all_authority(
1461 &mut self,
1462 run_id: &RunId,
1463 wait_request_id: &WaitRequestId,
1464 operation_ids: &[OperationId],
1465 ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1466 let mut dsl_ids = BTreeSet::new();
1467 let mut dsl_id_tokens = BTreeSet::new();
1468 let mut operation_token_by_id = BTreeMap::new();
1469 let mut operation_id_by_token = BTreeMap::new();
1470 for id in operation_ids {
1471 let token = mm_dsl::OperationId::from_domain(id);
1472 let raw_id = token.0.clone();
1473 dsl_ids.insert(raw_id.clone());
1474 dsl_id_tokens.insert(token.clone());
1475 operation_token_by_id.insert(raw_id.clone(), token.clone());
1476 operation_id_by_token.insert(token, raw_id);
1477 }
1478 self.resolve_wait_all_admission(
1479 wait_request_id,
1480 operation_ids,
1481 &dsl_ids,
1482 &dsl_id_tokens,
1483 &operation_token_by_id,
1484 &operation_id_by_token,
1485 )?;
1486 self.dsl_apply(
1487 mm_dsl::MeerkatMachineInput::RequestWaitAll {
1488 run_id: mm_dsl::RunId::from_domain(run_id),
1489 wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1490 operation_id_sequence: operation_ids
1491 .iter()
1492 .map(|id| mm_dsl::OperationId::from_domain(id).0)
1493 .collect(),
1494 operation_ids: dsl_ids,
1495 operation_id_tokens: dsl_id_tokens,
1496 operation_token_by_id,
1497 operation_id_by_token,
1498 },
1499 "RequestWaitAll",
1500 )?;
1501 if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1502 return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1503 }
1504 Ok(WaitAllAuthorityPlan::ActivateBarrier)
1505 }
1506
1507 fn owner_termination_targets(
1508 &self,
1509 ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1510 let mut targets = Vec::new();
1511 for id in self.operation_ids()? {
1512 let status = self.require_status(&id)?;
1513 if !Self::operation_status_is_terminal(&id, status)? {
1514 targets.push((id, status));
1515 }
1516 }
1517 Ok(targets)
1518 }
1519
1520 fn operation_status_is_terminal(
1521 operation_id: &OperationId,
1522 status: OperationStatus,
1523 ) -> Result<bool, OpsLifecycleError> {
1524 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1525 let effects = Self::apply_stateless_classifier(
1526 mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1527 operation_id: operation_id_key.clone(),
1528 status: mm_dsl::OperationStatus::from(status),
1529 },
1530 "ClassifyOperationTerminality",
1531 )?;
1532 let mut terminal = None;
1533 for effect in effects {
1534 match effect {
1535 mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1536 if operation_id == operation_id_key =>
1537 {
1538 terminal = Some(true);
1539 }
1540 mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1541 if operation_id == operation_id_key =>
1542 {
1543 terminal = Some(false);
1544 }
1545 other => {
1546 return Err(OpsLifecycleError::Internal(format!(
1547 "unexpected generated operation terminality effect: {other:?}"
1548 )));
1549 }
1550 }
1551 }
1552 terminal.ok_or_else(|| {
1553 OpsLifecycleError::Internal(format!(
1554 "generated operation terminality authority emitted no effect for {operation_id}"
1555 ))
1556 })
1557 }
1558
1559 fn operation_public_result_class(
1560 operation_id: &OperationId,
1561 status: OperationStatus,
1562 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1563 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1564 let effects = Self::apply_stateless_classifier(
1565 mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1566 operation_id: operation_id_key.clone(),
1567 status: mm_dsl::OperationStatus::from(status),
1568 },
1569 "ClassifyOperationPublicResult",
1570 )?;
1571 let mut result = None;
1572 for effect in effects {
1573 match effect {
1574 mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1575 operation_id,
1576 result: classified,
1577 } if operation_id == operation_id_key => {
1578 result = Some(OperationPublicResultClass::from(classified));
1579 }
1580 other => {
1581 return Err(OpsLifecycleError::Internal(format!(
1582 "unexpected generated operation public-result effect: {other:?}"
1583 )));
1584 }
1585 }
1586 }
1587 result.ok_or_else(|| {
1588 OpsLifecycleError::Internal(format!(
1589 "generated operation public-result authority emitted no effect for {operation_id}"
1590 ))
1591 })
1592 }
1593
1594 fn operation_transition_rejection_is_idempotent(
1595 operation_id: &OperationId,
1596 action: OperationLifecycleAction,
1597 status: OperationStatus,
1598 ) -> Result<bool, OpsLifecycleError> {
1599 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1600 let action = mm_dsl::OpLifecycleActionKind::from(action);
1601 let status = mm_dsl::OperationStatus::from(status);
1602 let effects = Self::apply_stateless_classifier(
1603 mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1604 operation_id: operation_id_key.clone(),
1605 action,
1606 status,
1607 },
1608 "ClassifyOperationTransitionIdempotence",
1609 )?;
1610 let mut idempotent = None;
1611 for effect in effects {
1612 match effect {
1613 mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1614 operation_id,
1615 action: effect_action,
1616 status: effect_status,
1617 } if operation_id == operation_id_key
1618 && effect_action == action
1619 && effect_status == status =>
1620 {
1621 idempotent = Some(true);
1622 }
1623 mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1624 operation_id,
1625 action: effect_action,
1626 status: effect_status,
1627 } if operation_id == operation_id_key
1628 && effect_action == action
1629 && effect_status == status =>
1630 {
1631 idempotent = Some(false);
1632 }
1633 other => {
1634 return Err(OpsLifecycleError::Internal(format!(
1635 "unexpected generated operation transition-idempotence effect: {other:?}"
1636 )));
1637 }
1638 }
1639 }
1640 idempotent.ok_or_else(|| {
1641 OpsLifecycleError::Internal(format!(
1642 "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1643 ))
1644 })
1645 }
1646
1647 fn operation_completion_feed_class(
1648 operation_id: &OperationId,
1649 kind: OperationKind,
1650 ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1651 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1652 let kind = mm_dsl::OperationKind::from(kind);
1653 let effects = Self::apply_stateless_classifier(
1654 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1655 operation_id: operation_id_key.clone(),
1656 kind,
1657 },
1658 "ClassifyOperationCompletionFeed",
1659 )?;
1660 let mut class = None;
1661 for effect in effects {
1662 match effect {
1663 mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1664 operation_id,
1665 result,
1666 } if operation_id == operation_id_key => {
1667 class = Some(result);
1668 }
1669 other => {
1670 return Err(OpsLifecycleError::Internal(format!(
1671 "unexpected generated operation completion-feed effect: {other:?}"
1672 )));
1673 }
1674 }
1675 }
1676 class.ok_or_else(|| {
1677 OpsLifecycleError::Internal(format!(
1678 "generated operation completion-feed authority emitted no effect for {operation_id}"
1679 ))
1680 })
1681 }
1682
1683 fn operation_completion_wake_class(
1684 operation_id: &OperationId,
1685 kind: OperationKind,
1686 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1687 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1688 let kind = mm_dsl::OperationKind::from(kind);
1689 let effects = Self::apply_stateless_classifier(
1690 mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1691 operation_id: operation_id_key.clone(),
1692 kind,
1693 },
1694 "ClassifyOperationCompletionWake",
1695 )?;
1696 let mut class = None;
1697 for effect in effects {
1698 match effect {
1699 mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1700 operation_id,
1701 result,
1702 } if operation_id == operation_id_key => {
1703 class = Some(OperationCompletionWakeClass::from(result));
1704 }
1705 other => {
1706 return Err(OpsLifecycleError::Internal(format!(
1707 "unexpected generated operation completion-wake effect: {other:?}"
1708 )));
1709 }
1710 }
1711 }
1712 class.ok_or_else(|| {
1713 OpsLifecycleError::Internal(format!(
1714 "generated operation completion-wake authority emitted no effect for {operation_id}"
1715 ))
1716 })
1717 }
1718
1719 fn operation_durability_class(
1720 operation_id: &OperationId,
1721 kind: OperationKind,
1722 ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1723 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1724 let kind = mm_dsl::OperationKind::from(kind);
1725 let effects = Self::apply_stateless_classifier(
1726 mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1727 operation_id: operation_id_key.clone(),
1728 kind,
1729 },
1730 "ClassifyOperationDurability",
1731 )?;
1732 let mut class = None;
1733 for effect in effects {
1734 match effect {
1735 mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1736 operation_id,
1737 result,
1738 } if operation_id == operation_id_key => {
1739 class = Some(result);
1740 }
1741 other => {
1742 return Err(OpsLifecycleError::Internal(format!(
1743 "unexpected generated operation durability effect: {other:?}"
1744 )));
1745 }
1746 }
1747 }
1748 class.ok_or_else(|| {
1749 OpsLifecycleError::Internal(format!(
1750 "generated operation durability authority emitted no effect for {operation_id}"
1751 ))
1752 })
1753 }
1754
1755 fn recovered_operation_record_disposition(
1756 operation_id: &OperationId,
1757 status: OperationStatus,
1758 kind: OperationKind,
1759 terminal_outcome_present: bool,
1760 terminal_payload_present: bool,
1761 completion_sequence_present: bool,
1762 ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1763 let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1764 let effects = Self::apply_stateless_classifier(
1765 mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1766 operation_id: operation_id_key.clone(),
1767 status: mm_dsl::OperationStatus::from(status),
1768 kind: mm_dsl::OperationKind::from(kind),
1769 terminal_outcome_present,
1770 terminal_payload_present,
1771 completion_sequence_present,
1772 },
1773 "ClassifyRecoveredOperationRecord",
1774 )?;
1775 let mut disposition = None;
1776 for effect in effects {
1777 match effect {
1778 mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1779 if operation_id == operation_id_key =>
1780 {
1781 disposition = Some(RecoveredOperationRecordDisposition::Retain);
1782 }
1783 mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1784 if operation_id == operation_id_key =>
1785 {
1786 disposition = Some(RecoveredOperationRecordDisposition::Discard);
1787 }
1788 other => {
1789 return Err(OpsLifecycleError::Internal(format!(
1790 "unexpected generated recovered-operation classification effect: {other:?}"
1791 )));
1792 }
1793 }
1794 }
1795 disposition.ok_or_else(|| {
1796 OpsLifecycleError::Internal(format!(
1797 "generated recovered-operation classifier emitted no effect for {operation_id}"
1798 ))
1799 })
1800 }
1801
1802 fn apply_stateless_classifier(
1803 input: mm_dsl::MeerkatMachineInput,
1804 label: &'static str,
1805 ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1806 let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1807 "ops stateless classifier Initialize must be accepted",
1808 );
1809 let transition =
1810 mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1811 OpsLifecycleError::Internal(format!(
1812 "DSL rejected ops transition ({label}): {err:?}"
1813 ))
1814 })?;
1815 Ok(transition.into_effects())
1816 }
1817
1818 fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1842 let satisfied = match self.try_satisfy_wait_all_authority() {
1843 Ok(Some(satisfied)) => satisfied,
1844 Ok(None) => return Ok(()),
1845 Err(err) => {
1846 if let Some(pending) = self.pending_wait.take() {
1854 drop(pending.sender);
1855 }
1856 self.wait_request_id = None;
1857 return Err(err);
1858 }
1859 };
1860 let shell_wait_id = self.wait_request_id.take();
1861 if shell_wait_id
1862 .as_ref()
1863 .is_some_and(|id| id != &satisfied.wait_request_id)
1864 {
1865 tracing::error!(
1866 shell_wait_request_id = ?shell_wait_id,
1867 authority_wait_request_id = %satisfied.wait_request_id,
1868 "generated wait_all authority satisfied a different wait request"
1869 );
1870 }
1871 if let Some(pending) = self.pending_wait.take() {
1872 if pending.wait_request_id == satisfied.wait_request_id {
1873 let _ = pending.sender.send(satisfied);
1874 } else if let Some(shell_wait_id) = shell_wait_id {
1875 tracing::error!(
1876 shell_wait_request_id = %shell_wait_id,
1877 pending_wait_request_id = %pending.wait_request_id,
1878 authority_wait_request_id = %satisfied.wait_request_id,
1879 "generated wait_all authority satisfied without a matching pending waiter"
1880 );
1881 }
1882 }
1883 Ok(())
1884 }
1885
1886 fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1892 let (tx, epoch_id, cursor_state) = match (
1893 &self.persist_tx,
1894 &self.persist_epoch_id,
1895 &self.persist_cursor_state,
1896 ) {
1897 (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1898 _ => return Ok(()),
1899 };
1900
1901 let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1902 let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1903 let request = OpsLifecyclePersistenceRequest {
1904 snapshot,
1905 result_tx,
1906 };
1907
1908 tx.send(request).map_err(|_| {
1909 OpsLifecycleError::Internal(
1910 "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1911 .into(),
1912 )
1913 })?;
1914 result_rx.recv().map_err(|_| {
1915 OpsLifecycleError::Internal(
1916 "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1917 .into(),
1918 )
1919 })?
1920 }
1921
1922 fn capture_snapshot(
1924 &self,
1925 epoch_id: meerkat_core::RuntimeEpochId,
1926 _cursor_state: &meerkat_core::EpochCursorState,
1927 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1928 let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1929 for op_id in self.operation_ids()? {
1930 let status = self.require_status(&op_id)?;
1931 let kind = self.require_kind(&op_id)?;
1932 if Self::operation_durability_class(&op_id, kind)?
1933 != mm_dsl::OperationDurabilityClass::Retain
1934 {
1935 continue;
1936 }
1937 let peer_ready = self.require_peer_ready(&op_id)?;
1938 let progress_count = self.require_progress_count(&op_id)?;
1939 let operation_source = self.operation_source(&op_id)?;
1940 let terminal_outcome = self.terminal_outcome(&op_id)?;
1941 let completion_sequence = self.completion_sequence(&op_id);
1942 if terminal_outcome.is_some() && completion_sequence.is_none() {
1943 return Err(OpsLifecycleError::Internal(format!(
1944 "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1945 )));
1946 }
1947 if terminal_outcome.is_none() && completion_sequence.is_some() {
1948 return Err(OpsLifecycleError::Internal(format!(
1949 "generated op terminal authority has completion sequence for non-terminal {op_id}"
1950 )));
1951 }
1952 let terminal_buffered = terminal_outcome.is_some();
1953 let watcher_count = self
1954 .records
1955 .get(&op_id)
1956 .map(|r| r.watchers.len() as u32)
1957 .unwrap_or(0);
1958 operations.insert(
1959 op_id,
1960 OperationCanonicalState {
1961 status,
1962 kind,
1963 operation_source,
1964 peer_ready,
1965 progress_count,
1966 watcher_count,
1967 terminal_outcome,
1968 completion_sequence,
1969 terminal_buffered,
1970 },
1971 );
1972 }
1973 let operation_specs: HashMap<OperationId, OperationSpec> = self
1974 .records
1975 .iter()
1976 .filter(|(id, _)| operations.contains_key(*id))
1977 .map(|(id, record)| {
1978 let mut spec = record.spec.clone();
1979 let operation_source = operations
1980 .get(id)
1981 .and_then(|state| state.operation_source.as_ref());
1982 Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1983 (id.clone(), spec)
1984 })
1985 .collect();
1986 let completed_order: VecDeque<OperationId> = self
1987 .completed_order
1988 .iter()
1989 .filter(|id| operations.contains_key(*id))
1990 .cloned()
1991 .collect();
1992 let active_count = operations
1993 .iter()
1994 .filter(|(id, state)| {
1995 matches!(
1996 Self::operation_status_is_terminal(id, state.status),
1997 Ok(false)
1998 )
1999 })
2000 .count();
2001 let authority_completion_entries = self.completion_feed_authority_entries()?;
2002 let (published_completion_entries_by_id, completion_feed_retention): (
2003 HashMap<OperationId, CompletionEntry>,
2004 PersistedCompletionFeedRetention,
2005 ) = {
2006 let inner = self
2007 .feed_buffer
2008 .inner
2009 .read()
2010 .unwrap_or_else(std::sync::PoisonError::into_inner);
2011 let mut entries = HashMap::new();
2012 for entry in &inner.entries {
2013 if !authority_completion_entries.contains_key(&entry.operation_id) {
2014 return Err(OpsLifecycleError::Internal(format!(
2015 "public completion feed projection for {} has no generated authority",
2016 entry.operation_id
2017 )));
2018 }
2019 if entries
2020 .insert(entry.operation_id.clone(), entry.clone())
2021 .is_some()
2022 {
2023 return Err(OpsLifecycleError::Internal(format!(
2024 "public completion feed projection for {} appeared more than once",
2025 entry.operation_id
2026 )));
2027 }
2028 }
2029 (
2030 entries,
2031 PersistedCompletionFeedRetention {
2032 dropped_through: inner.dropped_through,
2033 dropped_count: inner.dropped_count,
2034 },
2035 )
2036 };
2037 let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
2038 .iter()
2039 .map(|(operation_id, authority_entry)| {
2040 if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
2041 if projection.seq != authority_entry.seq
2042 || projection.kind != authority_entry.kind
2043 || projection.terminal_outcome != authority_entry.terminal_outcome
2044 {
2045 return Err(OpsLifecycleError::Internal(format!(
2046 "public completion feed projection for {operation_id} drifted from generated authority"
2047 )));
2048 }
2049 return Ok(projection.clone());
2050 }
2051
2052 let display_name = self
2053 .records
2054 .get(operation_id)
2055 .map(|record| record.spec.display_name.clone())
2056 .unwrap_or_default();
2057 let completed_at_ms = self.records.get(operation_id).and_then(|record| {
2058 record
2059 .completed_at
2060 .map(|completed_at| record.epoch_millis_for_instant(completed_at))
2061 });
2062
2063 Ok(CompletionEntry {
2064 seq: authority_entry.seq,
2065 operation_id: operation_id.clone(),
2066 kind: authority_entry.kind,
2067 display_name,
2068 terminal_outcome: authority_entry.terminal_outcome.clone(),
2069 completed_at_ms,
2070 })
2071 })
2072 .collect::<Result<_, _>>()?;
2073 completion_entries.sort_by_key(|entry| entry.seq);
2074
2075 let authority_state = RegistryCanonicalState {
2076 operations,
2077 completion_feed_entries: authority_completion_entries,
2078 completed_order,
2079 max_completed: self.max_completed,
2080 max_concurrent: self.max_concurrent,
2081 active_count,
2082 wait_request_id: self.wait_request_id.clone(),
2083 wait_operation_ids: self.wait_operation_ids()?,
2084 next_completion_seq: self.dsl.0.state().next_completion_seq,
2085 };
2086
2087 Ok(PersistedOpsSnapshot {
2088 epoch_id,
2089 authority_state,
2090 operation_specs,
2091 completion_entries,
2092 completion_feed_retention: Some(completion_feed_retention),
2093 cursors: self.completion_cursor_snapshot(),
2094 })
2095 }
2096
2097 fn shell_record_mut(
2098 &mut self,
2099 id: &OperationId,
2100 ) -> Result<&mut ShellRecord, OpsLifecycleError> {
2101 self.records
2102 .get_mut(id)
2103 .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
2104 }
2105
2106 fn collect_wait_outcomes(
2107 &self,
2108 operation_ids: &[OperationId],
2109 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
2110 operation_ids
2111 .iter()
2112 .map(|operation_id| {
2113 let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
2114 OpsLifecycleError::Internal(format!(
2115 "wait_all completed without terminal outcome for {operation_id}"
2116 ))
2117 })?;
2118 Ok((operation_id.clone(), outcome))
2119 })
2120 .collect()
2121 }
2122}
2123
2124impl Default for ShellState {
2125 fn default() -> Self {
2126 Self::new(DEFAULT_MAX_COMPLETED, None)
2127 }
2128}
2129
2130#[derive(Debug, Clone)]
2136pub struct OpsLifecycleConfig {
2137 pub max_completed: usize,
2140 pub max_concurrent: Option<usize>,
2142}
2143
2144impl Default for OpsLifecycleConfig {
2145 fn default() -> Self {
2146 Self {
2147 max_completed: DEFAULT_MAX_COMPLETED,
2148 max_concurrent: None,
2149 }
2150 }
2151}
2152
2153#[derive(Debug)]
2160pub struct RuntimeOpsLifecycleRegistry {
2161 state: Arc<RwLock<ShellState>>,
2162}
2163
2164#[derive(Debug, Clone)]
2165pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2166 pub operation_count: usize,
2167 pub active_count: usize,
2168 pub wait_request_id: Option<WaitRequestId>,
2169 pub pending_wait_present: bool,
2170 pub pending_wait_request_id: Option<WaitRequestId>,
2171 pub wait_operation_ids: Vec<OperationId>,
2172 pub operations: Vec<OperationLifecycleSnapshot>,
2173}
2174
2175impl Default for RuntimeOpsLifecycleRegistry {
2176 fn default() -> Self {
2177 Self {
2178 state: Arc::new(RwLock::new(ShellState::default())),
2179 }
2180 }
2181}
2182
2183impl RuntimeOpsLifecycleRegistry {
2184 pub fn new() -> Self {
2185 let dsl = new_ops_dsl_authority();
2186 let feed_capacity = DEFAULT_MAX_COMPLETED.max(1);
2187 let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2188 Self {
2189 state: Arc::new(RwLock::new(ShellState {
2190 dsl,
2191 records: HashMap::new(),
2192 pending_wait: None,
2193 completed_order: VecDeque::new(),
2194 max_completed: DEFAULT_MAX_COMPLETED,
2195 max_concurrent: None,
2196 wait_request_id: None,
2197 feed_buffer,
2198 persist_tx: None,
2199 persistence_sealed: false,
2200 persist_epoch_id: None,
2201 persist_cursor_state: None,
2202 owner_retired: false,
2203 })),
2204 }
2205 }
2206
2207 pub fn with_config(config: OpsLifecycleConfig) -> Self {
2208 Self {
2209 state: Arc::new(RwLock::new(ShellState::new(
2210 config.max_completed,
2211 config.max_concurrent,
2212 ))),
2213 }
2214 }
2215
2216 fn recover_completion_feed_entry(
2217 shell: &mut ShellState,
2218 operation_id: &OperationId,
2219 entry: &CompletionFeedCanonicalState,
2220 ) -> Result<(), OpsLifecycleError> {
2221 let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2222 let terminal_outcome_kind =
2223 mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2224 let effects = shell.dsl_apply_with_effects(
2225 mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2226 operation_id: expected_operation_id.clone(),
2227 kind: mm_dsl::OperationKind::from(entry.kind),
2228 terminal_outcome: terminal_outcome_kind,
2229 terminal_payload: entry.terminal_outcome.clone(),
2230 completion_sequence: entry.seq,
2231 },
2232 "RecoverCompletionFeedEntry",
2233 )?;
2234 let recovered = effects.iter().find_map(|effect| match effect {
2235 mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2236 operation_id,
2237 seq,
2238 kind,
2239 terminal_outcome,
2240 terminal_payload,
2241 } => Some((
2242 operation_id,
2243 *seq,
2244 OperationKind::from(*kind),
2245 *terminal_outcome,
2246 terminal_payload,
2247 )),
2248 _ => None,
2249 });
2250 let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2251 return Err(OpsLifecycleError::Internal(
2252 "generated completion-feed recovery emitted no recovered entry".into(),
2253 ));
2254 };
2255 if operation_id != &expected_operation_id
2256 || seq != entry.seq
2257 || kind != entry.kind
2258 || terminal_outcome != terminal_outcome_kind
2259 || terminal_payload != &entry.terminal_outcome
2260 {
2261 return Err(OpsLifecycleError::Internal(format!(
2262 "generated completion-feed recovery drifted for {operation_id}"
2263 )));
2264 }
2265 Ok(())
2266 }
2267
2268 pub fn set_persistence_channel(
2274 &self,
2275 tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2276 epoch_id: meerkat_core::RuntimeEpochId,
2277 cursor_state: Arc<meerkat_core::EpochCursorState>,
2278 ) {
2279 if let Ok(mut state) = self.state.write() {
2280 if state.owner_retired || state.persistence_sealed {
2281 tracing::warn!(
2282 "ignored attempt to rewire a terminally sealed ops persistence channel"
2283 );
2284 return;
2285 }
2286 state.persist_tx = Some(tx);
2287 state.persist_epoch_id = Some(epoch_id);
2288 state.persist_cursor_state = Some(cursor_state);
2289 }
2290 }
2291
2292 pub(crate) fn retire_owner_for_unregister(
2301 &self,
2302 reason: String,
2303 ) -> Result<(), OpsLifecycleError> {
2304 let closed_sender = {
2305 let mut state = self.write_state()?;
2306 if state.owner_retired {
2307 return Ok(());
2308 }
2309 terminate_owner_locked(&mut state, &reason)?;
2310 state.owner_retired = true;
2311 state.persistence_sealed = true;
2312 state.persist_epoch_id = None;
2313 state.persist_cursor_state = None;
2314 state.persist_tx.take()
2315 };
2316 drop(closed_sender);
2317 Ok(())
2318 }
2319
2320 pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2327 let PersistedOpsSnapshot {
2328 epoch_id: _,
2329 authority_state,
2330 operation_specs,
2331 completion_entries,
2332 completion_feed_retention,
2333 cursors,
2334 } = snapshot;
2335 let max_completed = authority_state.max_completed;
2336 let max_concurrent = authority_state.max_concurrent;
2337 let next_completion_seq = authority_state.next_completion_seq;
2338 let authority_completion_entries = authority_state.completion_feed_entries;
2339 let authority_operations = authority_state.operations;
2340 let mut shell = ShellState::new(max_completed, max_concurrent);
2341
2342 let mut retained_ids: HashSet<OperationId> = HashSet::new();
2347 for (op_id, op_state) in authority_operations {
2348 let terminal_outcome = op_state
2349 .terminal_outcome
2350 .as_ref()
2351 .map(mm_dsl::OperationTerminalOutcomeKind::from);
2352 let terminal_payload = op_state.terminal_outcome.clone();
2353 let disposition = ShellState::recovered_operation_record_disposition(
2354 &op_id,
2355 op_state.status,
2356 op_state.kind,
2357 terminal_outcome.is_some(),
2358 terminal_payload.is_some(),
2359 op_state.completion_sequence.is_some(),
2360 )?;
2361 if disposition == RecoveredOperationRecordDisposition::Discard {
2362 continue;
2363 }
2364 if let Some(spec_source) = operation_specs
2365 .get(&op_id)
2366 .and_then(|spec| spec.operation_source.as_ref())
2367 && op_state.operation_source.as_ref() != Some(spec_source)
2368 {
2369 return Err(OpsLifecycleError::Internal(format!(
2370 "persisted operation source mirror for {op_id} drifted from generated authority"
2371 )));
2372 }
2373 let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2374 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2375 status: mm_dsl::OperationStatus::from(op_state.status),
2376 kind: mm_dsl::OperationKind::from(op_state.kind),
2377 source: op_state
2378 .operation_source
2379 .as_ref()
2380 .map(mm_dsl::OperationSource::from_domain),
2381 peer_ready: op_state.peer_ready,
2382 progress_count: u64::from(op_state.progress_count),
2383 terminal_outcome,
2384 terminal_payload,
2385 completion_sequence: op_state.completion_sequence,
2386 };
2387 shell.dsl_apply(recovery, "RecoverOpRecord")?;
2388 let recovered_seq = shell.completion_sequence(&op_id);
2389 if op_state.completion_sequence != recovered_seq {
2390 return Err(OpsLifecycleError::Internal(format!(
2391 "generated op recovery completion sequence mismatch for {op_id}"
2392 )));
2393 }
2394 retained_ids.insert(op_id);
2395 }
2396 shell.dsl_apply(
2397 mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2398 next_completion_seq,
2399 },
2400 "RecoverOpsCompletionCursor",
2401 )?;
2402 shell.dsl_apply(
2403 mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2404 agent_applied_cursor: cursors.agent_applied_cursor,
2405 runtime_observed_cursor: cursors.runtime_observed_seq,
2406 runtime_injected_cursor: cursors.runtime_last_injected_seq,
2407 },
2408 "RecoverCompletionConsumerCursors",
2409 )?;
2410
2411 let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2414 .iter()
2415 .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2416 .collect();
2417 recovered_completed.sort_by_key(|(seq, _)| *seq);
2418 shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2419
2420 for (operation_id, entry) in &authority_completion_entries {
2424 if !retained_ids.contains(operation_id) {
2425 Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2426 }
2427 }
2428
2429 let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2430 for (operation_id, entry) in &authority_completion_entries {
2431 let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2432 return Err(OpsLifecycleError::Internal(format!(
2433 "persisted completion feed authority for {operation_id} was not recovered"
2434 )));
2435 };
2436 if recovered_entry != entry {
2437 return Err(OpsLifecycleError::Internal(format!(
2438 "persisted completion feed authority drifted from generated recovery for {operation_id}"
2439 )));
2440 }
2441 }
2442
2443 let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2447 for entry in completion_entries {
2448 let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2449 return Err(OpsLifecycleError::Internal(format!(
2450 "persisted completion feed projection for {} has no generated feed authority",
2451 entry.operation_id
2452 )));
2453 };
2454 if authority_entry.seq != entry.seq
2455 || authority_entry.kind != entry.kind
2456 || authority_entry.terminal_outcome != entry.terminal_outcome
2457 {
2458 return Err(OpsLifecycleError::Internal(format!(
2459 "persisted completion feed projection for {} drifted from generated feed authority",
2460 entry.operation_id
2461 )));
2462 }
2463 projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2464 }
2465
2466 let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2467 canonical_feed_entries.into_iter().collect();
2468 recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2469 if let Some(retention) = completion_feed_retention {
2470 if retention.dropped_count > 0 && recovered_entries.is_empty() {
2471 return Err(OpsLifecycleError::Internal(
2472 "persisted completion-feed retention has no retained public suffix".to_string(),
2473 ));
2474 }
2475 if recovered_entries
2476 .first()
2477 .is_some_and(|(_, entry)| entry.seq <= retention.dropped_through)
2478 {
2479 return Err(OpsLifecycleError::Internal(
2480 "persisted completion-feed retention overlaps the retained public suffix"
2481 .to_string(),
2482 ));
2483 }
2484 shell.feed_buffer.recover_evicted_prefix(retention)?;
2485 }
2486 for (operation_id, entry) in recovered_entries {
2487 let projection = projection_entries_by_id.get(&operation_id);
2488 let display_name = operation_specs
2489 .get(&operation_id)
2490 .map(|spec| spec.display_name.clone())
2491 .or_else(|| projection.map(|entry| entry.display_name.clone()))
2492 .unwrap_or_default();
2493 let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2494 shell.feed_buffer.push(CompletionEntry {
2495 seq: entry.seq,
2496 operation_id,
2497 kind: entry.kind,
2498 display_name,
2499 terminal_outcome: entry.terminal_outcome,
2500 completed_at_ms,
2501 });
2502 }
2503
2504 for (op_id, spec) in operation_specs {
2507 if retained_ids.contains(&op_id) {
2508 let mut spec = spec;
2509 let operation_source = shell.operation_source(&op_id)?;
2510 ShellState::align_spec_child_session_id_to_source(
2511 &mut spec,
2512 operation_source.as_ref(),
2513 );
2514 shell.records.insert(
2515 op_id,
2516 ShellRecord {
2517 spec,
2518 peer_handle: None,
2519 watchers: Vec::new(),
2520 created_at: Instant::now(),
2521 started_at: None,
2522 completed_at: None,
2523 created_at_wall: SystemTime::now(),
2524 },
2525 );
2526 }
2527 }
2528
2529 Ok(Self {
2530 state: Arc::new(RwLock::new(shell)),
2531 })
2532 }
2533
2534 pub fn capture_persistence_snapshot(
2539 &self,
2540 epoch_id: meerkat_core::RuntimeEpochId,
2541 cursor_state: &meerkat_core::EpochCursorState,
2542 ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2543 let state = self
2544 .state
2545 .read()
2546 .unwrap_or_else(std::sync::PoisonError::into_inner);
2547 state.capture_snapshot(epoch_id, cursor_state)
2548 }
2549
2550 pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2552 let state = self
2553 .state
2554 .read()
2555 .unwrap_or_else(std::sync::PoisonError::into_inner);
2556 state.completion_cursor_snapshot()
2557 }
2558
2559 pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2561 let state = self
2562 .state
2563 .read()
2564 .unwrap_or_else(std::sync::PoisonError::into_inner);
2565 Arc::new(RuntimeCompletionFeed {
2566 buffer: Arc::clone(&state.feed_buffer),
2567 authority: Arc::downgrade(&self.state),
2568 })
2569 }
2570
2571 pub(crate) fn diagnostic_snapshot(
2573 &self,
2574 ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2575 let state = self
2576 .state
2577 .read()
2578 .unwrap_or_else(std::sync::PoisonError::into_inner);
2579 let mut operations = state
2580 .operation_ids()?
2581 .into_iter()
2582 .map(|id| state.snapshot(&id))
2583 .collect::<Result<Vec<_>, _>>()?
2584 .into_iter()
2585 .flatten()
2586 .collect::<Vec<_>>();
2587 operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2588 Ok(RuntimeOpsDiagnosticSnapshot {
2589 operation_count: state.operation_count(),
2590 active_count: state.active_count(),
2591 wait_request_id: state.wait_request_id.clone(),
2592 pending_wait_present: state.pending_wait.is_some(),
2593 pending_wait_request_id: state
2594 .pending_wait
2595 .as_ref()
2596 .map(|pending_wait| pending_wait.wait_request_id.clone()),
2597 wait_operation_ids: state.wait_operation_ids()?,
2598 operations,
2599 })
2600 }
2601
2602 fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2603 self.state
2604 .read()
2605 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2606 }
2607
2608 fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2609 self.state
2610 .write()
2611 .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2612 }
2613
2614 fn cancel_wait_all_internal(
2615 &self,
2616 wait_request_id: &WaitRequestId,
2617 ) -> Result<(), OpsLifecycleError> {
2618 let mut state = self.write_state()?;
2619 match state.wait_request_id.as_ref() {
2620 Some(active) if active == wait_request_id => {
2621 state.dsl_apply(
2626 mm_dsl::MeerkatMachineInput::CancelWaitAll,
2627 "CancelWaitAll(cancel)",
2628 )?;
2629 state.wait_request_id = None;
2630 if state
2631 .pending_wait
2632 .as_ref()
2633 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2634 {
2635 state.pending_wait = None;
2636 }
2637 Ok(())
2638 }
2639 _ => {
2640 if state
2641 .pending_wait
2642 .as_ref()
2643 .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2644 {
2645 state.pending_wait = None;
2646 }
2647 Ok(())
2648 }
2649 }
2650 }
2651}
2652
2653enum WaitAllFutureState {
2654 Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2655 Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2656 Done,
2657}
2658
2659struct WaitAllFuture<'a> {
2660 registry: &'a RuntimeOpsLifecycleRegistry,
2661 wait_request_id: WaitRequestId,
2662 state: WaitAllFutureState,
2663}
2664
2665impl Future for WaitAllFuture<'_> {
2666 type Output = Result<WaitAllResult, OpsLifecycleError>;
2667
2668 fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2669 match &mut self.state {
2670 WaitAllFutureState::Ready(result) => {
2671 let ready = result.take().unwrap_or_else(|| {
2672 Err(OpsLifecycleError::Internal(
2673 "wait_all future polled after completion".into(),
2674 ))
2675 });
2676 self.state = WaitAllFutureState::Done;
2677 Poll::Ready(ready)
2678 }
2679 WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2680 Poll::Pending => Poll::Pending,
2681 Poll::Ready(Ok(satisfied)) => {
2682 let outcomes = match self.registry.read_state() {
2683 Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2684 Err(err) => Err(err),
2685 };
2686 self.state = WaitAllFutureState::Done;
2687 Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2688 outcomes,
2689 satisfied,
2690 }))
2691 }
2692 Poll::Ready(Err(_)) => {
2693 self.state = WaitAllFutureState::Done;
2694 Poll::Ready(Err(OpsLifecycleError::Internal(
2695 "wait_all completion channel dropped".into(),
2696 )))
2697 }
2698 },
2699 WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2700 "wait_all future polled after completion".into(),
2701 ))),
2702 }
2703 }
2704}
2705
2706impl Drop for WaitAllFuture<'_> {
2707 fn drop(&mut self) {
2708 if matches!(self.state, WaitAllFutureState::Waiting(_))
2709 && let Err(err) = self
2710 .registry
2711 .cancel_wait_all_internal(&self.wait_request_id)
2712 {
2713 tracing::error!(
2714 wait_request_id = %self.wait_request_id,
2715 error = %err,
2716 "generated wait_all authority rejected cancellation during drop"
2717 );
2718 }
2719 }
2720}
2721
2722fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2727 match action {
2728 mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2729 mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2730 mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2731 mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2732 mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2733 mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2734 mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2735 mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2736 mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2737 mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2738 }
2739}
2740
2741fn op_lifecycle_rejection_error_from_effects(
2742 id: &OperationId,
2743 requested_action: mm_dsl::OpLifecycleActionKind,
2744 effects: &[mm_dsl::MeerkatMachineEffect],
2745) -> Result<OpsLifecycleError, OpsLifecycleError> {
2746 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2747 let mut rejection = None;
2748 for effect in effects {
2749 let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2750 operation_id,
2751 action,
2752 reason,
2753 status,
2754 } = effect
2755 else {
2756 continue;
2757 };
2758 if rejection.is_some() {
2759 return Err(OpsLifecycleError::Internal(
2760 "generated op lifecycle authority emitted multiple rejection results".into(),
2761 ));
2762 }
2763 if operation_id != &expected_id || *action != requested_action {
2764 return Err(OpsLifecycleError::Internal(format!(
2765 "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2766 )));
2767 }
2768 let error = match reason {
2769 mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2770 if status.is_some() {
2771 return Err(OpsLifecycleError::Internal(
2772 "generated op lifecycle authority emitted not-found with status".into(),
2773 ));
2774 }
2775 OpsLifecycleError::NotFound(id.clone())
2776 }
2777 mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2778 let status = status.ok_or_else(|| {
2779 OpsLifecycleError::Internal(
2780 "generated op lifecycle authority emitted invalid-transition without status"
2781 .into(),
2782 )
2783 })?;
2784 OpsLifecycleError::InvalidTransition {
2785 id: id.clone(),
2786 status: OperationStatus::from(status),
2787 action: op_lifecycle_action_label(requested_action),
2788 }
2789 }
2790 mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2791 if status.is_none() {
2792 return Err(OpsLifecycleError::Internal(
2793 "generated op lifecycle authority emitted peer-not-expected without status"
2794 .into(),
2795 ));
2796 }
2797 OpsLifecycleError::PeerNotExpected(id.clone())
2798 }
2799 mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2800 if status.is_none() {
2801 return Err(OpsLifecycleError::Internal(
2802 "generated op lifecycle authority emitted already-peer-ready without status"
2803 .into(),
2804 ));
2805 }
2806 OpsLifecycleError::AlreadyPeerReady(id.clone())
2807 }
2808 };
2809 rejection = Some(error);
2810 }
2811 rejection.ok_or_else(|| {
2812 OpsLifecycleError::Internal(
2813 "generated op lifecycle authority emitted no rejection result".into(),
2814 )
2815 })
2816}
2817
2818fn classify_generated_op_rejection(
2819 state: &mut ShellState,
2820 err: mm_dsl::MeerkatMachineTransitionError,
2821 id: &OperationId,
2822 action: mm_dsl::OpLifecycleActionKind,
2823) -> OpsLifecycleError {
2824 match err {
2825 mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2826 match state.dsl_apply_with_effects(
2827 mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2828 operation_id: mm_dsl::OperationId::from_domain(id).0,
2829 action,
2830 },
2831 "ResolveOpLifecycleTransitionRejection",
2832 ) {
2833 Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2834 .unwrap_or_else(|err| err),
2835 Err(err) => err,
2836 }
2837 }
2838 other => OpsLifecycleError::Internal(format!(
2839 "DSL rejected ops transition ({}): {other:?}",
2840 op_lifecycle_action_label(action)
2841 )),
2842 }
2843}
2844
2845fn apply_op_transition(
2846 state: &mut ShellState,
2847 id: &OperationId,
2848 input: mm_dsl::MeerkatMachineInput,
2849 action: mm_dsl::OpLifecycleActionKind,
2850) -> Result<(), OpsLifecycleError> {
2851 state.ensure_owner_active()?;
2852 state
2853 .dsl_apply_raw(input)
2854 .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2855}
2856
2857fn apply_terminal_op_transition_and_persist(
2858 state: &mut ShellState,
2859 id: &OperationId,
2860 input: mm_dsl::MeerkatMachineInput,
2861 action: mm_dsl::OpLifecycleActionKind,
2862) -> Result<(), OpsLifecycleError> {
2863 let previous_snapshot = state.dsl.0.snapshot();
2864 apply_op_transition(state, id, input, action)?;
2865 if let Err(err) = state.maybe_persist() {
2866 state.dsl.0.restore_snapshot(previous_snapshot);
2867 return Err(err);
2868 }
2869 Ok(())
2870}
2871
2872fn terminate_owner_locked(state: &mut ShellState, reason: &str) -> Result<(), OpsLifecycleError> {
2873 state.ensure_owner_active()?;
2874 let to_terminate = state.owner_termination_targets()?;
2875
2876 for (op_id, _status) in &to_terminate {
2877 let terminal_outcome = OperationTerminalOutcome::Terminated {
2878 reason: reason.to_owned(),
2879 };
2880 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2881
2882 apply_terminal_op_transition_and_persist(
2883 state,
2884 op_id,
2885 mm_dsl::MeerkatMachineInput::TerminateOp {
2886 operation_id: mm_dsl::OperationId::from_domain(op_id).0,
2887 outcome: outcome_kind,
2888 payload: terminal_outcome,
2889 },
2890 mm_dsl::OpLifecycleActionKind::Terminate,
2891 )?;
2892
2893 if let Some(entry) = state.finalize_terminal(op_id)? {
2894 state.publish_completion_entry(Some(entry));
2895 }
2896 }
2897 Ok(())
2898}
2899
2900fn op_registration_error_from_effects(
2901 id: &OperationId,
2902 effects: &[mm_dsl::MeerkatMachineEffect],
2903) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2904 let expected_id = mm_dsl::OperationId::from_domain(id).0;
2905 let mut admission = None;
2906 for effect in effects {
2907 let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2908 operation_id,
2909 result,
2910 reject_reason,
2911 max_concurrent_limit,
2912 active_op_count,
2913 } = effect
2914 else {
2915 continue;
2916 };
2917 if admission.is_some() {
2918 return Err(OpsLifecycleError::Internal(
2919 "generated op registration authority emitted multiple admission results".into(),
2920 ));
2921 }
2922 if operation_id != &expected_id {
2923 return Err(OpsLifecycleError::Internal(format!(
2924 "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2925 )));
2926 }
2927 admission = Some(match result {
2928 mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2929 if reject_reason.is_some() {
2930 return Err(OpsLifecycleError::Internal(
2931 "generated op registration authority accepted with rejection reason".into(),
2932 ));
2933 }
2934 None
2935 }
2936 mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2937 let reason = reject_reason.ok_or_else(|| {
2938 OpsLifecycleError::Internal(
2939 "generated op registration authority rejected without reason".into(),
2940 )
2941 })?;
2942 let error = match reason {
2943 mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2944 OpsLifecycleError::AlreadyRegistered(id.clone())
2945 }
2946 mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2947 let limit = max_concurrent_limit.ok_or_else(|| {
2948 OpsLifecycleError::Internal(
2949 "generated op registration authority rejected capacity without limit"
2950 .into(),
2951 )
2952 })?;
2953 OpsLifecycleError::MaxConcurrentExceeded {
2954 limit: limit as usize,
2955 active: *active_op_count as usize,
2956 }
2957 }
2958 };
2959 Some(error)
2960 }
2961 });
2962 }
2963 admission.ok_or_else(|| {
2964 OpsLifecycleError::Internal(
2965 "generated op registration authority emitted no admission result".into(),
2966 )
2967 })
2968}
2969
2970impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2971 fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2972 self.register_operation_with_admission_limit(spec, None)
2973 }
2974
2975 fn register_operation_with_admission_limit(
2976 &self,
2977 mut spec: OperationSpec,
2978 max_concurrent: Option<usize>,
2979 ) -> Result<(), OpsLifecycleError> {
2980 let mut state = self.write_state()?;
2981 state.ensure_owner_active()?;
2982 let operation_id = spec.id.clone();
2983 let kind = spec.kind;
2984 let max_concurrent = max_concurrent
2985 .or(state.max_concurrent)
2986 .map(|limit| limit as u64);
2987
2988 let effects = state.dsl_apply_with_effects(
2989 mm_dsl::MeerkatMachineInput::RegisterOp {
2990 operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2991 kind: mm_dsl::OperationKind::from_domain(&kind),
2992 source: spec
2993 .operation_source
2994 .as_ref()
2995 .map(mm_dsl::OperationSource::from_domain),
2996 max_concurrent,
2997 },
2998 "RegisterOp",
2999 )?;
3000 if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
3001 return Err(error);
3002 }
3003
3004 let authority_operation_source = state.operation_source(&operation_id)?;
3005 ShellState::align_spec_child_session_id_to_source(
3006 &mut spec,
3007 authority_operation_source.as_ref(),
3008 );
3009
3010 state.records.insert(operation_id, ShellRecord::new(spec));
3012 Ok(())
3013 }
3014
3015 fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3016 let mut state = self.write_state()?;
3017
3018 apply_op_transition(
3019 &mut state,
3020 id,
3021 mm_dsl::MeerkatMachineInput::StartOp {
3022 operation_id: mm_dsl::OperationId::from_domain(id).0,
3023 },
3024 mm_dsl::OpLifecycleActionKind::Start,
3025 )?;
3026
3027 if let Some(shell) = state.records.get_mut(id) {
3029 shell.started_at = Some(Instant::now());
3030 }
3031 Ok(())
3032 }
3033
3034 fn provisioning_failed(
3035 &self,
3036 id: &OperationId,
3037 error: String,
3038 ) -> Result<(), OpsLifecycleError> {
3039 let mut state = self.write_state()?;
3040
3041 let terminal_outcome = OperationTerminalOutcome::Failed { error };
3042 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3043
3044 apply_terminal_op_transition_and_persist(
3045 &mut state,
3046 id,
3047 mm_dsl::MeerkatMachineInput::FailOp {
3048 operation_id: mm_dsl::OperationId::from_domain(id).0,
3049 outcome: outcome_kind,
3050 payload: terminal_outcome,
3051 },
3052 mm_dsl::OpLifecycleActionKind::Fail,
3053 )?;
3054
3055 let completion_entry = state.finalize_terminal(id)?;
3056 state.publish_completion_entry(completion_entry);
3057 Ok(())
3058 }
3059
3060 fn peer_ready(
3061 &self,
3062 id: &OperationId,
3063 peer: OperationPeerHandle,
3064 ) -> Result<(), OpsLifecycleError> {
3065 let mut state = self.write_state()?;
3066
3067 apply_op_transition(
3068 &mut state,
3069 id,
3070 mm_dsl::MeerkatMachineInput::PeerReadyOp {
3071 operation_id: mm_dsl::OperationId::from_domain(id).0,
3072 },
3073 mm_dsl::OpLifecycleActionKind::PeerReady,
3074 )?;
3075
3076 if let Some(shell) = state.records.get_mut(id) {
3078 shell.peer_handle = Some(peer);
3079 }
3080 Ok(())
3081 }
3082
3083 fn register_watcher(
3084 &self,
3085 id: &OperationId,
3086 ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
3087 let mut state = self.write_state()?;
3088
3089 if !state.contains(id) {
3090 return Err(OpsLifecycleError::NotFound(id.clone()));
3091 }
3092
3093 if let Some(outcome) = state.terminal_outcome(id)? {
3095 return Ok(resolved_operation_completion_watch(outcome));
3096 }
3097
3098 let shell = state.shell_record_mut(id)?;
3100 let (tx, rx) = tokio::sync::oneshot::channel();
3101 let watch = operation_completion_watch_from_receiver(rx);
3102 shell.watchers.push(OperationCompletionNotifier::new(tx));
3103 Ok(watch)
3104 }
3105
3106 fn report_progress(
3107 &self,
3108 id: &OperationId,
3109 _update: OperationProgressUpdate,
3110 ) -> Result<(), OpsLifecycleError> {
3111 let mut state = self.write_state()?;
3112
3113 apply_op_transition(
3114 &mut state,
3115 id,
3116 mm_dsl::MeerkatMachineInput::ProgressReportedOp {
3117 operation_id: mm_dsl::OperationId::from_domain(id).0,
3118 },
3119 mm_dsl::OpLifecycleActionKind::ProgressReported,
3120 )?;
3121 Ok(())
3122 }
3123
3124 fn complete_operation(
3125 &self,
3126 id: &OperationId,
3127 result: OperationResult,
3128 ) -> Result<(), OpsLifecycleError> {
3129 let mut state = self.write_state()?;
3130
3131 let terminal_outcome = OperationTerminalOutcome::Completed(result);
3132 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3133
3134 apply_terminal_op_transition_and_persist(
3135 &mut state,
3136 id,
3137 mm_dsl::MeerkatMachineInput::CompleteOp {
3138 operation_id: mm_dsl::OperationId::from_domain(id).0,
3139 outcome: outcome_kind,
3140 payload: terminal_outcome,
3141 },
3142 mm_dsl::OpLifecycleActionKind::Complete,
3143 )?;
3144
3145 let completion_entry = state.finalize_terminal(id)?;
3146 state.publish_completion_entry(completion_entry);
3147 Ok(())
3148 }
3149
3150 fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
3151 let mut state = self.write_state()?;
3152
3153 let terminal_outcome = OperationTerminalOutcome::Failed { error };
3154 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3155
3156 apply_terminal_op_transition_and_persist(
3157 &mut state,
3158 id,
3159 mm_dsl::MeerkatMachineInput::FailOp {
3160 operation_id: mm_dsl::OperationId::from_domain(id).0,
3161 outcome: outcome_kind,
3162 payload: terminal_outcome,
3163 },
3164 mm_dsl::OpLifecycleActionKind::Fail,
3165 )?;
3166
3167 let completion_entry = state.finalize_terminal(id)?;
3168 state.publish_completion_entry(completion_entry);
3169 Ok(())
3170 }
3171
3172 fn abort_provisioning(
3173 &self,
3174 id: &OperationId,
3175 reason: Option<String>,
3176 ) -> Result<(), OpsLifecycleError> {
3177 let mut state = self.write_state()?;
3178
3179 let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
3180 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3181
3182 apply_terminal_op_transition_and_persist(
3183 &mut state,
3184 id,
3185 mm_dsl::MeerkatMachineInput::AbortOp {
3186 operation_id: mm_dsl::OperationId::from_domain(id).0,
3187 outcome: outcome_kind,
3188 payload: terminal_outcome,
3189 },
3190 mm_dsl::OpLifecycleActionKind::Abort,
3191 )?;
3192
3193 let completion_entry = state.finalize_terminal(id)?;
3194 state.publish_completion_entry(completion_entry);
3195 Ok(())
3196 }
3197
3198 fn rollback_unreturned_operation(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3199 let mut state = self.write_state()?;
3200 state.ensure_owner_active()?;
3201 let expected_id = mm_dsl::OperationId::from_domain(id).0;
3202 let effects = state.dsl_apply_with_effects(
3203 mm_dsl::MeerkatMachineInput::RollbackUnreturnedOp {
3204 operation_id: expected_id.clone(),
3205 },
3206 "RollbackUnreturnedOp",
3207 )?;
3208 let mut acknowledgements = effects.iter().filter(|effect| {
3209 matches!(
3210 effect,
3211 mm_dsl::MeerkatMachineEffect::UnreturnedOpRolledBack { operation_id }
3212 if operation_id == &expected_id
3213 )
3214 });
3215 if acknowledgements.next().is_none() || acknowledgements.next().is_some() {
3216 return Err(OpsLifecycleError::Internal(format!(
3217 "generated rollback authority did not emit exactly one matching acknowledgement for {id}"
3218 )));
3219 }
3220 state.records.remove(id);
3221 state.completed_order.retain(|queued| queued != id);
3222 Ok(())
3223 }
3224
3225 fn cancel_operation(
3226 &self,
3227 id: &OperationId,
3228 reason: Option<String>,
3229 ) -> Result<(), OpsLifecycleError> {
3230 let mut state = self.write_state()?;
3231
3232 let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
3233 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3234
3235 apply_terminal_op_transition_and_persist(
3236 &mut state,
3237 id,
3238 mm_dsl::MeerkatMachineInput::CancelOp {
3239 operation_id: mm_dsl::OperationId::from_domain(id).0,
3240 outcome: outcome_kind,
3241 payload: terminal_outcome,
3242 },
3243 mm_dsl::OpLifecycleActionKind::Cancel,
3244 )?;
3245
3246 let completion_entry = state.finalize_terminal(id)?;
3247 state.publish_completion_entry(completion_entry);
3248 Ok(())
3249 }
3250
3251 fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3252 let mut state = self.write_state()?;
3253
3254 apply_op_transition(
3255 &mut state,
3256 id,
3257 mm_dsl::MeerkatMachineInput::RetireRequestedOp {
3258 operation_id: mm_dsl::OperationId::from_domain(id).0,
3259 },
3260 mm_dsl::OpLifecycleActionKind::RetireRequested,
3261 )?;
3262 Ok(())
3263 }
3264
3265 fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3266 let mut state = self.write_state()?;
3267
3268 let terminal_outcome = OperationTerminalOutcome::Retired;
3269 let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3270
3271 apply_terminal_op_transition_and_persist(
3272 &mut state,
3273 id,
3274 mm_dsl::MeerkatMachineInput::RetireCompletedOp {
3275 operation_id: mm_dsl::OperationId::from_domain(id).0,
3276 outcome: outcome_kind,
3277 payload: terminal_outcome,
3278 },
3279 mm_dsl::OpLifecycleActionKind::RetireCompleted,
3280 )?;
3281
3282 let completion_entry = state.finalize_terminal(id)?;
3283 state.publish_completion_entry(completion_entry);
3284 Ok(())
3285 }
3286
3287 fn snapshot(
3288 &self,
3289 id: &OperationId,
3290 ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3291 let state = self.read_state()?;
3292 state.snapshot(id)
3293 }
3294
3295 fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3296 let state = self.read_state()?;
3297 let mut snapshots = Vec::new();
3298 for id in state.operation_ids()? {
3299 let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3300 OpsLifecycleError::Internal(format!(
3301 "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3302 ))
3303 })?;
3304 snapshots.push(snapshot);
3305 }
3306 snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3307 Ok(snapshots)
3308 }
3309
3310 fn classify_operation_terminality(
3311 &self,
3312 id: &OperationId,
3313 status: OperationStatus,
3314 ) -> Result<bool, OpsLifecycleError> {
3315 ShellState::operation_status_is_terminal(id, status)
3316 }
3317
3318 fn classify_operation_public_result(
3319 &self,
3320 id: &OperationId,
3321 ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3322 let state = self.read_state()?;
3323 let status = match state.status(id) {
3324 Some(status) => status,
3325 None if state.records.contains_key(id)
3326 || state.has_generated_operation_record_fact(id) =>
3327 {
3328 return Err(OpsLifecycleError::Internal(format!(
3329 "generated op lifecycle authority missing status for {id}"
3330 )));
3331 }
3332 None => OperationStatus::Absent,
3333 };
3334 ShellState::operation_public_result_class(id, status)
3335 }
3336
3337 fn classify_operation_completion_wake(
3338 &self,
3339 id: &OperationId,
3340 kind: OperationKind,
3341 ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3342 ShellState::operation_completion_wake_class(id, kind)
3343 }
3344
3345 fn classify_operation_transition_idempotence(
3346 &self,
3347 id: &OperationId,
3348 action: OperationLifecycleAction,
3349 ) -> Result<bool, OpsLifecycleError> {
3350 let state = self.read_state()?;
3351 let status = match state.status(id) {
3352 Some(status) => status,
3353 None if state.records.contains_key(id)
3354 || state.has_generated_operation_record_fact(id) =>
3355 {
3356 return Err(OpsLifecycleError::Internal(format!(
3357 "generated op lifecycle authority missing status for {id}"
3358 )));
3359 }
3360 None => OperationStatus::Absent,
3361 };
3362 ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3363 }
3364
3365 fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3366 let mut state = self.write_state()?;
3367 if state.owner_retired {
3368 return Ok(());
3369 }
3370 terminate_owner_locked(&mut state, &reason)
3371 }
3372
3373 fn collect_completed(
3374 &self,
3375 ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3376 let mut state = self.write_state()?;
3377 state.ensure_owner_active()?;
3378
3379 let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3380 let mut collected = Vec::with_capacity(ids.len());
3381 for id in ids {
3382 let outcome = state.terminal_outcome(&id)?;
3383 state.dsl_apply(
3384 mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3385 operation_id: mm_dsl::OperationId::from_domain(&id).0,
3386 },
3387 "CollectCompletedOp",
3388 )?;
3389 state.completed_order.retain(|queued| queued != &id);
3390 state.records.remove(&id);
3391 if let Some(outcome) = outcome {
3392 collected.push((id, outcome));
3393 }
3394 }
3395 Ok(collected)
3396 }
3397
3398 fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3399 Some(self.completion_feed_handle())
3400 }
3401
3402 fn completion_cursor(
3403 &self,
3404 consumer: CompletionCursorConsumer,
3405 ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3406 let state = self.read_state()?;
3407 Ok(Some(state.completion_cursor(consumer)))
3408 }
3409
3410 fn advance_completion_cursor(
3411 &self,
3412 consumer: CompletionCursorConsumer,
3413 cursor: CompletionSeq,
3414 projection: Option<&meerkat_core::EpochCursorState>,
3415 ) -> Result<CompletionSeq, OpsLifecycleError> {
3416 let mut state = self.write_state()?;
3417 state.ensure_owner_active()?;
3418 let input = match consumer {
3419 CompletionCursorConsumer::AgentApplied => {
3420 mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3421 }
3422 CompletionCursorConsumer::RuntimeObserved => {
3423 mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3424 }
3425 CompletionCursorConsumer::RuntimeInjected => {
3426 mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3427 }
3428 };
3429 let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3430 let advanced = effects
3431 .iter()
3432 .find_map(|effect| match (consumer, effect) {
3433 (
3434 CompletionCursorConsumer::AgentApplied,
3435 mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3436 ) => Some(*cursor),
3437 (
3438 CompletionCursorConsumer::RuntimeObserved,
3439 mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3440 cursor,
3441 },
3442 ) => Some(*cursor),
3443 (
3444 CompletionCursorConsumer::RuntimeInjected,
3445 mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3446 cursor,
3447 },
3448 ) => Some(*cursor),
3449 _ => None,
3450 })
3451 .ok_or_else(|| {
3452 OpsLifecycleError::Internal(format!(
3453 "generated completion cursor transition emitted no feedback for {consumer:?}"
3454 ))
3455 })?;
3456 if let Some(projection) = projection {
3457 projection.project_authorized_completion_cursor(consumer, advanced);
3458 }
3459 Ok(advanced)
3460 }
3461
3462 fn wait_all(
3463 &self,
3464 run_id: &RunId,
3465 ids: &[OperationId],
3466 ) -> std::pin::Pin<
3467 Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3468 > {
3469 let wait_request_id = WaitRequestId::new();
3470 let owned_ids = ids.to_vec();
3471
3472 let state = match self.write_state() {
3473 Ok(mut state) => {
3474 if let Err(error) = state.ensure_owner_active() {
3475 return Box::pin(WaitAllFuture {
3476 registry: self,
3477 wait_request_id,
3478 state: WaitAllFutureState::Ready(Some(Err(error))),
3479 });
3480 }
3481 match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3482 Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3483 let outcomes =
3484 state
3485 .collect_wait_outcomes(&satisfied.operation_ids)
3486 .map(|outcomes| WaitAllResult {
3487 outcomes,
3488 satisfied,
3489 });
3490 WaitAllFutureState::Ready(Some(outcomes))
3491 }
3492 Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3493 if state.pending_wait.is_some() {
3494 let rollback = state.dsl_apply(
3500 mm_dsl::MeerkatMachineInput::CancelWaitAll,
3501 "CancelWaitAll(rollback)",
3502 );
3503 return Box::pin(WaitAllFuture {
3504 registry: self,
3505 wait_request_id,
3506 state: WaitAllFutureState::Ready(Some(Err(match rollback {
3507 Ok(()) => OpsLifecycleError::Internal(
3508 "wait_all started while a pending wait sender already existed"
3509 .into(),
3510 ),
3511 Err(err) => err,
3512 }))),
3513 });
3514 }
3515 state.wait_request_id = Some(wait_request_id.clone());
3516 let (sender, receiver) = tokio::sync::oneshot::channel();
3517 state.pending_wait = Some(PendingWaitState {
3518 wait_request_id: wait_request_id.clone(),
3519 sender,
3520 });
3521 WaitAllFutureState::Waiting(receiver)
3522 }
3523 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3524 }
3525 }
3526 Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3527 };
3528
3529 Box::pin(WaitAllFuture {
3530 registry: self,
3531 wait_request_id,
3532 state,
3533 })
3534 }
3535}
3536
3537#[cfg(test)]
3538#[allow(clippy::unwrap_used, clippy::panic)]
3539mod tests {
3540 use super::*;
3541 use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3542 use meerkat_core::lifecycle::RunId;
3543 use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3544 use meerkat_core::types::SessionId;
3545 use std::sync::atomic::Ordering;
3546 use uuid::Uuid;
3547
3548 fn test_run_id() -> RunId {
3549 RunId(Uuid::from_u128(1))
3550 }
3551
3552 fn background_spec(name: &str) -> OperationSpec {
3553 OperationSpec {
3554 id: OperationId::new(),
3555 kind: OperationKind::BackgroundToolOp,
3556 owner_session_id: SessionId::new(),
3557 display_name: name.into(),
3558 source_label: "test".into(),
3559 operation_source: None,
3560 child_session_id: None,
3561 expect_peer_channel: false,
3562 }
3563 }
3564
3565 #[tokio::test]
3566 async fn late_watchers_resolve_immediately() {
3567 let registry = RuntimeOpsLifecycleRegistry::new();
3568 let spec = background_spec("late");
3569 let op_id = spec.id.clone();
3570 registry.register_operation(spec).unwrap();
3571 registry.provisioning_succeeded(&op_id).unwrap();
3572 registry
3573 .complete_operation(
3574 &op_id,
3575 OperationResult {
3576 id: op_id.clone(),
3577 content: "done".into(),
3578 is_error: false,
3579 duration_ms: 1,
3580 tokens_used: 0,
3581 },
3582 )
3583 .unwrap();
3584
3585 let watch = registry.register_watcher(&op_id).unwrap();
3586 match watch
3587 .await
3588 .expect("operation completion watch should resolve")
3589 {
3590 OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3591 other => panic!("expected completed outcome, got {other:?}"),
3592 }
3593 }
3594
3595 #[tokio::test]
3596 async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3597 let (tx, rx) = tokio::sync::oneshot::channel();
3598 let watch = operation_completion_watch_from_receiver(rx);
3599 drop(tx);
3600
3601 assert_eq!(
3602 watch.await,
3603 Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3604 );
3605 }
3606
3607 #[test]
3608 fn peer_ready_requires_peer_expectation() {
3609 let registry = RuntimeOpsLifecycleRegistry::new();
3610 let spec = background_spec("no-peer");
3611 let op_id = spec.id.clone();
3612 registry.register_operation(spec).unwrap();
3613 registry.provisioning_succeeded(&op_id).unwrap();
3614
3615 let result = registry.peer_ready(
3616 &op_id,
3617 OperationPeerHandle {
3618 peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3619 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3620 "peer",
3621 PeerId::new(),
3622 "inproc://peer",
3623 )
3624 .unwrap(),
3625 },
3626 );
3627 assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3628 }
3629
3630 #[test]
3636 fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3637 let op_id = OperationId::new();
3638 let outcomes = vec![
3639 (
3640 OperationTerminalOutcome::Completed(OperationResult {
3641 id: op_id.clone(),
3642 content: "done".into(),
3643 is_error: false,
3644 duration_ms: 7,
3645 tokens_used: 42,
3646 }),
3647 mm_dsl::OperationTerminalOutcomeKind::Completed,
3648 ),
3649 (
3650 OperationTerminalOutcome::Failed {
3651 error: "boom".into(),
3652 },
3653 mm_dsl::OperationTerminalOutcomeKind::Failed,
3654 ),
3655 (
3656 OperationTerminalOutcome::Aborted {
3657 reason: Some("user aborted".into()),
3658 },
3659 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3660 ),
3661 (
3662 OperationTerminalOutcome::Aborted { reason: None },
3663 mm_dsl::OperationTerminalOutcomeKind::Aborted,
3664 ),
3665 (
3666 OperationTerminalOutcome::Cancelled {
3667 reason: Some("cancelled".into()),
3668 },
3669 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3670 ),
3671 (
3672 OperationTerminalOutcome::Cancelled { reason: None },
3673 mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3674 ),
3675 (
3676 OperationTerminalOutcome::Retired,
3677 mm_dsl::OperationTerminalOutcomeKind::Retired,
3678 ),
3679 (
3680 OperationTerminalOutcome::Terminated {
3681 reason: "owner stopped".into(),
3682 },
3683 mm_dsl::OperationTerminalOutcomeKind::Terminated,
3684 ),
3685 ];
3686
3687 for (outcome, expected_kind) in &outcomes {
3688 assert_eq!(
3689 mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3690 *expected_kind,
3691 "typed payload {outcome:?} must classify to {expected_kind:?}"
3692 );
3693 let read = ShellState::checked_terminal_payload(
3694 *expected_kind,
3695 outcome,
3696 "test authority",
3697 "test-op",
3698 )
3699 .expect("matching discriminant must read back the exact payload");
3700 assert_eq!(&read, outcome);
3701 }
3702
3703 let err = ShellState::checked_terminal_payload(
3705 mm_dsl::OperationTerminalOutcomeKind::Completed,
3706 &OperationTerminalOutcome::Retired,
3707 "test authority",
3708 "test-op",
3709 )
3710 .expect_err("variant mismatch must be rejected");
3711 assert!(matches!(err, OpsLifecycleError::Internal(_)));
3712 }
3713
3714 #[test]
3720 fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3721 let registry = RuntimeOpsLifecycleRegistry::new();
3722 let spec = background_spec("variant-mismatch");
3723 let op_id = spec.id.clone();
3724 registry.register_operation(spec).unwrap();
3725 registry.provisioning_succeeded(&op_id).unwrap();
3726
3727 let mut state = registry.write_state().unwrap();
3728 let err = state
3729 .dsl_apply(
3730 mm_dsl::MeerkatMachineInput::CompleteOp {
3731 operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3732 outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3733 payload: OperationTerminalOutcome::Retired,
3735 },
3736 "CompleteOp",
3737 )
3738 .expect_err("machine must reject payload variant mismatch");
3739 assert!(
3746 matches!(
3747 &err,
3748 OpsLifecycleError::Internal(message)
3749 if message.contains("GuardRejected") && message.contains("CompleteOp")
3750 ),
3751 "expected generated guard rejection for CompleteOp, got: {err:?}"
3752 );
3753 drop(state);
3754
3755 registry
3758 .complete_operation(
3759 &op_id,
3760 OperationResult {
3761 id: op_id.clone(),
3762 content: "done".into(),
3763 is_error: false,
3764 duration_ms: 1,
3765 tokens_used: 0,
3766 },
3767 )
3768 .expect("matching variant must complete");
3769 }
3770
3771 #[test]
3772 fn duplicate_registration_rejection_is_generated() {
3773 let registry = RuntimeOpsLifecycleRegistry::new();
3774 let spec = background_spec("duplicate");
3775 let op_id = spec.id.clone();
3776
3777 registry.register_operation(spec.clone()).unwrap();
3778 let result = registry.register_operation(spec);
3779
3780 assert!(matches!(
3781 result,
3782 Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3783 ));
3784 }
3785
3786 #[test]
3787 fn invalid_transition_rejection_is_generated() {
3788 let registry = RuntimeOpsLifecycleRegistry::new();
3789 let spec = background_spec("invalid-transition");
3790 let op_id = spec.id.clone();
3791 registry.register_operation(spec).unwrap();
3792
3793 let result = registry.complete_operation(
3794 &op_id,
3795 OperationResult {
3796 id: op_id.clone(),
3797 content: "too-early".into(),
3798 is_error: false,
3799 duration_ms: 1,
3800 tokens_used: 0,
3801 },
3802 );
3803
3804 assert!(matches!(
3805 result,
3806 Err(OpsLifecycleError::InvalidTransition {
3807 id,
3808 status: OperationStatus::Provisioning,
3809 action: "complete_operation",
3810 }) if id == op_id
3811 ));
3812 }
3813
3814 #[tokio::test]
3815 async fn multi_listener_completion() {
3816 let registry = RuntimeOpsLifecycleRegistry::new();
3817 let spec = background_spec("multi");
3818 let op_id = spec.id.clone();
3819 registry.register_operation(spec).unwrap();
3820 registry.provisioning_succeeded(&op_id).unwrap();
3821
3822 let watch1 = registry.register_watcher(&op_id).unwrap();
3823 let watch2 = registry.register_watcher(&op_id).unwrap();
3824 let watch3 = registry.register_watcher(&op_id).unwrap();
3825
3826 registry
3827 .complete_operation(
3828 &op_id,
3829 OperationResult {
3830 id: op_id.clone(),
3831 content: "multi-done".into(),
3832 is_error: false,
3833 duration_ms: 1,
3834 tokens_used: 0,
3835 },
3836 )
3837 .unwrap();
3838
3839 for watch in [watch1, watch2, watch3] {
3840 match watch
3841 .await
3842 .expect("operation completion watch should resolve")
3843 {
3844 OperationTerminalOutcome::Completed(result) => {
3845 assert_eq!(result.content, "multi-done");
3846 }
3847 other => panic!("expected completed, got {other:?}"),
3848 }
3849 }
3850 }
3851
3852 #[tokio::test]
3853 async fn wait_all_returns_all_outcomes() {
3854 let registry = RuntimeOpsLifecycleRegistry::new();
3855
3856 let spec_a = background_spec("a");
3857 let id_a = spec_a.id.clone();
3858 registry.register_operation(spec_a).unwrap();
3859 registry.provisioning_succeeded(&id_a).unwrap();
3860
3861 let spec_b = background_spec("b");
3862 let id_b = spec_b.id.clone();
3863 registry.register_operation(spec_b).unwrap();
3864 registry.provisioning_succeeded(&id_b).unwrap();
3865
3866 registry
3867 .complete_operation(
3868 &id_a,
3869 OperationResult {
3870 id: id_a.clone(),
3871 content: "a-done".into(),
3872 is_error: false,
3873 duration_ms: 1,
3874 tokens_used: 0,
3875 },
3876 )
3877 .unwrap();
3878 registry.fail_operation(&id_b, "b-error".into()).unwrap();
3879
3880 let wait_result = registry
3881 .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3882 .await
3883 .unwrap();
3884 assert_eq!(wait_result.outcomes.len(), 2);
3885 assert_eq!(wait_result.outcomes[0].0, id_a);
3886 assert!(matches!(
3887 wait_result.outcomes[0].1,
3888 OperationTerminalOutcome::Completed(_)
3889 ));
3890 assert_eq!(wait_result.outcomes[1].0, id_b);
3891 assert!(matches!(
3892 wait_result.outcomes[1].1,
3893 OperationTerminalOutcome::Failed { .. }
3894 ));
3895 assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3897 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3898 }
3899
3900 #[tokio::test]
3903 async fn wait_all_trait_path_submits_through_authority() {
3904 let registry = RuntimeOpsLifecycleRegistry::new();
3905 let spec = background_spec("trait-wait");
3906 let op_id = spec.id.clone();
3907 registry.register_operation(spec).unwrap();
3908 registry.provisioning_succeeded(&op_id).unwrap();
3909 registry
3910 .complete_operation(
3911 &op_id,
3912 OperationResult {
3913 id: op_id.clone(),
3914 content: "done".into(),
3915 is_error: false,
3916 duration_ms: 1,
3917 tokens_used: 0,
3918 },
3919 )
3920 .unwrap();
3921
3922 let trait_ref: &dyn OpsLifecycleRegistry = ®istry;
3924 let wait_result = trait_ref
3925 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3926 .await
3927 .unwrap();
3928 assert_eq!(wait_result.outcomes.len(), 1);
3929 assert!(matches!(
3930 wait_result.outcomes[0].1,
3931 OperationTerminalOutcome::Completed(_)
3932 ));
3933 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3935 assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3936 let state = registry.read_state().unwrap();
3937 assert!(
3938 !state.wait_active(),
3939 "already-satisfied wait_all must be cleared by generated satisfaction authority"
3940 );
3941 assert!(state.wait_operation_ids().unwrap().is_empty());
3942 }
3943
3944 #[tokio::test]
3945 async fn wait_all_duplicate_rejection_is_generated() {
3946 let registry = RuntimeOpsLifecycleRegistry::new();
3947 let spec = background_spec("duplicate-wait");
3948 let op_id = spec.id.clone();
3949 registry.register_operation(spec).unwrap();
3950 registry.provisioning_succeeded(&op_id).unwrap();
3951
3952 let result = registry
3953 .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3954 .await;
3955
3956 assert!(matches!(
3957 result,
3958 Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3959 ));
3960 let state = registry.read_state().unwrap();
3961 assert!(
3962 !state.wait_active(),
3963 "duplicate wait rejection must not create a shell or machine barrier"
3964 );
3965 assert!(state.wait_operation_ids().unwrap().is_empty());
3966 }
3967
3968 #[tokio::test]
3969 async fn wait_all_active_rejection_is_generated() {
3970 let registry = RuntimeOpsLifecycleRegistry::new();
3971 let spec = background_spec("active-wait");
3972 let op_id = spec.id.clone();
3973 registry.register_operation(spec).unwrap();
3974 registry.provisioning_succeeded(&op_id).unwrap();
3975
3976 let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3977 let result = registry
3978 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3979 .await;
3980
3981 assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3982 drop(active_wait);
3983 let state = registry.read_state().unwrap();
3984 assert!(!state.wait_active());
3985 assert!(state.wait_operation_ids().unwrap().is_empty());
3986 }
3987
3988 #[tokio::test]
3989 async fn wait_all_unknown_operation_rejection_is_generated() {
3990 let registry = RuntimeOpsLifecycleRegistry::new();
3991 let op_id = OperationId::new();
3992
3993 let result = registry
3994 .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3995 .await;
3996
3997 assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3998 let state = registry.read_state().unwrap();
3999 assert!(!state.wait_active());
4000 assert!(state.wait_operation_ids().unwrap().is_empty());
4001 }
4002
4003 #[tokio::test]
4004 async fn wait_all_resolves_from_authority_owned_wait_request() {
4005 let registry = RuntimeOpsLifecycleRegistry::new();
4006 let run_id = test_run_id();
4007
4008 let spec = background_spec("pending");
4009 let op_id = spec.id.clone();
4010 registry.register_operation(spec).unwrap();
4011 registry.provisioning_succeeded(&op_id).unwrap();
4012
4013 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
4014 tokio::pin!(wait_fut);
4015 assert!(
4016 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
4017 .await
4018 .is_err()
4019 );
4020
4021 let active_wait_request_id = {
4022 let state = registry.read_state().unwrap();
4023 let wait_request_id = match state.wait_request_id.clone() {
4024 Some(wait_request_id) => wait_request_id,
4025 None => panic!("wait request should be active"),
4026 };
4027 assert_eq!(
4028 state.wait_operation_ids().unwrap().as_slice(),
4029 std::slice::from_ref(&op_id)
4030 );
4031 wait_request_id
4032 };
4033
4034 registry
4035 .complete_operation(
4036 &op_id,
4037 OperationResult {
4038 id: op_id.clone(),
4039 content: "done".into(),
4040 is_error: false,
4041 duration_ms: 1,
4042 tokens_used: 0,
4043 },
4044 )
4045 .unwrap();
4046
4047 let wait_result = wait_fut.await.unwrap();
4048 assert_eq!(
4049 wait_result.satisfied.wait_request_id,
4050 active_wait_request_id
4051 );
4052 assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
4053 assert!(matches!(
4054 wait_result.outcomes.as_slice(),
4055 [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
4056 ));
4057 assert!(registry.read_state().unwrap().wait_request_id.is_none());
4058 }
4059
4060 #[tokio::test]
4061 async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
4062 let registry = RuntimeOpsLifecycleRegistry::new();
4063 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
4064 registry.set_persistence_channel(
4065 tx,
4066 meerkat_core::RuntimeEpochId::new(),
4067 Arc::new(meerkat_core::EpochCursorState::new()),
4068 );
4069 let worker = std::thread::spawn(move || {
4070 let request = rx.blocking_recv().expect("persistence request");
4071 let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
4072 "injected persistence failure".to_string(),
4073 )));
4074 });
4075
4076 let spec = background_spec("feed-after-persist");
4077 let op_id = spec.id.clone();
4078 registry.register_operation(spec).unwrap();
4079 registry.provisioning_succeeded(&op_id).unwrap();
4080 let watch = registry.register_watcher(&op_id).unwrap();
4081
4082 let err = registry
4083 .complete_operation(
4084 &op_id,
4085 OperationResult {
4086 id: op_id.clone(),
4087 content: "done".into(),
4088 is_error: false,
4089 duration_ms: 1,
4090 tokens_used: 0,
4091 },
4092 )
4093 .expect_err("persistence failure must fail the terminal transition");
4094 assert!(
4095 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
4096 "unexpected persistence error: {err:?}"
4097 );
4098 worker.join().expect("persistence worker");
4099
4100 let feed = registry.completion_feed_handle();
4101 let batch = feed.list_since(0);
4102 assert!(
4103 batch.entries.is_empty(),
4104 "completion feed must not publish entries before durable snapshot success"
4105 );
4106 assert_eq!(batch.watermark, 0);
4107 assert!(
4108 tokio::time::timeout(std::time::Duration::from_millis(10), watch)
4109 .await
4110 .is_err(),
4111 "watchers must not resolve before durable terminal snapshot success"
4112 );
4113 assert_eq!(
4114 registry.snapshot(&op_id).unwrap().unwrap().status,
4115 OperationStatus::Running,
4116 "failed persistence must roll the generated terminal transition back"
4117 );
4118
4119 let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
4120 registry.set_persistence_channel(
4121 tx,
4122 meerkat_core::RuntimeEpochId::new(),
4123 Arc::new(meerkat_core::EpochCursorState::new()),
4124 );
4125 let worker = std::thread::spawn(move || {
4126 let request = rx.blocking_recv().expect("second persistence request");
4127 let _ = request.result_tx.send(Ok(()));
4128 });
4129 registry
4130 .complete_operation(
4131 &op_id,
4132 OperationResult {
4133 id: op_id.clone(),
4134 content: "done".into(),
4135 is_error: false,
4136 duration_ms: 1,
4137 tokens_used: 0,
4138 },
4139 )
4140 .expect("rolled-back terminal transition should remain retryable");
4141 worker.join().expect("second persistence worker");
4142
4143 let batch = feed.list_since(0);
4144 assert_eq!(batch.entries.len(), 1);
4145 assert_eq!(batch.entries[0].operation_id, op_id);
4146 }
4147
4148 #[test]
4149 fn bounded_completion_feed_reports_exact_evicted_cursor_gap() {
4150 let buffer = Arc::new(FeedBuffer::new(1));
4151 for seq in 1..=2 {
4152 let operation_id = OperationId::new();
4153 buffer.push(CompletionEntry {
4154 seq,
4155 operation_id: operation_id.clone(),
4156 kind: OperationKind::BackgroundToolOp,
4157 display_name: format!("completion-{seq}"),
4158 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
4159 id: operation_id,
4160 content: "done".into(),
4161 is_error: false,
4162 duration_ms: 1,
4163 tokens_used: 0,
4164 }),
4165 completed_at_ms: None,
4166 });
4167 }
4168 let feed = RuntimeCompletionFeed {
4169 buffer,
4170 authority: std::sync::Weak::new(),
4171 };
4172
4173 let batch = feed.list_since(0);
4174 assert_eq!(
4175 batch
4176 .entries
4177 .iter()
4178 .map(|entry| entry.seq)
4179 .collect::<Vec<_>>(),
4180 vec![2]
4181 );
4182 assert_eq!(
4183 batch.gap_after(0),
4184 Some(meerkat_core::completion_feed::CompletionFeedGap {
4185 requested_after_seq: 0,
4186 dropped_through: 1,
4187 watermark: 2,
4188 })
4189 );
4190 assert_eq!(
4191 batch.gap_after(1),
4192 None,
4193 "a cursor at the exact eviction boundary can read the retained suffix"
4194 );
4195 }
4196
4197 #[test]
4198 fn runtime_completion_feed_rehydrates_evicted_obligation_from_generated_authority() {
4199 let registry = RuntimeOpsLifecycleRegistry::new();
4200 {
4201 let state = registry.read_state().unwrap();
4202 let mut buffer = state
4203 .feed_buffer
4204 .inner
4205 .write()
4206 .unwrap_or_else(std::sync::PoisonError::into_inner);
4207 buffer.max_retained = 1;
4208 }
4209
4210 for label in ["first", "second"] {
4211 let spec = background_spec(label);
4212 let operation_id = spec.id.clone();
4213 registry.register_operation(spec).unwrap();
4214 registry.provisioning_succeeded(&operation_id).unwrap();
4215 registry
4216 .complete_operation(
4217 &operation_id,
4218 OperationResult {
4219 id: operation_id.clone(),
4220 content: "done".into(),
4221 is_error: false,
4222 duration_ms: 1,
4223 tokens_used: 0,
4224 },
4225 )
4226 .unwrap();
4227 }
4228
4229 let batch = registry.completion_feed_handle().list_since(0);
4230 assert_eq!(
4231 batch
4232 .entries
4233 .iter()
4234 .map(|entry| entry.seq)
4235 .collect::<Vec<_>>(),
4236 vec![1, 2],
4237 "the generated authority must reconstruct the evicted completion"
4238 );
4239 assert_eq!(
4240 batch.gap_after(0),
4241 None,
4242 "successful authority reconstruction closes the bounded projection gap"
4243 );
4244 }
4245
4246 #[test]
4247 fn generated_and_projection_retention_expose_one_typed_gap() {
4248 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4249 max_completed: 1,
4250 max_concurrent: None,
4251 });
4252 for label in ["first", "second"] {
4253 let spec = background_spec(label);
4254 let operation_id = spec.id.clone();
4255 registry.register_operation(spec).unwrap();
4256 registry.provisioning_succeeded(&operation_id).unwrap();
4257 registry
4258 .complete_operation(
4259 &operation_id,
4260 OperationResult {
4261 id: operation_id.clone(),
4262 content: "done".into(),
4263 is_error: false,
4264 duration_ms: 1,
4265 tokens_used: 0,
4266 },
4267 )
4268 .unwrap();
4269 }
4270
4271 let batch = registry.completion_feed_handle().list_since(0);
4272 assert_eq!(
4273 batch
4274 .entries
4275 .iter()
4276 .map(|entry| entry.seq)
4277 .collect::<Vec<_>>(),
4278 vec![2]
4279 );
4280 assert_eq!(
4281 batch.gap_after(0),
4282 Some(meerkat_core::completion_feed::CompletionFeedGap {
4283 requested_after_seq: 0,
4284 dropped_through: 1,
4285 watermark: 2,
4286 }),
4287 "authority and projection eviction must meet at one typed retention boundary"
4288 );
4289 }
4290
4291 #[tokio::test]
4292 async fn dropping_wait_all_future_cancels_active_wait_request() {
4293 let registry = RuntimeOpsLifecycleRegistry::new();
4294 let run_id = test_run_id();
4295
4296 let spec = background_spec("cancelled-wait");
4297 let op_id = spec.id.clone();
4298 registry.register_operation(spec).unwrap();
4299 registry.provisioning_succeeded(&op_id).unwrap();
4300
4301 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
4302 drop(wait_fut);
4303
4304 let state = registry.read_state().unwrap();
4305 assert!(state.wait_request_id.is_none());
4306 assert!(state.wait_operation_ids().unwrap().is_empty());
4307 assert!(!state.wait_active());
4308 }
4309
4310 #[tokio::test]
4316 async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
4317 let registry = RuntimeOpsLifecycleRegistry::new();
4318 let run_id = test_run_id();
4319
4320 let spec = background_spec("corrupt-barrier");
4321 let op_id = spec.id.clone();
4322 registry.register_operation(spec).unwrap();
4323 registry.provisioning_succeeded(&op_id).unwrap();
4324
4325 let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
4327 tokio::pin!(wait_fut);
4328 assert!(
4329 tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
4330 .await
4331 .is_err(),
4332 "barrier waiter must still be pending before corruption"
4333 );
4334 assert!(registry.read_state().unwrap().wait_request_id.is_some());
4335
4336 {
4340 let mut state = registry.write_state().unwrap();
4341 let mut machine_state = state.dsl.0.state().clone();
4342 machine_state.wait_run_id = None;
4343 state.dsl = DslAuthority(Box::new(
4344 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4345 ));
4346 }
4347
4348 let err = registry
4351 .complete_operation(
4352 &op_id,
4353 OperationResult {
4354 id: op_id.clone(),
4355 content: "done".into(),
4356 is_error: false,
4357 duration_ms: 1,
4358 tokens_used: 0,
4359 },
4360 )
4361 .expect_err("corrupt wait authority must fail the terminal transition");
4362 assert!(
4363 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
4364 "unexpected terminal error: {err:?}"
4365 );
4366
4367 let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
4369 .await
4370 .expect("waiter must resolve, not hang, after authority corruption");
4371 match waiter_result {
4372 Err(OpsLifecycleError::Internal(message)) => assert!(
4373 message.contains("wait_all completion channel dropped"),
4374 "unexpected waiter error message: {message}"
4375 ),
4376 other => panic!("expected dropped-channel Internal error, got {other:?}"),
4377 }
4378 }
4379
4380 #[test]
4384 fn completion_cursor_propagates_poison_not_none() {
4385 let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
4386
4387 let poison_registry = std::sync::Arc::clone(®istry);
4389 let join = std::thread::spawn(move || {
4390 let _guard = poison_registry.write_state().unwrap();
4391 panic!("intentional panic to poison ops lifecycle registry lock");
4392 });
4393 assert!(
4394 join.join().is_err(),
4395 "poisoning thread must have panicked while holding the write guard"
4396 );
4397
4398 let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
4399 let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
4400 match result {
4401 Err(OpsLifecycleError::Internal(message)) => assert!(
4402 message.contains("ops lifecycle registry poisoned"),
4403 "unexpected cursor error message: {message}"
4404 ),
4405 other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
4406 }
4407 }
4408
4409 #[test]
4410 fn terminate_owner_only_targets_non_terminal_operations() {
4411 let registry = RuntimeOpsLifecycleRegistry::new();
4412
4413 let running_spec = background_spec("running");
4414 let running_id = running_spec.id.clone();
4415 registry.register_operation(running_spec).unwrap();
4416 registry.provisioning_succeeded(&running_id).unwrap();
4417
4418 let completed_spec = background_spec("completed");
4419 let completed_id = completed_spec.id.clone();
4420 registry.register_operation(completed_spec).unwrap();
4421 registry.provisioning_succeeded(&completed_id).unwrap();
4422 registry
4423 .complete_operation(
4424 &completed_id,
4425 OperationResult {
4426 id: completed_id.clone(),
4427 content: "done".into(),
4428 is_error: false,
4429 duration_ms: 1,
4430 tokens_used: 0,
4431 },
4432 )
4433 .unwrap();
4434
4435 registry.terminate_owner("shutdown".into()).unwrap();
4436
4437 assert!(matches!(
4438 registry.snapshot(&running_id).unwrap().unwrap().status,
4439 OperationStatus::Terminated
4440 ));
4441 assert!(matches!(
4442 registry.snapshot(&completed_id).unwrap().unwrap().status,
4443 OperationStatus::Completed
4444 ));
4445 }
4446
4447 #[test]
4448 fn unregister_retirement_fences_detached_late_callbacks() {
4449 let registry = RuntimeOpsLifecycleRegistry::new();
4450 let running_spec = background_spec("detached-running");
4451 let running_id = running_spec.id.clone();
4452 registry.register_operation(running_spec).unwrap();
4453 registry.provisioning_succeeded(&running_id).unwrap();
4454
4455 registry
4456 .retire_owner_for_unregister("owner unregistered".into())
4457 .unwrap();
4458 assert!(matches!(
4459 registry.snapshot(&running_id).unwrap().unwrap().status,
4460 OperationStatus::Terminated
4461 ));
4462
4463 let late_result = OperationResult {
4464 id: running_id.clone(),
4465 content: "late detached completion".into(),
4466 is_error: false,
4467 duration_ms: 1,
4468 tokens_used: 0,
4469 };
4470 assert_eq!(
4471 registry.complete_operation(&running_id, late_result),
4472 Err(OpsLifecycleError::OwnerRetired)
4473 );
4474 assert_eq!(
4475 registry.register_operation(background_spec("late-new-op")),
4476 Err(OpsLifecycleError::OwnerRetired)
4477 );
4478 assert_eq!(
4479 registry.report_progress(
4480 &running_id,
4481 OperationProgressUpdate {
4482 message: "late progress".into(),
4483 percent: None,
4484 },
4485 ),
4486 Err(OpsLifecycleError::OwnerRetired)
4487 );
4488 }
4489
4490 #[test]
4491 fn collect_completed_drains_terminal_operations() {
4492 let registry = RuntimeOpsLifecycleRegistry::new();
4493
4494 let spec_a = background_spec("a");
4495 let id_a = spec_a.id.clone();
4496 registry.register_operation(spec_a).unwrap();
4497 registry.provisioning_succeeded(&id_a).unwrap();
4498 registry
4499 .complete_operation(
4500 &id_a,
4501 OperationResult {
4502 id: id_a.clone(),
4503 content: "done".into(),
4504 is_error: false,
4505 duration_ms: 1,
4506 tokens_used: 0,
4507 },
4508 )
4509 .unwrap();
4510
4511 let spec_b = background_spec("b");
4512 let id_b = spec_b.id.clone();
4513 registry.register_operation(spec_b).unwrap();
4514
4515 let collected = registry.collect_completed().unwrap();
4516 assert_eq!(collected.len(), 1);
4517 assert_eq!(collected[0].0, id_a);
4518
4519 assert!(registry.snapshot(&id_a).unwrap().is_none());
4520 assert!(registry.snapshot(&id_b).unwrap().is_some());
4521
4522 let collected2 = registry.collect_completed().unwrap();
4523 assert!(collected2.is_empty());
4524 }
4525
4526 #[test]
4527 fn bounded_completed_retention_evicts_oldest() {
4528 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4529 max_completed: 3,
4530 max_concurrent: None,
4531 });
4532
4533 let mut ids = Vec::new();
4534 for i in 0..5 {
4535 let spec = background_spec(&format!("op-{i}"));
4536 let id = spec.id.clone();
4537 registry.register_operation(spec).unwrap();
4538 registry.provisioning_succeeded(&id).unwrap();
4539 registry
4540 .complete_operation(
4541 &id,
4542 OperationResult {
4543 id: id.clone(),
4544 content: format!("done-{i}"),
4545 is_error: false,
4546 duration_ms: 1,
4547 tokens_used: 0,
4548 },
4549 )
4550 .unwrap();
4551 ids.push(id);
4552 }
4553
4554 assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4555 assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4556 assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4557 assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4558 assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4559 }
4560
4561 #[test]
4562 fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4563 let registry = RuntimeOpsLifecycleRegistry::new();
4564
4565 let completed_spec = background_spec("completed");
4566 let completed_id = completed_spec.id.clone();
4567 registry.register_operation(completed_spec).unwrap();
4568 registry.provisioning_succeeded(&completed_id).unwrap();
4569 registry
4570 .complete_operation(
4571 &completed_id,
4572 OperationResult {
4573 id: completed_id.clone(),
4574 content: "done".into(),
4575 is_error: false,
4576 duration_ms: 1,
4577 tokens_used: 0,
4578 },
4579 )
4580 .unwrap();
4581
4582 let running_spec = background_spec("running");
4583 let running_id = running_spec.id.clone();
4584 registry.register_operation(running_spec).unwrap();
4585 registry.provisioning_succeeded(&running_id).unwrap();
4586
4587 let cursor_state = meerkat_core::EpochCursorState::new();
4588 let snapshot = registry
4589 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4590 .unwrap();
4591 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4592
4593 assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4594 assert!(recovered.snapshot(&running_id).unwrap().is_none());
4595
4596 let collected = recovered.collect_completed().unwrap();
4597 assert_eq!(collected.len(), 1);
4598 assert_eq!(collected[0].0, completed_id);
4599 }
4600
4601 #[test]
4602 fn recovered_snapshot_retains_running_detached_job_wait_binding() {
4603 let registry = RuntimeOpsLifecycleRegistry::new();
4604 let owner_session_id = SessionId::new();
4605 let job_id = "job_recovery_wait";
4606 let operation_id = OperationId::for_detached_job_wait(&owner_session_id, "realm-a", job_id);
4607 let source = OperationSource::detached_job("realm-a", job_id);
4608 registry
4609 .register_operation(OperationSpec {
4610 id: operation_id.clone(),
4611 kind: OperationKind::DetachedJobWait,
4612 owner_session_id: owner_session_id.clone(),
4613 display_name: "await detached job".into(),
4614 source_label: "await_job".into(),
4615 operation_source: Some(source.clone()),
4616 child_session_id: None,
4617 expect_peer_channel: false,
4618 })
4619 .unwrap();
4620 registry.provisioning_succeeded(&operation_id).unwrap();
4621
4622 let cursor_state = meerkat_core::EpochCursorState::new();
4623 let snapshot = registry
4624 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4625 .unwrap();
4626 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4627 let wait = recovered
4628 .snapshot(&operation_id)
4629 .unwrap()
4630 .expect("running await binding survives recovery");
4631
4632 assert_eq!(wait.status, OperationStatus::Running);
4633 assert_eq!(wait.owner_session_id, owner_session_id);
4634 assert_eq!(wait.operation_source, Some(source));
4635 assert!(wait.terminal_outcome.is_none());
4636 assert!(
4637 recovered
4638 .completion_feed()
4639 .expect("completion feed")
4640 .list_since(0)
4641 .entries
4642 .is_empty(),
4643 "recovery of a wait binding must not mint completion"
4644 );
4645 }
4646
4647 #[test]
4648 fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4649 let registry = RuntimeOpsLifecycleRegistry::new();
4650
4651 let mut spec = background_spec("capacity");
4652 spec.kind = OperationKind::BackgroundToolCapacitySlot;
4653 let operation_id = spec.id.clone();
4654 registry.register_operation(spec).unwrap();
4655 registry.provisioning_succeeded(&operation_id).unwrap();
4656 registry.mark_retired(&operation_id).unwrap();
4657
4658 assert!(registry.snapshot(&operation_id).unwrap().is_none());
4659
4660 let cursor_state = meerkat_core::EpochCursorState::new();
4661 let snapshot = registry
4662 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4663 .unwrap();
4664 assert!(
4665 !snapshot
4666 .authority_state
4667 .operations
4668 .contains_key(&operation_id)
4669 );
4670 assert!(!snapshot.operation_specs.contains_key(&operation_id));
4671 assert!(snapshot.completion_entries.is_empty());
4672
4673 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4674 assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4675 }
4676
4677 #[test]
4678 fn rollback_unreturned_background_operation_restores_absence_without_completion() {
4679 let registry = RuntimeOpsLifecycleRegistry::new();
4680 let spec = background_spec("unreturned");
4681 let operation_id = spec.id.clone();
4682
4683 registry
4684 .register_operation(spec)
4685 .expect("register unreturned operation");
4686 registry
4687 .provisioning_succeeded(&operation_id)
4688 .expect("start unreturned operation");
4689 registry
4690 .rollback_unreturned_operation(&operation_id)
4691 .expect("generated rollback should restore absence");
4692
4693 assert!(
4694 registry
4695 .snapshot(&operation_id)
4696 .expect("snapshot after rollback")
4697 .is_none(),
4698 "never-returned operation must not retain public lifecycle state"
4699 );
4700 let feed = registry
4701 .completion_feed()
4702 .expect("runtime registry exposes completion feed");
4703 assert!(
4704 feed.list_since(0).entries.is_empty(),
4705 "never-returned operation must not mint a phantom completion"
4706 );
4707 }
4708
4709 #[test]
4710 fn recovered_snapshot_uses_authority_operation_source() {
4711 let registry = RuntimeOpsLifecycleRegistry::new();
4712 let child_session_id = SessionId::new();
4713 let operation_source = OperationSource::session_child(child_session_id.clone());
4714 let spec = OperationSpec {
4715 id: OperationId::new(),
4716 kind: OperationKind::MobMemberChild,
4717 owner_session_id: SessionId::new(),
4718 display_name: "source-recovery".into(),
4719 source_label: "test".into(),
4720 operation_source: Some(operation_source.clone()),
4721 child_session_id: Some(child_session_id),
4722 expect_peer_channel: true,
4723 };
4724 let operation_id = spec.id.clone();
4725
4726 registry.register_operation(spec).unwrap();
4727 registry.provisioning_succeeded(&operation_id).unwrap();
4728 registry.mark_retired(&operation_id).unwrap();
4729
4730 let cursor_state = meerkat_core::EpochCursorState::new();
4731 let mut snapshot = registry
4732 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4733 .unwrap();
4734 assert_eq!(
4735 snapshot
4736 .authority_state
4737 .operations
4738 .get(&operation_id)
4739 .and_then(|state| state.operation_source.as_ref()),
4740 Some(&operation_source)
4741 );
4742
4743 snapshot
4744 .operation_specs
4745 .get_mut(&operation_id)
4746 .expect("persisted spec")
4747 .operation_source = None;
4748 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4749 assert_eq!(
4750 recovered
4751 .snapshot(&operation_id)
4752 .unwrap()
4753 .unwrap()
4754 .operation_source,
4755 Some(operation_source)
4756 );
4757 }
4758
4759 #[test]
4760 fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4761 let registry = RuntimeOpsLifecycleRegistry::new();
4762 let child_session_id = SessionId::new();
4763 let operation_source = OperationSource::session_child(child_session_id.clone());
4764 let spec = OperationSpec {
4765 id: OperationId::new(),
4766 kind: OperationKind::MobMemberChild,
4767 owner_session_id: SessionId::new(),
4768 display_name: "source-drift".into(),
4769 source_label: "test".into(),
4770 operation_source: Some(operation_source),
4771 child_session_id: Some(child_session_id),
4772 expect_peer_channel: true,
4773 };
4774 let operation_id = spec.id.clone();
4775
4776 registry.register_operation(spec).unwrap();
4777 registry.provisioning_succeeded(&operation_id).unwrap();
4778 registry.mark_retired(&operation_id).unwrap();
4779
4780 let cursor_state = meerkat_core::EpochCursorState::new();
4781 let mut snapshot = registry
4782 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4783 .unwrap();
4784 snapshot
4785 .operation_specs
4786 .get_mut(&operation_id)
4787 .expect("persisted spec")
4788 .operation_source = Some(OperationSource::session_child(SessionId::new()));
4789
4790 let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4791 .expect_err("source mirror drift must fail recovery");
4792 assert!(
4793 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4794 "unexpected recovery error: {err:?}"
4795 );
4796 }
4797
4798 #[test]
4799 fn persisted_authority_state_serializes_explicit_no_operation_source() {
4800 let registry = RuntimeOpsLifecycleRegistry::new();
4801
4802 let spec = background_spec("explicit-no-source");
4803 let operation_id = spec.id.clone();
4804 registry.register_operation(spec).unwrap();
4805 registry.provisioning_succeeded(&operation_id).unwrap();
4806
4807 let cursor_state = meerkat_core::EpochCursorState::new();
4808 let snapshot = registry
4809 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4810 .unwrap();
4811 let value = serde_json::to_value(&snapshot).unwrap();
4812 let operations = value
4813 .get("authority_state")
4814 .and_then(|state| state.get("operations"))
4815 .and_then(serde_json::Value::as_object)
4816 .expect("serialized authority operations");
4817 let persisted_state = operations
4818 .values()
4819 .next()
4820 .and_then(serde_json::Value::as_object)
4821 .expect("serialized operation state");
4822
4823 assert!(
4824 persisted_state
4825 .get("operation_source")
4826 .is_some_and(serde_json::Value::is_null),
4827 "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4828 );
4829
4830 let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4831 assert_eq!(
4832 recovered_snapshot
4833 .authority_state
4834 .operations
4835 .get(&operation_id)
4836 .expect("round-tripped operation")
4837 .operation_source,
4838 None
4839 );
4840 }
4841
4842 #[test]
4843 fn persisted_authority_state_rejects_missing_operation_source_fact() {
4844 let registry = RuntimeOpsLifecycleRegistry::new();
4845
4846 let spec = background_spec("missing-source-fact");
4847 registry.register_operation(spec).unwrap();
4848
4849 let cursor_state = meerkat_core::EpochCursorState::new();
4850 let snapshot = registry
4851 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4852 .unwrap();
4853 let mut value = serde_json::to_value(&snapshot).unwrap();
4854 let operations = value
4855 .get_mut("authority_state")
4856 .and_then(|state| state.get_mut("operations"))
4857 .and_then(serde_json::Value::as_object_mut)
4858 .expect("serialized authority operations");
4859 let operation_state = operations
4860 .values_mut()
4861 .next()
4862 .and_then(serde_json::Value::as_object_mut)
4863 .expect("serialized operation state");
4864 assert!(operation_state.remove("operation_source").is_some());
4865
4866 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4867 .expect_err("missing generated source fact must fail recovery snapshot decoding");
4868 assert!(
4869 err.to_string().contains("operation_source"),
4870 "unexpected decode error: {err}"
4871 );
4872 }
4873
4874 #[test]
4875 fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4876 let registry = RuntimeOpsLifecycleRegistry::new();
4877
4878 let spec = background_spec("missing-feed-authority");
4879 let operation_id = spec.id.clone();
4880 registry.register_operation(spec).unwrap();
4881 registry.provisioning_succeeded(&operation_id).unwrap();
4882 registry
4883 .complete_operation(
4884 &operation_id,
4885 OperationResult {
4886 id: operation_id.clone(),
4887 content: "done".into(),
4888 is_error: false,
4889 duration_ms: 1,
4890 tokens_used: 0,
4891 },
4892 )
4893 .unwrap();
4894
4895 let cursor_state = meerkat_core::EpochCursorState::new();
4896 let snapshot = registry
4897 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4898 .unwrap();
4899 let mut value = serde_json::to_value(&snapshot).unwrap();
4900 let authority_state = value
4901 .get_mut("authority_state")
4902 .and_then(serde_json::Value::as_object_mut)
4903 .expect("serialized authority state");
4904 assert!(authority_state.remove("completion_feed_entries").is_some());
4905
4906 let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4907 .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4908 assert!(
4909 err.to_string().contains("completion_feed_entries"),
4910 "unexpected decode error: {err}"
4911 );
4912 }
4913
4914 #[test]
4915 fn public_child_session_projection_uses_authority_operation_source() {
4916 let registry = RuntimeOpsLifecycleRegistry::new();
4917 let authority_child_session_id = SessionId::new();
4918 let stale_shell_child_session_id = SessionId::new();
4919 let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4920 let spec = OperationSpec {
4921 id: OperationId::new(),
4922 kind: OperationKind::MobMemberChild,
4923 owner_session_id: SessionId::new(),
4924 display_name: "child-projection".into(),
4925 source_label: "test".into(),
4926 operation_source: Some(operation_source),
4927 child_session_id: Some(stale_shell_child_session_id),
4928 expect_peer_channel: true,
4929 };
4930 let operation_id = spec.id.clone();
4931
4932 registry.register_operation(spec).unwrap();
4933
4934 assert_eq!(
4935 registry
4936 .snapshot(&operation_id)
4937 .unwrap()
4938 .unwrap()
4939 .child_session_id,
4940 Some(authority_child_session_id.clone())
4941 );
4942
4943 let cursor_state = meerkat_core::EpochCursorState::new();
4944 let snapshot = registry
4945 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4946 .unwrap();
4947 assert_eq!(
4948 snapshot
4949 .operation_specs
4950 .get(&operation_id)
4951 .expect("persisted spec")
4952 .child_session_id,
4953 Some(authority_child_session_id)
4954 );
4955 }
4956
4957 #[test]
4958 fn generated_terminal_payload_projection_fails_closed() {
4959 let registry = RuntimeOpsLifecycleRegistry::new();
4960
4961 let spec = background_spec("terminal-payload-drift");
4962 let operation_id = spec.id.clone();
4963 registry.register_operation(spec).unwrap();
4964 registry.provisioning_succeeded(&operation_id).unwrap();
4965 registry
4966 .complete_operation(
4967 &operation_id,
4968 OperationResult {
4969 id: operation_id.clone(),
4970 content: "done".into(),
4971 is_error: false,
4972 duration_ms: 1,
4973 tokens_used: 0,
4974 },
4975 )
4976 .unwrap();
4977
4978 {
4979 let mut state = registry.write_state().unwrap();
4980 let mut machine_state = state.dsl.0.state().clone();
4981 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4982 machine_state
4983 .op_terminal_payload
4984 .insert(operation_id_key, OperationTerminalOutcome::Retired);
4985 state.dsl = DslAuthority(Box::new(
4986 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4987 ));
4988 }
4989
4990 let err = match registry.register_watcher(&operation_id) {
4991 Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4992 Err(err) => err,
4993 };
4994 assert!(
4995 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4996 "unexpected watcher error: {err:?}"
4997 );
4998 let err = registry
4999 .snapshot(&operation_id)
5000 .expect_err("invalid generated terminal payload must reject public snapshot");
5001 assert!(
5002 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
5003 "unexpected public snapshot error: {err:?}"
5004 );
5005
5006 let cursor_state = meerkat_core::EpochCursorState::new();
5007 let err = match registry
5008 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5009 {
5010 Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
5011 Err(err) => err,
5012 };
5013 assert!(
5014 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
5015 "unexpected snapshot error: {err:?}"
5016 );
5017
5018 let err = match registry.collect_completed() {
5019 Ok(_) => panic!("invalid generated terminal payload must reject collection"),
5020 Err(err) => err,
5021 };
5022 assert!(
5023 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
5024 "unexpected collection error: {err:?}"
5025 );
5026 }
5027
5028 #[test]
5029 fn generated_terminal_payload_missing_projection_fails_closed() {
5030 let registry = RuntimeOpsLifecycleRegistry::new();
5031
5032 let spec = background_spec("terminal-payload-missing");
5033 let operation_id = spec.id.clone();
5034 registry.register_operation(spec).unwrap();
5035 registry.provisioning_succeeded(&operation_id).unwrap();
5036 registry
5037 .fail_operation(&operation_id, "boom".into())
5038 .unwrap();
5039
5040 {
5041 let mut state = registry.write_state().unwrap();
5042 let mut machine_state = state.dsl.0.state().clone();
5043 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5044 machine_state.op_terminal_payload.remove(&operation_id_key);
5045 state.dsl = DslAuthority(Box::new(
5046 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5047 ));
5048 }
5049
5050 let err = match registry.register_watcher(&operation_id) {
5051 Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
5052 Err(err) => err,
5053 };
5054 assert!(
5055 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
5056 "unexpected watcher error: {err:?}"
5057 );
5058 let err = registry
5059 .snapshot(&operation_id)
5060 .expect_err("missing generated terminal payload must reject public snapshot");
5061 assert!(
5062 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
5063 "unexpected public snapshot error: {err:?}"
5064 );
5065 }
5066
5067 #[test]
5068 fn generated_terminal_status_without_outcome_fails_closed() {
5069 let registry = RuntimeOpsLifecycleRegistry::new();
5070
5071 let spec = background_spec("terminal-outcome-missing");
5072 let operation_id = spec.id.clone();
5073 registry.register_operation(spec).unwrap();
5074 registry.provisioning_succeeded(&operation_id).unwrap();
5075 registry
5076 .fail_operation(&operation_id, "boom".into())
5077 .unwrap();
5078
5079 {
5080 let mut state = registry.write_state().unwrap();
5081 let mut machine_state = state.dsl.0.state().clone();
5082 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5083 machine_state.op_terminal_outcomes.remove(&operation_id_key);
5084 state.dsl = DslAuthority(Box::new(
5085 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5086 ));
5087 }
5088
5089 let err = registry
5090 .snapshot(&operation_id)
5091 .expect_err("terminal status without outcome must reject public snapshot");
5092 assert!(
5093 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
5094 "unexpected public snapshot error: {err:?}"
5095 );
5096
5097 let err = match registry.collect_completed() {
5098 Ok(_) => panic!("terminal status without outcome must reject collection"),
5099 Err(err) => err,
5100 };
5101 assert!(
5102 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
5103 "unexpected collection error: {err:?}"
5104 );
5105 }
5106
5107 #[test]
5108 fn generated_operation_source_projection_fails_closed() {
5109 let registry = RuntimeOpsLifecycleRegistry::new();
5110 let child_session_id = SessionId::new();
5111 let operation_source = OperationSource::session_child(child_session_id.clone());
5112 let spec = OperationSpec {
5113 id: OperationId::new(),
5114 kind: OperationKind::MobMemberChild,
5115 owner_session_id: SessionId::new(),
5116 display_name: "source-authority-drift".into(),
5117 source_label: "test".into(),
5118 operation_source: Some(operation_source),
5119 child_session_id: Some(child_session_id),
5120 expect_peer_channel: true,
5121 };
5122 let operation_id = spec.id.clone();
5123
5124 registry.register_operation(spec).unwrap();
5125
5126 {
5127 let mut state = registry.write_state().unwrap();
5128 let mut machine_state = state.dsl.0.state().clone();
5129 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5130 machine_state
5131 .op_sources
5132 .get_mut(&operation_id_key)
5133 .expect("generated operation source")
5134 .session_id = None;
5135 state.dsl = DslAuthority(Box::new(
5136 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5137 ));
5138 }
5139
5140 let err = registry
5141 .snapshot(&operation_id)
5142 .expect_err("invalid generated operation source must reject public snapshot");
5143 assert!(
5144 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
5145 "unexpected public snapshot error: {err:?}"
5146 );
5147 let err = registry
5148 .list_operations()
5149 .expect_err("invalid generated operation source must reject public operation list");
5150 assert!(
5151 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
5152 "unexpected operation list error: {err:?}"
5153 );
5154
5155 let cursor_state = meerkat_core::EpochCursorState::new();
5156 let err = match registry
5157 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5158 {
5159 Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
5160 Err(err) => err,
5161 };
5162 assert!(
5163 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
5164 "unexpected snapshot error: {err:?}"
5165 );
5166 }
5167
5168 #[test]
5169 fn generated_operation_id_projection_fails_closed() {
5170 let registry = RuntimeOpsLifecycleRegistry::new();
5171
5172 {
5173 let mut state = registry.write_state().unwrap();
5174 let mut machine_state = state.dsl.0.state().clone();
5175 machine_state.op_statuses.insert(
5176 "not-json-operation-id".into(),
5177 mm_dsl::OperationStatus::Running,
5178 );
5179 state.dsl = DslAuthority(Box::new(
5180 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5181 ));
5182 }
5183
5184 let err = registry
5185 .list_operations()
5186 .expect_err("invalid generated operation id must reject public operation list");
5187 assert!(
5188 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
5189 "unexpected operation list error: {err:?}"
5190 );
5191
5192 let cursor_state = meerkat_core::EpochCursorState::new();
5193 let err = match registry
5194 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5195 {
5196 Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
5197 Err(err) => err,
5198 };
5199 assert!(
5200 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
5201 "unexpected persistence snapshot error: {err:?}"
5202 );
5203 }
5204
5205 #[test]
5206 fn generated_missing_kind_projection_fails_closed() {
5207 let registry = RuntimeOpsLifecycleRegistry::new();
5208 let spec = background_spec("missing-kind");
5209 let operation_id = spec.id.clone();
5210 registry.register_operation(spec).unwrap();
5211
5212 {
5213 let mut state = registry.write_state().unwrap();
5214 let mut machine_state = state.dsl.0.state().clone();
5215 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5216 machine_state.op_kinds.remove(&operation_id_key);
5217 state.dsl = DslAuthority(Box::new(
5218 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5219 ));
5220 }
5221
5222 let err = registry
5223 .snapshot(&operation_id)
5224 .expect_err("missing generated kind must reject public snapshot");
5225 assert!(
5226 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
5227 "unexpected public snapshot error: {err:?}"
5228 );
5229 let err = registry
5230 .list_operations()
5231 .expect_err("missing generated kind must reject public list");
5232 assert!(
5233 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
5234 "unexpected public list error: {err:?}"
5235 );
5236
5237 let cursor_state = meerkat_core::EpochCursorState::new();
5238 let err = registry
5239 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5240 .expect_err("missing generated kind must reject persistence snapshot");
5241 assert!(
5242 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
5243 "unexpected persistence snapshot error: {err:?}"
5244 );
5245 }
5246
5247 #[test]
5248 fn generated_missing_status_projection_fails_closed() {
5249 let registry = RuntimeOpsLifecycleRegistry::new();
5250 let spec = background_spec("missing-status");
5251 let operation_id = spec.id.clone();
5252 registry.register_operation(spec).unwrap();
5253
5254 {
5255 let mut state = registry.write_state().unwrap();
5256 let mut machine_state = state.dsl.0.state().clone();
5257 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5258 machine_state.op_statuses.remove(&operation_id_key);
5259 state.dsl = DslAuthority(Box::new(
5260 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5261 ));
5262 }
5263
5264 let err = registry
5265 .snapshot(&operation_id)
5266 .expect_err("missing generated status must reject public snapshot");
5267 assert!(
5268 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
5269 "unexpected public snapshot error: {err:?}"
5270 );
5271 let err = registry
5272 .list_operations()
5273 .expect_err("missing generated status must reject public list");
5274 assert!(
5275 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
5276 "unexpected public list error: {err:?}"
5277 );
5278 let err = registry
5279 .classify_operation_public_result(&operation_id)
5280 .expect_err("missing generated status must reject public-result classification");
5281 assert!(
5282 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
5283 "unexpected public-result error: {err:?}"
5284 );
5285 }
5286
5287 #[test]
5288 fn generated_retiring_public_result_remains_running_until_terminal() {
5289 let registry = RuntimeOpsLifecycleRegistry::new();
5290 let spec = background_spec("retiring-public-result");
5291 let operation_id = spec.id.clone();
5292 registry.register_operation(spec).unwrap();
5293 registry.provisioning_succeeded(&operation_id).unwrap();
5294 registry.request_retire(&operation_id).unwrap();
5295
5296 let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
5297 assert_eq!(snapshot.status, OperationStatus::Retiring);
5298 assert!(snapshot.terminal_outcome.is_none());
5299 assert!(!snapshot.terminal);
5300 assert_eq!(
5301 snapshot.public_result_class,
5302 OperationPublicResultClass::Running
5303 );
5304 assert_eq!(
5305 registry
5306 .classify_operation_public_result(&operation_id)
5307 .unwrap(),
5308 OperationPublicResultClass::Running
5309 );
5310 }
5311
5312 #[test]
5313 fn generated_missing_peer_ready_projection_fails_closed() {
5314 let registry = RuntimeOpsLifecycleRegistry::new();
5315 let spec = background_spec("missing-peer-ready");
5316 let operation_id = spec.id.clone();
5317 registry.register_operation(spec).unwrap();
5318
5319 {
5320 let mut state = registry.write_state().unwrap();
5321 let mut machine_state = state.dsl.0.state().clone();
5322 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5323 machine_state.op_peer_ready.remove(&operation_id_key);
5324 state.dsl = DslAuthority(Box::new(
5325 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5326 ));
5327 }
5328
5329 let err = registry
5330 .snapshot(&operation_id)
5331 .expect_err("missing generated peer-ready fact must reject public snapshot");
5332 assert!(
5333 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
5334 "unexpected public snapshot error: {err:?}"
5335 );
5336
5337 let cursor_state = meerkat_core::EpochCursorState::new();
5338 let err = registry
5339 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5340 .expect_err("missing generated peer-ready fact must reject persistence snapshot");
5341 assert!(
5342 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
5343 "unexpected persistence snapshot error: {err:?}"
5344 );
5345 }
5346
5347 #[test]
5348 fn generated_missing_progress_count_projection_fails_closed() {
5349 let registry = RuntimeOpsLifecycleRegistry::new();
5350 let spec = background_spec("missing-progress-count");
5351 let operation_id = spec.id.clone();
5352 registry.register_operation(spec).unwrap();
5353
5354 {
5355 let mut state = registry.write_state().unwrap();
5356 let mut machine_state = state.dsl.0.state().clone();
5357 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5358 machine_state.op_progress_counts.remove(&operation_id_key);
5359 state.dsl = DslAuthority(Box::new(
5360 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5361 ));
5362 }
5363
5364 let err = registry
5365 .snapshot(&operation_id)
5366 .expect_err("missing generated progress count must reject public snapshot");
5367 assert!(
5368 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
5369 "unexpected public snapshot error: {err:?}"
5370 );
5371
5372 let cursor_state = meerkat_core::EpochCursorState::new();
5373 let err = registry
5374 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5375 .expect_err("missing generated progress count must reject persistence snapshot");
5376 assert!(
5377 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
5378 "unexpected persistence snapshot error: {err:?}"
5379 );
5380 }
5381
5382 #[test]
5383 fn generated_terminal_sequence_missing_persistence_fails_closed() {
5384 let registry = RuntimeOpsLifecycleRegistry::new();
5385 let spec = background_spec("terminal-sequence-missing");
5386 let operation_id = spec.id.clone();
5387 registry.register_operation(spec).unwrap();
5388 registry.provisioning_succeeded(&operation_id).unwrap();
5389 registry
5390 .complete_operation(
5391 &operation_id,
5392 OperationResult {
5393 id: operation_id.clone(),
5394 content: "done".into(),
5395 is_error: false,
5396 duration_ms: 1,
5397 tokens_used: 0,
5398 },
5399 )
5400 .unwrap();
5401
5402 {
5403 let mut state = registry.write_state().unwrap();
5404 let mut machine_state = state.dsl.0.state().clone();
5405 let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5406 machine_state.op_completion_seq.remove(&operation_id_key);
5407 state.dsl = DslAuthority(Box::new(
5408 mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5409 ));
5410 }
5411
5412 let cursor_state = meerkat_core::EpochCursorState::new();
5413 let err = registry
5414 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5415 .expect_err("missing generated terminal sequence must reject persistence snapshot");
5416 assert!(
5417 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
5418 "unexpected persistence snapshot error: {err:?}"
5419 );
5420 }
5421
5422 #[test]
5423 fn generated_record_without_shell_projection_fails_closed() {
5424 let registry = RuntimeOpsLifecycleRegistry::new();
5425 let spec = background_spec("missing-shell-record");
5426 let operation_id = spec.id.clone();
5427 registry.register_operation(spec).unwrap();
5428
5429 {
5430 let mut state = registry.write_state().unwrap();
5431 state.records.remove(&operation_id);
5432 }
5433
5434 let err = registry
5435 .snapshot(&operation_id)
5436 .expect_err("generated operation without shell record must reject public snapshot");
5437 assert!(
5438 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5439 "unexpected public snapshot error: {err:?}"
5440 );
5441 let err = registry
5442 .list_operations()
5443 .expect_err("generated operation without shell record must reject public list");
5444 assert!(
5445 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5446 "unexpected public list error: {err:?}"
5447 );
5448 }
5449
5450 #[test]
5451 fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
5452 let registry = RuntimeOpsLifecycleRegistry::new();
5453 let child_session_id = SessionId::new();
5454 let operation_source = OperationSource::session_child(child_session_id.clone());
5455 let spec = OperationSpec {
5456 id: OperationId::new(),
5457 kind: OperationKind::MobMemberChild,
5458 owner_session_id: SessionId::new(),
5459 display_name: "child-drift".into(),
5460 source_label: "test".into(),
5461 operation_source: Some(operation_source),
5462 child_session_id: Some(child_session_id.clone()),
5463 expect_peer_channel: true,
5464 };
5465 let operation_id = spec.id.clone();
5466
5467 registry.register_operation(spec).unwrap();
5468 registry.provisioning_succeeded(&operation_id).unwrap();
5469 registry.mark_retired(&operation_id).unwrap();
5470
5471 let cursor_state = meerkat_core::EpochCursorState::new();
5472 let mut snapshot = registry
5473 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5474 .unwrap();
5475 snapshot
5476 .operation_specs
5477 .get_mut(&operation_id)
5478 .expect("persisted spec")
5479 .child_session_id = Some(SessionId::new());
5480
5481 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5482 assert_eq!(
5483 recovered
5484 .snapshot(&operation_id)
5485 .unwrap()
5486 .unwrap()
5487 .child_session_id,
5488 Some(child_session_id)
5489 );
5490 }
5491
5492 #[test]
5493 fn completion_wake_class_is_generated_by_operation_kind() {
5494 let registry = RuntimeOpsLifecycleRegistry::new();
5495 let operation_id = OperationId::new();
5496
5497 assert_eq!(
5498 registry
5499 .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
5500 .unwrap(),
5501 OperationCompletionWakeClass::Wake
5502 );
5503 assert_eq!(
5504 registry
5505 .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
5506 .unwrap(),
5507 OperationCompletionWakeClass::Ignore
5508 );
5509 assert_eq!(
5510 registry
5511 .classify_operation_completion_wake(
5512 &operation_id,
5513 OperationKind::BackgroundToolCapacitySlot,
5514 )
5515 .unwrap(),
5516 OperationCompletionWakeClass::Ignore
5517 );
5518 }
5519
5520 #[test]
5521 fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
5522 let registry = RuntimeOpsLifecycleRegistry::new();
5523
5524 let running_spec = background_spec("running");
5525 let running_id = running_spec.id.clone();
5526 registry.register_operation(running_spec.clone()).unwrap();
5527 registry.provisioning_succeeded(&running_id).unwrap();
5528
5529 let cursor_state = meerkat_core::EpochCursorState::new();
5530 let mut snapshot = registry
5531 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5532 .unwrap();
5533 snapshot.completion_entries.push(CompletionEntry {
5534 seq: 1,
5535 operation_id: running_id.clone(),
5536 kind: running_spec.kind,
5537 display_name: running_spec.display_name,
5538 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5539 id: running_id,
5540 content: "phantom".into(),
5541 is_error: false,
5542 duration_ms: 1,
5543 tokens_used: 0,
5544 }),
5545 completed_at_ms: None,
5546 });
5547
5548 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5549 Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5550 Err(err) => err,
5551 };
5552 assert!(
5553 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5554 "unexpected recovery error: {err:?}"
5555 );
5556 }
5557
5558 #[test]
5559 fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5560 let registry = RuntimeOpsLifecycleRegistry::new();
5561 let phantom_id = OperationId::new();
5562 {
5563 let state = registry.read_state().unwrap();
5564 state.feed_buffer.push(CompletionEntry {
5565 seq: 1,
5566 operation_id: phantom_id.clone(),
5567 kind: OperationKind::BackgroundToolOp,
5568 display_name: "phantom".into(),
5569 terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5570 id: phantom_id,
5571 content: "phantom".into(),
5572 is_error: false,
5573 duration_ms: 1,
5574 tokens_used: 0,
5575 }),
5576 completed_at_ms: None,
5577 });
5578 }
5579
5580 let cursor_state = meerkat_core::EpochCursorState::new();
5581 let err = match registry
5582 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5583 {
5584 Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5585 Err(err) => err,
5586 };
5587 assert!(
5588 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5589 "unexpected capture error: {err:?}"
5590 );
5591 }
5592
5593 #[test]
5594 fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5595 let registry = RuntimeOpsLifecycleRegistry::new();
5596
5597 let spec = background_spec("terminal");
5598 let operation_id = spec.id.clone();
5599 registry.register_operation(spec).unwrap();
5600 registry.provisioning_succeeded(&operation_id).unwrap();
5601 registry
5602 .complete_operation(
5603 &operation_id,
5604 OperationResult {
5605 id: operation_id.clone(),
5606 content: "done".into(),
5607 is_error: false,
5608 duration_ms: 1,
5609 tokens_used: 0,
5610 },
5611 )
5612 .unwrap();
5613
5614 let cursor_state = meerkat_core::EpochCursorState::new();
5615 let mut snapshot = registry
5616 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5617 .unwrap();
5618 let phantom_id = OperationId::new();
5619 let phantom_result = OperationResult {
5620 id: phantom_id.clone(),
5621 content: "phantom".into(),
5622 is_error: false,
5623 duration_ms: 1,
5624 tokens_used: 0,
5625 };
5626 let phantom_entry = CompletionFeedCanonicalState {
5627 seq: snapshot.authority_state.next_completion_seq + 1,
5628 kind: OperationKind::BackgroundToolOp,
5629 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5630 };
5631 snapshot
5632 .authority_state
5633 .completion_feed_entries
5634 .insert(phantom_id.clone(), phantom_entry);
5635 snapshot.completion_entries.push(CompletionEntry {
5636 seq: snapshot.authority_state.next_completion_seq + 1,
5637 operation_id: phantom_id.clone(),
5638 kind: OperationKind::BackgroundToolOp,
5639 display_name: "phantom".into(),
5640 terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5641 completed_at_ms: None,
5642 });
5643
5644 let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5645 Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5646 Err(err) => err,
5647 };
5648 assert!(
5649 matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5650 "unexpected recovery error: {err:?}"
5651 );
5652 }
5653
5654 #[test]
5655 fn recovered_completed_order_uses_generated_completion_sequences() {
5656 let registry = RuntimeOpsLifecycleRegistry::new();
5657
5658 let spec_a = background_spec("a");
5659 let id_a = spec_a.id.clone();
5660 registry.register_operation(spec_a).unwrap();
5661 registry.provisioning_succeeded(&id_a).unwrap();
5662 registry
5663 .complete_operation(
5664 &id_a,
5665 OperationResult {
5666 id: id_a.clone(),
5667 content: "a".into(),
5668 is_error: false,
5669 duration_ms: 1,
5670 tokens_used: 0,
5671 },
5672 )
5673 .unwrap();
5674
5675 let spec_b = background_spec("b");
5676 let id_b = spec_b.id.clone();
5677 registry.register_operation(spec_b).unwrap();
5678 registry.provisioning_succeeded(&id_b).unwrap();
5679 registry
5680 .complete_operation(
5681 &id_b,
5682 OperationResult {
5683 id: id_b.clone(),
5684 content: "b".into(),
5685 is_error: false,
5686 duration_ms: 1,
5687 tokens_used: 0,
5688 },
5689 )
5690 .unwrap();
5691
5692 let cursor_state = meerkat_core::EpochCursorState::new();
5693 let mut snapshot = registry
5694 .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5695 .unwrap();
5696 snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5697
5698 let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5699 let collected = recovered.collect_completed().unwrap();
5700
5701 assert_eq!(collected[0].0, id_a);
5702 assert_eq!(collected[1].0, id_b);
5703 }
5704
5705 #[test]
5706 fn max_concurrent_enforcement() {
5707 let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5708 max_completed: DEFAULT_MAX_COMPLETED,
5709 max_concurrent: Some(2),
5710 });
5711
5712 let spec_a = background_spec("a");
5713 let id_a = spec_a.id.clone();
5714 registry.register_operation(spec_a).unwrap();
5715
5716 let spec_b = background_spec("b");
5717 registry.register_operation(spec_b).unwrap();
5718
5719 let spec_c = background_spec("c");
5720 let result = registry.register_operation(spec_c);
5721 assert!(matches!(
5722 result,
5723 Err(OpsLifecycleError::MaxConcurrentExceeded {
5724 limit: 2,
5725 active: 2,
5726 })
5727 ));
5728
5729 registry.provisioning_succeeded(&id_a).unwrap();
5730 registry
5731 .complete_operation(
5732 &id_a,
5733 OperationResult {
5734 id: id_a.clone(),
5735 content: "done".into(),
5736 is_error: false,
5737 duration_ms: 1,
5738 tokens_used: 0,
5739 },
5740 )
5741 .unwrap();
5742
5743 let spec_d = background_spec("d");
5744 assert!(registry.register_operation(spec_d).is_ok());
5745 }
5746
5747 #[test]
5748 fn snapshot_includes_timestamps() {
5749 let registry = RuntimeOpsLifecycleRegistry::new();
5750 let spec = background_spec("timed");
5751 let op_id = spec.id.clone();
5752 registry.register_operation(spec).unwrap();
5753
5754 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5755 assert!(snap1.created_at_ms > 0);
5756 assert!(snap1.started_at_ms.is_none());
5757 assert!(snap1.completed_at_ms.is_none());
5758 assert!(snap1.elapsed_ms.is_none());
5759
5760 registry.provisioning_succeeded(&op_id).unwrap();
5761 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5762 assert!(snap2.started_at_ms.is_some());
5763 assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5764
5765 registry
5766 .complete_operation(
5767 &op_id,
5768 OperationResult {
5769 id: op_id.clone(),
5770 content: "done".into(),
5771 is_error: false,
5772 duration_ms: 1,
5773 tokens_used: 0,
5774 },
5775 )
5776 .unwrap();
5777 let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5778 assert!(snap3.completed_at_ms.is_some());
5779 assert!(snap3.elapsed_ms.is_some());
5780 assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5781 }
5782
5783 #[test]
5784 fn snapshot_includes_peer_handle() {
5785 let registry = RuntimeOpsLifecycleRegistry::new();
5786 let child_session_id = SessionId::new();
5787 let spec = OperationSpec {
5788 id: OperationId::new(),
5789 kind: OperationKind::MobMemberChild,
5790 owner_session_id: SessionId::new(),
5791 display_name: "peer-test".into(),
5792 source_label: "test".into(),
5793 operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5794 child_session_id: Some(child_session_id),
5795 expect_peer_channel: true,
5796 };
5797 let op_id = spec.id.clone();
5798 registry.register_operation(spec).unwrap();
5799 registry.provisioning_succeeded(&op_id).unwrap();
5800
5801 let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5802 assert!(snap1.peer_handle.is_none());
5803
5804 let handle = OperationPeerHandle {
5805 peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5806 trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5807 "member-x",
5808 PeerId::new(),
5809 "inproc://x",
5810 )
5811 .unwrap(),
5812 };
5813 registry.peer_ready(&op_id, handle).unwrap();
5814
5815 let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5816 assert_eq!(
5817 snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5818 "member-x"
5819 );
5820 }
5821}