Skip to main content

meerkat_runtime/
ops_lifecycle.rs

1//! In-memory runtime implementation of the shared async-operation lifecycle seam.
2//!
3//! Per-operation canonical lifecycle state lives in the MeerkatMachine DSL
4//! authority (`op_statuses`, `op_terminal_outcomes`, `op_kinds`,
5//! `op_sources`, `op_peer_ready`, `op_progress_counts`, `active_op_count`,
6//! `wait_active`, `wait_operation_ids`). This shell layer owns pure mechanics: watcher
7//! channels, timestamps, peer handles, snapshot assembly, FIFO eviction
8//! bookkeeping, the completion feed buffer, and typed delivery of generated
9//! admission/rejection feedback.
10//!
11//! Per-transition legality ("is `CompleteOp` legal on a `Provisioning` op?")
12//! is NOT owned by the shell — it lives in the DSL's `from_status_valid`
13//! guards on each op-lifecycle transition. The shell's only job on a
14//! `GuardRejected` rejection is to ask the generated rejection resolver for
15//! the public result class.
16
17use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24    CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31    CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32    OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33    OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34    OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35    OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36    WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43// ---------------------------------------------------------------------------
44// Serde-only persisted canonical state shells
45// ---------------------------------------------------------------------------
46//
47// These structures preserve the persisted fact shape of `PersistedOpsSnapshot`.
48// They are pure serde shells — no methods beyond read-only field accessors,
49// no authority behavior. Optional generated facts that can be explicitly
50// absent still serialize as present `null` so recovery can distinguish
51// generated "none" from a missing persisted fact.
52
53fn deserialize_required_operation_source<'de, D>(
54    deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57    D: serde::Deserializer<'de>,
58{
59    <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62/// Canonical per-operation state as captured in persisted snapshots.
63#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65    status: OperationStatus,
66    kind: OperationKind,
67    #[serde(deserialize_with = "deserialize_required_operation_source")]
68    operation_source: Option<OperationSource>,
69    peer_ready: bool,
70    progress_count: u32,
71    watcher_count: u32,
72    terminal_outcome: Option<OperationTerminalOutcome>,
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    completion_sequence: Option<CompletionSeq>,
75    terminal_buffered: bool,
76}
77
78/// Generated-owned public completion feed fact captured from DSL authority.
79#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81    seq: CompletionSeq,
82    kind: OperationKind,
83    terminal_outcome: OperationTerminalOutcome,
84}
85
86/// Canonical registry-level state as captured in persisted snapshots.
87#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89    operations: HashMap<OperationId, OperationCanonicalState>,
90    completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91    completed_order: VecDeque<OperationId>,
92    max_completed: usize,
93    max_concurrent: Option<usize>,
94    active_count: usize,
95    wait_request_id: Option<WaitRequestId>,
96    wait_operation_ids: Vec<OperationId>,
97    next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101    /// Maximum completed operations retained at capture time.
102    pub fn max_completed(&self) -> usize {
103        self.max_completed
104    }
105
106    /// Maximum concurrent non-terminal operations at capture time.
107    pub fn max_concurrent(&self) -> Option<usize> {
108        self.max_concurrent
109    }
110
111    /// Number of operations captured in the snapshot.
112    pub fn operation_count(&self) -> usize {
113        self.operations.len()
114    }
115
116    /// Number of generated-owned public completion feed entries captured.
117    pub fn completion_feed_count(&self) -> usize {
118        self.completion_feed_entries.len()
119    }
120}
121
122// ---------------------------------------------------------------------------
123// Serializable snapshot for persistence
124// ---------------------------------------------------------------------------
125
126/// Serializable snapshot of the ops lifecycle registry state.
127///
128/// Captured on terminal transitions for durable persistence. Contains
129/// canonical state, operation specs, persisted completion feed entries, and
130/// consumer cursor values. Wire format preserved verbatim from legacy
131/// runtime versions for backward compatibility.
132#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134    /// Epoch identity at capture time.
135    pub epoch_id: meerkat_core::RuntimeEpochId,
136    /// Canonical machine-owned state at capture time.
137    pub authority_state: RegistryCanonicalState,
138    /// Per-operation specs for shell record reconstruction.
139    pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140    /// Persisted completion feed projection metadata. Canonical feed truth is
141    /// captured in `authority_state.completion_feed_entries`.
142    pub completion_entries: Vec<CompletionEntry>,
143    /// Consumer cursor snapshot at capture time.
144    pub cursors: meerkat_core::EpochCursorSnapshot,
145}
146
147#[derive(Debug)]
148pub struct OpsLifecyclePersistenceRequest {
149    snapshot: PersistedOpsSnapshot,
150    result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
151}
152
153impl OpsLifecyclePersistenceRequest {
154    pub fn snapshot(&self) -> &PersistedOpsSnapshot {
155        &self.snapshot
156    }
157
158    pub fn complete(self, result: Result<(), OpsLifecycleError>) {
159        let _ = self.result_tx.send(result);
160    }
161}
162
163// ---------------------------------------------------------------------------
164// Concrete completion feed buffer
165// ---------------------------------------------------------------------------
166
167/// Shared inner state of the completion feed buffer.
168///
169/// Protected by the registry's `RwLock<ShellState>` for writes, and by its
170/// own `RwLock` for reads by external consumers (agent boundary, idle wake).
171#[derive(Debug)]
172struct FeedBufferInner {
173    entries: VecDeque<CompletionEntry>,
174    watermark: CompletionSeq,
175    max_retained: usize,
176}
177
178/// Shared completion feed buffer owned by the runtime registry.
179///
180/// The registry writes entries under its own write lock. External consumers
181/// read through the [`RuntimeCompletionFeed`] handle.
182#[derive(Debug)]
183struct FeedBuffer {
184    inner: RwLock<FeedBufferInner>,
185    /// Atomic mirror of watermark for lock-free `watermark()` reads.
186    watermark_atomic: AtomicU64,
187    /// Notifies all waiters when new entries are appended.
188    notify: tokio::sync::Notify,
189}
190
191impl FeedBuffer {
192    fn new(max_retained: usize) -> Self {
193        Self {
194            inner: RwLock::new(FeedBufferInner {
195                entries: VecDeque::new(),
196                watermark: 0,
197                max_retained,
198            }),
199            watermark_atomic: AtomicU64::new(0),
200            notify: tokio::sync::Notify::new(),
201        }
202    }
203
204    fn push(&self, entry: CompletionEntry) {
205        let mut inner = self
206            .inner
207            .write()
208            .unwrap_or_else(std::sync::PoisonError::into_inner);
209        let seq = entry.seq;
210        inner.entries.push_back(entry);
211        inner.watermark = seq;
212
213        // Evict oldest if over capacity.
214        while inner.entries.len() > inner.max_retained {
215            inner.entries.pop_front();
216        }
217
218        drop(inner);
219
220        self.watermark_atomic.store(seq, Ordering::Release);
221        self.notify.notify_waiters();
222    }
223}
224
225/// Read-only handle to the runtime completion feed.
226///
227/// Implements [`CompletionFeed`] for external consumers. Obtained via
228/// [`RuntimeOpsLifecycleRegistry::completion_feed()`].
229#[derive(Debug, Clone)]
230pub struct RuntimeCompletionFeed {
231    buffer: Arc<FeedBuffer>,
232}
233
234impl CompletionFeed for RuntimeCompletionFeed {
235    fn watermark(&self) -> CompletionSeq {
236        self.buffer.watermark_atomic.load(Ordering::Acquire)
237    }
238
239    fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
240        let inner = self
241            .buffer
242            .inner
243            .read()
244            .unwrap_or_else(std::sync::PoisonError::into_inner);
245        let entries: Vec<CompletionEntry> = inner
246            .entries
247            .iter()
248            .filter(|e| e.seq > after_seq)
249            .cloned()
250            .collect();
251        let watermark = inner.watermark;
252        CompletionBatch { entries, watermark }
253    }
254
255    fn wait_for_advance(
256        &self,
257        after_seq: CompletionSeq,
258    ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
259        Box::pin(async move {
260            loop {
261                // Register the waiter BEFORE reading the watermark.
262                // notify_waiters() in push() only wakes already-registered
263                // listeners — if we read first and push() lands between the
264                // read and notified().await, the wake is lost.
265                let notified = self.buffer.notify.notified();
266                let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
267                if current > after_seq {
268                    return current;
269                }
270                notified.await;
271            }
272        })
273    }
274}
275
276// ---------------------------------------------------------------------------
277// Shell-only per-operation record (not part of canonical machine state)
278// ---------------------------------------------------------------------------
279
280#[derive(Debug)]
281struct OperationCompletionNotifier {
282    tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
283}
284
285impl OperationCompletionNotifier {
286    fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
287        Self { tx }
288    }
289
290    fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
291        let _ = self.tx.send(outcome.clone());
292    }
293}
294
295fn operation_completion_watch_from_receiver(
296    rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
297) -> OperationCompletionWatch {
298    Box::pin(async move {
299        rx.await
300            .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
301    })
302}
303
304fn resolved_operation_completion_watch(
305    outcome: OperationTerminalOutcome,
306) -> OperationCompletionWatch {
307    Box::pin(async move { Ok(outcome) })
308}
309
310/// Shell-owned data for a single operation. Canonical lifecycle state lives in
311/// the DSL authority; this struct holds I/O concerns that the DSL has no
312/// knowledge of.
313#[derive(Debug)]
314struct ShellRecord {
315    spec: OperationSpec,
316    peer_handle: Option<OperationPeerHandle>,
317    /// Private waiter plumbing. Notifiers are drained only from
318    /// `finalize_terminal()` after the generated authority has accepted and
319    /// stored a terminal outcome.
320    watchers: Vec<OperationCompletionNotifier>,
321    // Monotonic timestamps for elapsed computation
322    created_at: Instant,
323    started_at: Option<Instant>,
324    completed_at: Option<Instant>,
325    // Wall-clock anchor captured at creation for epoch millis
326    created_at_wall: SystemTime,
327}
328
329#[derive(Debug)]
330struct PendingWaitState {
331    wait_request_id: WaitRequestId,
332    sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
333}
334
335enum WaitAllAuthorityPlan {
336    AlreadySatisfied(WaitAllSatisfied),
337    ActivateBarrier,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq)]
341enum RecoveredOperationRecordDisposition {
342    Retain,
343    Discard,
344}
345
346impl ShellRecord {
347    fn new(spec: OperationSpec) -> Self {
348        Self {
349            spec,
350            peer_handle: None,
351            watchers: Vec::new(),
352            created_at: Instant::now(),
353            started_at: None,
354            completed_at: None,
355            created_at_wall: SystemTime::now(),
356        }
357    }
358
359    fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
360        wall_anchor
361            .duration_since(UNIX_EPOCH)
362            .map(|d| d.as_millis() as u64)
363            .unwrap_or(0)
364    }
365
366    fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
367        // Compute wall time for a given instant using the wall-clock anchor:
368        // wall_time = created_at_wall + (instant - created_at)
369        let offset = instant.saturating_duration_since(self.created_at);
370        let wall = self.created_at_wall + offset;
371        Self::epoch_millis(&wall)
372    }
373
374    /// Notify all watchers with the given terminal outcome and drain the list.
375    fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
376        for watcher in std::mem::take(&mut self.watchers) {
377            watcher.notify_after_generated_terminal(outcome);
378        }
379    }
380
381    /// Mark the completion timestamp.
382    fn mark_completed(&mut self) {
383        self.completed_at = Some(Instant::now());
384    }
385}
386
387// ---------------------------------------------------------------------------
388// Combined shell state: DSL authority + shell records
389// ---------------------------------------------------------------------------
390
391#[derive(Debug)]
392struct ShellState {
393    /// DSL authority — sole source of truth for per-op canonical state.
394    dsl: DslAuthority,
395    /// Shell-owned per-operation records (specs, watchers, timestamps, peer handles).
396    records: HashMap<OperationId, ShellRecord>,
397    /// Pending wait-all coordination (oneshot channel).
398    pending_wait: Option<PendingWaitState>,
399    /// FIFO ordering of completed operation IDs for bounded eviction.
400    completed_order: VecDeque<OperationId>,
401    /// Maximum completed operations to retain.
402    max_completed: usize,
403    /// Maximum concurrent non-terminal operations (None = unlimited).
404    max_concurrent: Option<usize>,
405    /// Oneshot correlation id for the currently-pending `wait_all` future.
406    ///
407    /// Barrier membership (`wait_operation_ids`) and activation (`wait_active`)
408    /// are DSL-owned. This field is pure transport mechanics — the identity
409    /// the oneshot sender is tagged with so `Drop` can correlate cancellation.
410    wait_request_id: Option<WaitRequestId>,
411    /// Shared feed buffer for completion events.
412    feed_buffer: Arc<FeedBuffer>,
413    /// Persistence channel for durable snapshot writes (set via `set_persistence_channel`).
414    persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
415    /// Terminal unregister permanently fences any later channel replacement.
416    persistence_sealed: bool,
417    /// Epoch ID for persistence snapshots.
418    persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
419    /// Shared cursor state for persistence snapshots.
420    persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
421    /// Mechanical epoch fence closed by canonical owner teardown.
422    ///
423    /// Generated authority terminalizes each known operation. This bit closes
424    /// the shell admission door so detached callbacks that retained an Arc to
425    /// the registry cannot create or mutate operations after unregister.
426    owner_retired: bool,
427}
428
429/// Wrapper around the DSL authority that provides `Debug` output.
430///
431/// The generated `MeerkatMachineAuthority` does not derive `Debug`, but
432/// `ShellState` requires it. This wrapper delegates to the inner state's
433/// `Debug` impl.
434struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
435
436impl std::fmt::Debug for DslAuthority {
437    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
438        f.debug_struct("DslAuthority")
439            .field("state", self.0.state())
440            .finish()
441    }
442}
443
444/// Create a DSL authority initialized through generated authority. Per-op
445/// transitions guard only on `op_statuses.contains_key(operation_id)`, so the
446/// phase stays in `Idle` permanently (they all `to Idle`).
447fn new_ops_dsl_authority() -> DslAuthority {
448    DslAuthority(Box::new(
449        crate::meerkat_machine::dsl_authority::new_initialized_authority(
450            "ops lifecycle DSL Initialize must be accepted",
451        ),
452    ))
453}
454
455impl ShellState {
456    fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
457        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
458        let dsl = new_ops_dsl_authority();
459        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
460        let feed_capacity = max_completed.saturating_mul(4).max(1024);
461        tracing::info!(
462            feed_capacity,
463            "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
464        );
465        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
466        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
467        Self {
468            dsl,
469            records: HashMap::new(),
470            pending_wait: None,
471            completed_order: VecDeque::new(),
472            max_completed,
473            max_concurrent,
474            wait_request_id: None,
475            // Feed buffer is larger than max_completed to absorb bursts.
476            // Entries are only evicted by buffer capacity, not by consumer cursor,
477            // so the buffer must be large enough that consumers drain before
478            // the oldest entry is evicted.
479            feed_buffer,
480            persist_tx: None,
481            persistence_sealed: false,
482            persist_epoch_id: None,
483            persist_cursor_state: None,
484            owner_retired: false,
485        }
486    }
487
488    fn ensure_owner_active(&self) -> Result<(), OpsLifecycleError> {
489        if self.owner_retired {
490            Err(OpsLifecycleError::OwnerRetired)
491        } else {
492            Ok(())
493        }
494    }
495
496    /// Apply a DSL input, mapping transition errors into
497    /// [`OpsLifecycleError::Internal`]. Callers that need to distinguish
498    /// guard rejections (legal-transition violations) from internal desync
499    /// should use [`Self::dsl_apply_raw`] and classify the error themselves;
500    /// this helper is for DSL inputs whose preconditions the caller has
501    /// already fully validated (e.g., `RequestWaitAll`, `SatisfyWaitAll`).
502    fn dsl_apply(
503        &mut self,
504        input: mm_dsl::MeerkatMachineInput,
505        context: &str,
506    ) -> Result<(), OpsLifecycleError> {
507        self.dsl_apply_raw(input).map_err(|err| {
508            OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
509        })
510    }
511
512    /// Apply a DSL input, returning the raw kernel-level rejection so callers
513    /// can distinguish `GuardRejected` (a legitimate legality violation, e.g.,
514    /// `complete_operation` on a `Provisioning` op) from
515    /// `NoMatchingTransition` (a shell/DSL desync). Op-lifecycle entry points
516    /// feed guard rejections into generated rejection feedback before surfacing
517    /// public result classes.
518    fn dsl_apply_raw(
519        &mut self,
520        input: mm_dsl::MeerkatMachineInput,
521    ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
522        mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
523    }
524
525    fn dsl_apply_with_effects(
526        &mut self,
527        input: mm_dsl::MeerkatMachineInput,
528        context: &str,
529    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
530        let transition =
531            mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
532                OpsLifecycleError::Internal(format!(
533                    "DSL rejected ops transition ({context}): {err:?}"
534                ))
535            })?;
536        Ok(transition.into_effects())
537    }
538
539    /// Fail-closed read of a typed terminal payload entry: the payload IS the
540    /// domain [`OperationTerminalOutcome`] (K8b fold — no JSON codec), but the
541    /// shell still refuses to surface a payload whose variant disagrees with
542    /// the recorded discriminant.
543    fn checked_terminal_payload(
544        kind: mm_dsl::OperationTerminalOutcomeKind,
545        payload: &OperationTerminalOutcome,
546        authority: &str,
547        operation_id: &str,
548    ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
549        if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
550            return Err(OpsLifecycleError::Internal(format!(
551                "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
552            )));
553        }
554        Ok(payload.clone())
555    }
556
557    /// Read the DSL operation status for `id`, or `None` if not registered.
558    fn status(&self, id: &OperationId) -> Option<OperationStatus> {
559        let id_key = mm_dsl::OperationId::from_domain(id).0;
560        self.dsl
561            .0
562            .state()
563            .op_statuses
564            .get(&id_key)
565            .copied()
566            .map(OperationStatus::from)
567    }
568
569    fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
570        self.status(id).ok_or_else(|| {
571            OpsLifecycleError::Internal(format!(
572                "generated op lifecycle authority missing status for {id}"
573            ))
574        })
575    }
576
577    /// Read the DSL operation kind for `id`, or `None` if not registered.
578    fn kind(&self, id: &OperationId) -> Option<OperationKind> {
579        let id_key = mm_dsl::OperationId::from_domain(id).0;
580        self.dsl
581            .0
582            .state()
583            .op_kinds
584            .get(&id_key)
585            .copied()
586            .map(OperationKind::from)
587    }
588
589    fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
590        self.kind(id).ok_or_else(|| {
591            OpsLifecycleError::Internal(format!(
592                "generated op lifecycle authority missing kind for {id}"
593            ))
594        })
595    }
596
597    fn operation_source(
598        &self,
599        id: &OperationId,
600    ) -> Result<Option<OperationSource>, OpsLifecycleError> {
601        let id_key = mm_dsl::OperationId::from_domain(id).0;
602        self.dsl
603            .0
604            .state()
605            .op_sources
606            .get(&id_key)
607            .map(|source| {
608                source.to_domain().map_err(|error| {
609                    OpsLifecycleError::Internal(format!(
610                        "generated operation source authority has invalid source for {id}: {error}"
611                    ))
612                })
613            })
614            .transpose()
615    }
616
617    fn child_session_id_from_operation_source(
618        operation_source: Option<&OperationSource>,
619    ) -> Option<SessionId> {
620        match operation_source {
621            Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
622            Some(OperationSource::BackendPeer { .. } | OperationSource::DetachedJob { .. })
623            | None => None,
624        }
625    }
626
627    fn align_spec_child_session_id_to_source(
628        spec: &mut OperationSpec,
629        operation_source: Option<&OperationSource>,
630    ) {
631        spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
632    }
633
634    /// Read the peer-ready flag for `id`.
635    fn peer_ready(&self, id: &OperationId) -> Option<bool> {
636        let id_key = mm_dsl::OperationId::from_domain(id).0;
637        self.dsl.0.state().op_peer_ready.get(&id_key).copied()
638    }
639
640    fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
641        self.peer_ready(id).ok_or_else(|| {
642            OpsLifecycleError::Internal(format!(
643                "generated op peer wiring authority missing peer-ready fact for {id}"
644            ))
645        })
646    }
647
648    /// Read the progress counter for `id`.
649    fn progress_count(&self, id: &OperationId) -> Option<u32> {
650        let id_key = mm_dsl::OperationId::from_domain(id).0;
651        self.dsl
652            .0
653            .state()
654            .op_progress_counts
655            .get(&id_key)
656            .map(|v| (*v).min(u32::MAX as u64) as u32)
657    }
658
659    fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
660        self.progress_count(id).ok_or_else(|| {
661            OpsLifecycleError::Internal(format!(
662                "generated op progress authority missing progress count for {id}"
663            ))
664        })
665    }
666
667    /// Read the terminal outcome for `id` by pairing the DSL's typed
668    /// discriminant with the companion payload JSON. Returns `None` when the
669    /// op has no recorded terminal discriminant.
670    fn terminal_outcome(
671        &self,
672        id: &OperationId,
673    ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
674        let id_key = mm_dsl::OperationId::from_domain(id).0;
675        let state = self.dsl.0.state();
676        let status = self.status(id);
677        let terminal = match status {
678            Some(status) => Self::operation_status_is_terminal(id, status)?,
679            None => false,
680        };
681        let kind = state.op_terminal_outcomes.get(&id_key).copied();
682        let Some(kind) = kind else {
683            if terminal {
684                return Err(OpsLifecycleError::Internal(format!(
685                    "generated op terminal authority missing terminal outcome for {id}"
686                )));
687            }
688            return Ok(None);
689        };
690        if !terminal {
691            return Err(OpsLifecycleError::Internal(format!(
692                "generated op terminal authority has terminal outcome for non-terminal {id}"
693            )));
694        }
695        let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
696            OpsLifecycleError::Internal(format!(
697                "generated op terminal authority missing terminal payload for {id}"
698            ))
699        })?;
700        Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
701            .map(Some)
702    }
703
704    /// Whether the operation is currently tracked in DSL state.
705    fn contains(&self, id: &OperationId) -> bool {
706        let id_key = mm_dsl::OperationId::from_domain(id).0;
707        self.dsl.0.state().op_statuses.contains_key(&id_key)
708    }
709
710    /// Number of non-terminal operations (derived from DSL state).
711    fn active_count(&self) -> usize {
712        self.dsl.0.state().active_op_count as usize
713    }
714
715    /// Number of operations currently tracked (including terminal).
716    fn operation_count(&self) -> usize {
717        self.dsl.0.state().op_statuses.len()
718    }
719
720    /// Iterate over all tracked operation IDs (DSL keys converted to domain).
721    fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
722        let mut ids = BTreeSet::new();
723        let state = self.dsl.0.state();
724        Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
725        Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
726        Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
727        Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
728        Self::collect_operation_id_keys(
729            &mut ids,
730            "op_progress_counts",
731            state.op_progress_counts.keys(),
732        )?;
733        Self::collect_operation_id_keys(
734            &mut ids,
735            "op_terminal_outcomes",
736            state.op_terminal_outcomes.keys(),
737        )?;
738        Self::collect_operation_id_keys(
739            &mut ids,
740            "op_terminal_payload",
741            state.op_terminal_payload.keys(),
742        )?;
743        Self::collect_operation_id_keys(
744            &mut ids,
745            "op_completion_seq",
746            state.op_completion_seq.keys(),
747        )?;
748        ids.extend(self.records.keys().cloned());
749        Ok(ids.into_iter().collect())
750    }
751
752    fn collect_operation_id_keys<'a, I>(
753        ids: &mut BTreeSet<OperationId>,
754        field: &str,
755        keys: I,
756    ) -> Result<(), OpsLifecycleError>
757    where
758        I: IntoIterator<Item = &'a String>,
759    {
760        for key in keys {
761            let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
762                OpsLifecycleError::Internal(format!(
763                    "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
764                ))
765            })?;
766            ids.insert(id);
767        }
768        Ok(())
769    }
770
771    fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
772        let id_key = mm_dsl::OperationId::from_domain(id).0;
773        let state = self.dsl.0.state();
774        state.op_statuses.contains_key(&id_key)
775            || state.op_kinds.contains_key(&id_key)
776            || state.op_sources.contains_key(&id_key)
777            || state.op_peer_ready.contains_key(&id_key)
778            || state.op_progress_counts.contains_key(&id_key)
779            || state.op_terminal_outcomes.contains_key(&id_key)
780            || state.op_terminal_payload.contains_key(&id_key)
781            || state.op_completion_seq.contains_key(&id_key)
782    }
783
784    /// Read the DSL-minted completion sequence for a terminal operation.
785    fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
786        let id_key = mm_dsl::OperationId::from_domain(id).0;
787        self.dsl.0.state().op_completion_seq.get(&id_key).copied()
788    }
789
790    fn completion_feed_authority_entries(
791        &self,
792    ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
793        let state = self.dsl.0.state();
794        let sequence_keys: BTreeSet<String> =
795            state.completion_feed_sequences.keys().cloned().collect();
796        let companion_domains: [(&str, BTreeSet<String>); 3] = [
797            (
798                "completion_feed_kinds",
799                state.completion_feed_kinds.keys().cloned().collect(),
800            ),
801            (
802                "completion_feed_terminal_outcomes",
803                state
804                    .completion_feed_terminal_outcomes
805                    .keys()
806                    .cloned()
807                    .collect(),
808            ),
809            (
810                "completion_feed_terminal_payload",
811                state
812                    .completion_feed_terminal_payload
813                    .keys()
814                    .cloned()
815                    .collect(),
816            ),
817        ];
818        for (field, keys) in companion_domains {
819            if keys != sequence_keys {
820                return Err(OpsLifecycleError::Internal(format!(
821                    "generated completion feed authority has mismatched {field} domain"
822                )));
823            }
824        }
825
826        let mut entries = HashMap::new();
827        for (id_key, seq) in &state.completion_feed_sequences {
828            if !state.completion_sequence_claims.contains(seq) {
829                return Err(OpsLifecycleError::Internal(format!(
830                    "generated completion feed authority sequence {seq} for {id_key} is not claimed"
831                )));
832            }
833            let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
834                OpsLifecycleError::Internal(format!(
835                    "generated completion feed authority used invalid operation id key {id_key}: {error}"
836                ))
837            })?;
838            let kind = state
839                .completion_feed_kinds
840                .get(id_key)
841                .copied()
842                .map(OperationKind::from)
843                .ok_or_else(|| {
844                    OpsLifecycleError::Internal(format!(
845                        "generated completion feed authority missing kind for {id_key}"
846                    ))
847                })?;
848            let outcome_kind = state
849                .completion_feed_terminal_outcomes
850                .get(id_key)
851                .copied()
852                .ok_or_else(|| {
853                    OpsLifecycleError::Internal(format!(
854                        "generated completion feed authority missing terminal outcome for {id_key}"
855                    ))
856                })?;
857            let payload = state
858                .completion_feed_terminal_payload
859                .get(id_key)
860                .ok_or_else(|| {
861                    OpsLifecycleError::Internal(format!(
862                        "generated completion feed authority missing terminal payload for {id_key}"
863                    ))
864                })?;
865            let terminal_outcome = Self::checked_terminal_payload(
866                outcome_kind,
867                payload,
868                "generated completion feed authority",
869                id_key,
870            )?;
871            entries.insert(
872                operation_id,
873                CompletionFeedCanonicalState {
874                    seq: *seq,
875                    kind,
876                    terminal_outcome,
877                },
878            );
879        }
880        Ok(entries)
881    }
882
883    fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
884        let state = self.dsl.0.state();
885        match consumer {
886            CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
887            CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
888            CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
889        }
890    }
891
892    fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
893        meerkat_core::EpochCursorSnapshot {
894            agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
895            runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
896            runtime_last_injected_seq: self
897                .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
898        }
899    }
900
901    /// Build a snapshot from DSL state + shell record.
902    fn snapshot(
903        &self,
904        id: &OperationId,
905    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
906        let Some(shell) = self.records.get(id) else {
907            if self.has_generated_operation_record_fact(id) {
908                return Err(OpsLifecycleError::Internal(format!(
909                    "generated op lifecycle authority has operation facts without shell projection record for {id}"
910                )));
911            }
912            return Ok(None);
913        };
914        let kind = self.require_kind(id)?;
915        let status = self.require_status(id)?;
916        let terminal = Self::operation_status_is_terminal(id, status)?;
917        let public_result_class = Self::operation_public_result_class(id, status)?;
918        let peer_ready = self.require_peer_ready(id)?;
919        let progress_count = self.require_progress_count(id)?;
920        let operation_source = self.operation_source(id)?;
921        let terminal_outcome = self.terminal_outcome(id)?;
922
923        let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
924        let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
925        let completed_at_ms = shell
926            .completed_at
927            .map(|i| shell.epoch_millis_for_instant(i));
928        let elapsed_ms = shell.completed_at.map(|completed| {
929            completed
930                .saturating_duration_since(shell.created_at)
931                .as_millis() as u64
932        });
933
934        Ok(Some(OperationLifecycleSnapshot {
935            id: shell.spec.id.clone(),
936            kind,
937            owner_session_id: shell.spec.owner_session_id.clone(),
938            display_name: shell.spec.display_name.clone(),
939            source_label: shell.spec.source_label.clone(),
940            child_session_id: Self::child_session_id_from_operation_source(
941                operation_source.as_ref(),
942            ),
943            operation_source,
944            expect_peer_channel: shell.spec.expect_peer_channel,
945            status,
946            terminal,
947            public_result_class,
948            peer_ready,
949            progress_count,
950            watcher_count: shell.watchers.len() as u32,
951            terminal_outcome,
952            peer_handle: shell.peer_handle.clone(),
953            created_at_ms,
954            started_at_ms,
955            completed_at_ms,
956            elapsed_ms,
957        }))
958    }
959
960    /// Emit shell-side mechanics for a terminal transition: notify watchers,
961    /// push generated-authorized CompletionEntry rows, retain in FIFO, evict
962    /// as needed. Called AFTER the DSL transition has already persisted the
963    /// terminal status + outcome.
964    fn finalize_terminal(
965        &mut self,
966        id: &OperationId,
967    ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
968        let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
969            OpsLifecycleError::Internal(format!(
970                "generated op terminal transition did not mint terminal outcome for {id}"
971            ))
972        })?;
973        let kind = self.require_kind(id)?;
974
975        // Notify watchers and mark completion timestamp.
976        if let Some(shell) = self.records.get_mut(id) {
977            shell.notify_watchers(&outcome);
978            shell.mark_completed();
979        }
980
981        if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
982        {
983            self.dsl_apply(
984                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
985                    operation_id: mm_dsl::OperationId::from_domain(id).0,
986                },
987                "CollectCompletedOp",
988            )?;
989            self.records.remove(id);
990            self.completed_order.retain(|queued| queued != id);
991            return Ok(None);
992        }
993
994        let mut completion_entry = None;
995        if Self::operation_completion_feed_class(id, kind)?
996            == mm_dsl::OperationCompletionFeedClass::Emit
997        {
998            let feed_authority = self
999                .completion_feed_authority_entries()?
1000                .remove(id)
1001                .ok_or_else(|| {
1002                    OpsLifecycleError::Internal(format!(
1003                        "generated op terminal transition did not mint completion feed authority for {id}"
1004                    ))
1005                })?;
1006            if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
1007                return Err(OpsLifecycleError::Internal(format!(
1008                    "generated completion feed authority drifted from terminal op authority for {id}"
1009                )));
1010            }
1011            let seq = self.completion_sequence(id).ok_or_else(|| {
1012                OpsLifecycleError::Internal(format!(
1013                    "generated op terminal transition did not mint completion sequence for {id}"
1014                ))
1015            })?;
1016            if feed_authority.seq != seq {
1017                return Err(OpsLifecycleError::Internal(format!(
1018                    "generated completion feed authority sequence drifted for {id}"
1019                )));
1020            }
1021            let display_name = self
1022                .records
1023                .get(id)
1024                .map(|r| r.spec.display_name.clone())
1025                .unwrap_or_default();
1026            let completed_at_ms = self
1027                .records
1028                .get(id)
1029                .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1030            completion_entry = Some(CompletionEntry {
1031                seq: feed_authority.seq,
1032                operation_id: id.clone(),
1033                kind: feed_authority.kind,
1034                display_name,
1035                terminal_outcome: feed_authority.terminal_outcome,
1036                completed_at_ms,
1037            });
1038        }
1039
1040        // FIFO retention + eviction.
1041        self.completed_order.push_back(id.clone());
1042        while self.completed_order.len() > self.max_completed {
1043            if let Some(evicted) = self.completed_order.pop_front() {
1044                self.dsl_apply(
1045                    mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1046                        operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1047                    },
1048                    "EvictCompletedOp",
1049                )?;
1050                self.records.remove(&evicted);
1051            }
1052        }
1053
1054        // Satisfy a pending wait request if all its ops are now terminal. On
1055        // authority-invariant corruption this propagates the typed fault (and
1056        // drops the barrier oneshot so the waiter resolves to Err) instead of
1057        // reporting the op terminal with a silently-hung barrier.
1058        self.maybe_satisfy_wait()?;
1059        Ok(completion_entry)
1060    }
1061
1062    fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1063        if let Some(entry) = entry {
1064            self.feed_buffer.push(entry);
1065        }
1066    }
1067
1068    /// Read barrier membership from DSL state (sole owner).
1069    fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1070        self.dsl
1071            .0
1072            .state()
1073            .wait_operation_ids
1074            .iter()
1075            .map(|key| {
1076                serde_json::from_str::<OperationId>(key).map_err(|error| {
1077                    OpsLifecycleError::Internal(format!(
1078                        "generated wait operation identity authority used invalid operation id key {key}: {error}"
1079                    ))
1080                })
1081            })
1082            .collect()
1083    }
1084
1085    /// Whether the DSL has a barrier wait active.
1086    #[cfg(test)]
1087    fn wait_active(&self) -> bool {
1088        self.dsl.0.state().wait_active
1089    }
1090
1091    fn wait_all_satisfied_from_effects(
1092        effects: &[mm_dsl::MeerkatMachineEffect],
1093    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1094        let mut satisfied = None;
1095        for effect in effects {
1096            let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1097                wait_request_id,
1098                run_id,
1099                operation_ids,
1100            } = effect
1101            else {
1102                continue;
1103            };
1104            if satisfied.is_some() {
1105                return Err(OpsLifecycleError::Internal(
1106                    "generated wait_all authority emitted multiple satisfaction effects".into(),
1107                ));
1108            }
1109            let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1110                OpsLifecycleError::Internal(format!(
1111                    "generated wait_all authority emitted invalid wait request id '{}': {err}",
1112                    wait_request_id.0
1113                ))
1114            })?;
1115            let mut ids = Vec::with_capacity(operation_ids.len());
1116            for operation_id in operation_ids {
1117                ids.push(
1118                    serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1119                        OpsLifecycleError::Internal(format!(
1120                            "generated wait_all authority emitted invalid operation id '{}': {err}",
1121                            operation_id.0
1122                        ))
1123                    })?,
1124                );
1125            }
1126            satisfied = Some(WaitAllSatisfied {
1127                wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1128                run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1129                    OpsLifecycleError::Internal(format!(
1130                        "generated wait_all authority emitted invalid run id '{}': {err}",
1131                        run_id.0
1132                    ))
1133                })?),
1134                operation_ids: ids,
1135            });
1136        }
1137        Ok(satisfied)
1138    }
1139
1140    fn parse_wait_all_operation_id(
1141        raw: &str,
1142        context: &str,
1143    ) -> Result<OperationId, OpsLifecycleError> {
1144        serde_json::from_str::<OperationId>(raw).map_err(|err| {
1145            OpsLifecycleError::Internal(format!(
1146                "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1147            ))
1148        })
1149    }
1150
1151    fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1152        let mut seen = HashSet::new();
1153        operation_ids
1154            .iter()
1155            .find(|operation_id| !seen.insert((*operation_id).clone()))
1156            .cloned()
1157    }
1158
1159    fn wait_all_admission_error_from_effects(
1160        wait_request_id: &WaitRequestId,
1161        effects: &[mm_dsl::MeerkatMachineEffect],
1162    ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1163        let mut admission = None;
1164        for effect in effects {
1165            let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1166                wait_request_id: resolved_wait_request_id,
1167                result,
1168                reject_reason,
1169                rejected_operation_id,
1170            } = effect
1171            else {
1172                continue;
1173            };
1174            if admission.is_some() {
1175                return Err(OpsLifecycleError::Internal(
1176                    "generated wait_all authority emitted multiple admission results".into(),
1177                ));
1178            }
1179            let resolved_uuid =
1180                uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1181                    OpsLifecycleError::Internal(format!(
1182                        "generated wait_all authority emitted invalid wait request id '{}': {err}",
1183                        resolved_wait_request_id.0
1184                    ))
1185                })?;
1186            let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1187            if &resolved_wait_request_id != wait_request_id {
1188                return Err(OpsLifecycleError::Internal(format!(
1189                    "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1190                )));
1191            }
1192            admission = Some(match result {
1193                mm_dsl::WaitAllAdmissionResultKind::Accept => {
1194                    if reject_reason.is_some() || rejected_operation_id.is_some() {
1195                        return Err(OpsLifecycleError::Internal(
1196                            "generated wait_all authority accepted with rejection payload".into(),
1197                        ));
1198                    }
1199                    None
1200                }
1201                mm_dsl::WaitAllAdmissionResultKind::Reject => {
1202                    let reason = reject_reason.ok_or_else(|| {
1203                        OpsLifecycleError::Internal(
1204                            "generated wait_all authority rejected without reason".into(),
1205                        )
1206                    })?;
1207                    let error = match reason {
1208                        mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1209                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1210                                OpsLifecycleError::Internal(
1211                                    "generated wait_all authority rejected duplicate without operation id"
1212                                        .into(),
1213                                )
1214                            })?;
1215                            OpsLifecycleError::DuplicateWaitOperation(
1216                                Self::parse_wait_all_operation_id(raw, "duplicate")?,
1217                            )
1218                        }
1219                        mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1220                            if rejected_operation_id.is_some() {
1221                                return Err(OpsLifecycleError::Internal(
1222                                    "generated wait_all authority rejected active wait with operation id"
1223                                        .into(),
1224                                ));
1225                            }
1226                            OpsLifecycleError::WaitAlreadyActive
1227                        }
1228                        mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1229                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1230                                OpsLifecycleError::Internal(
1231                                    "generated wait_all authority rejected missing operation without operation id"
1232                                        .into(),
1233                                )
1234                            })?;
1235                            OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1236                                raw, "missing",
1237                            )?)
1238                        }
1239                    };
1240                    Some(error)
1241                }
1242            });
1243        }
1244        admission.ok_or_else(|| {
1245            OpsLifecycleError::Internal(
1246                "generated wait_all authority emitted no admission result".into(),
1247            )
1248        })
1249    }
1250
1251    fn resolve_wait_all_admission(
1252        &mut self,
1253        wait_request_id: &WaitRequestId,
1254        operation_ids: &[OperationId],
1255        dsl_ids: &BTreeSet<String>,
1256        dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1257        operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1258        operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1259    ) -> Result<(), OpsLifecycleError> {
1260        let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1261            .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1262        let not_found = operation_ids
1263            .iter()
1264            .find(|operation_id| !self.contains(operation_id))
1265            .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1266        let dsl_id_sequence: Vec<String> = operation_ids
1267            .iter()
1268            .map(|id| mm_dsl::OperationId::from_domain(id).0)
1269            .collect();
1270        let effects = self.dsl_apply_with_effects(
1271            mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1272                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1273                operation_id_sequence: dsl_id_sequence,
1274                operation_ids: dsl_ids.clone(),
1275                operation_id_tokens: dsl_id_tokens.clone(),
1276                operation_token_by_id: operation_token_by_id.clone(),
1277                operation_id_by_token: operation_id_by_token.clone(),
1278                duplicate_operation_id: duplicate,
1279                not_found_operation_id: not_found,
1280            },
1281            "ResolveWaitAllAdmission",
1282        )?;
1283        if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1284        {
1285            return Err(error);
1286        }
1287        Ok(())
1288    }
1289
1290    fn try_satisfy_wait_all_authority(
1291        &mut self,
1292    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1293        let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1294            return Ok(None);
1295        };
1296        let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1297            return Err(OpsLifecycleError::Internal(
1298                "generated wait_all authority has active wait without run id".into(),
1299            ));
1300        };
1301        let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1302        let transition = match mm_dsl::MeerkatMachineMutator::apply(
1303            &mut *self.dsl.0,
1304            mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1305                wait_request_id: dsl_wait_request_id,
1306                run_id: dsl_run_id,
1307                operation_id_tokens: dsl_operation_id_tokens,
1308            },
1309        ) {
1310            Ok(transition) => transition,
1311            Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1312            Err(err) => {
1313                return Err(OpsLifecycleError::Internal(format!(
1314                    "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1315                )));
1316            }
1317        };
1318        Self::wait_all_satisfied_from_effects(transition.effects())?
1319            .ok_or_else(|| {
1320                OpsLifecycleError::Internal(
1321                    "generated wait_all authority accepted satisfaction without effect".into(),
1322                )
1323            })
1324            .map(Some)
1325    }
1326
1327    fn begin_wait_all_authority(
1328        &mut self,
1329        run_id: &RunId,
1330        wait_request_id: &WaitRequestId,
1331        operation_ids: &[OperationId],
1332    ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1333        let mut dsl_ids = BTreeSet::new();
1334        let mut dsl_id_tokens = BTreeSet::new();
1335        let mut operation_token_by_id = BTreeMap::new();
1336        let mut operation_id_by_token = BTreeMap::new();
1337        for id in operation_ids {
1338            let token = mm_dsl::OperationId::from_domain(id);
1339            let raw_id = token.0.clone();
1340            dsl_ids.insert(raw_id.clone());
1341            dsl_id_tokens.insert(token.clone());
1342            operation_token_by_id.insert(raw_id.clone(), token.clone());
1343            operation_id_by_token.insert(token, raw_id);
1344        }
1345        self.resolve_wait_all_admission(
1346            wait_request_id,
1347            operation_ids,
1348            &dsl_ids,
1349            &dsl_id_tokens,
1350            &operation_token_by_id,
1351            &operation_id_by_token,
1352        )?;
1353        self.dsl_apply(
1354            mm_dsl::MeerkatMachineInput::RequestWaitAll {
1355                run_id: mm_dsl::RunId::from_domain(run_id),
1356                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1357                operation_id_sequence: operation_ids
1358                    .iter()
1359                    .map(|id| mm_dsl::OperationId::from_domain(id).0)
1360                    .collect(),
1361                operation_ids: dsl_ids,
1362                operation_id_tokens: dsl_id_tokens,
1363                operation_token_by_id,
1364                operation_id_by_token,
1365            },
1366            "RequestWaitAll",
1367        )?;
1368        if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1369            return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1370        }
1371        Ok(WaitAllAuthorityPlan::ActivateBarrier)
1372    }
1373
1374    fn owner_termination_targets(
1375        &self,
1376    ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1377        let mut targets = Vec::new();
1378        for id in self.operation_ids()? {
1379            let status = self.require_status(&id)?;
1380            if !Self::operation_status_is_terminal(&id, status)? {
1381                targets.push((id, status));
1382            }
1383        }
1384        Ok(targets)
1385    }
1386
1387    fn operation_status_is_terminal(
1388        operation_id: &OperationId,
1389        status: OperationStatus,
1390    ) -> Result<bool, OpsLifecycleError> {
1391        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1392        let effects = Self::apply_stateless_classifier(
1393            mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1394                operation_id: operation_id_key.clone(),
1395                status: mm_dsl::OperationStatus::from(status),
1396            },
1397            "ClassifyOperationTerminality",
1398        )?;
1399        let mut terminal = None;
1400        for effect in effects {
1401            match effect {
1402                mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1403                    if operation_id == operation_id_key =>
1404                {
1405                    terminal = Some(true);
1406                }
1407                mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1408                    if operation_id == operation_id_key =>
1409                {
1410                    terminal = Some(false);
1411                }
1412                other => {
1413                    return Err(OpsLifecycleError::Internal(format!(
1414                        "unexpected generated operation terminality effect: {other:?}"
1415                    )));
1416                }
1417            }
1418        }
1419        terminal.ok_or_else(|| {
1420            OpsLifecycleError::Internal(format!(
1421                "generated operation terminality authority emitted no effect for {operation_id}"
1422            ))
1423        })
1424    }
1425
1426    fn operation_public_result_class(
1427        operation_id: &OperationId,
1428        status: OperationStatus,
1429    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1430        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1431        let effects = Self::apply_stateless_classifier(
1432            mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1433                operation_id: operation_id_key.clone(),
1434                status: mm_dsl::OperationStatus::from(status),
1435            },
1436            "ClassifyOperationPublicResult",
1437        )?;
1438        let mut result = None;
1439        for effect in effects {
1440            match effect {
1441                mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1442                    operation_id,
1443                    result: classified,
1444                } if operation_id == operation_id_key => {
1445                    result = Some(OperationPublicResultClass::from(classified));
1446                }
1447                other => {
1448                    return Err(OpsLifecycleError::Internal(format!(
1449                        "unexpected generated operation public-result effect: {other:?}"
1450                    )));
1451                }
1452            }
1453        }
1454        result.ok_or_else(|| {
1455            OpsLifecycleError::Internal(format!(
1456                "generated operation public-result authority emitted no effect for {operation_id}"
1457            ))
1458        })
1459    }
1460
1461    fn operation_transition_rejection_is_idempotent(
1462        operation_id: &OperationId,
1463        action: OperationLifecycleAction,
1464        status: OperationStatus,
1465    ) -> Result<bool, OpsLifecycleError> {
1466        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1467        let action = mm_dsl::OpLifecycleActionKind::from(action);
1468        let status = mm_dsl::OperationStatus::from(status);
1469        let effects = Self::apply_stateless_classifier(
1470            mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1471                operation_id: operation_id_key.clone(),
1472                action,
1473                status,
1474            },
1475            "ClassifyOperationTransitionIdempotence",
1476        )?;
1477        let mut idempotent = None;
1478        for effect in effects {
1479            match effect {
1480                mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1481                    operation_id,
1482                    action: effect_action,
1483                    status: effect_status,
1484                } if operation_id == operation_id_key
1485                    && effect_action == action
1486                    && effect_status == status =>
1487                {
1488                    idempotent = Some(true);
1489                }
1490                mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1491                    operation_id,
1492                    action: effect_action,
1493                    status: effect_status,
1494                } if operation_id == operation_id_key
1495                    && effect_action == action
1496                    && effect_status == status =>
1497                {
1498                    idempotent = Some(false);
1499                }
1500                other => {
1501                    return Err(OpsLifecycleError::Internal(format!(
1502                        "unexpected generated operation transition-idempotence effect: {other:?}"
1503                    )));
1504                }
1505            }
1506        }
1507        idempotent.ok_or_else(|| {
1508            OpsLifecycleError::Internal(format!(
1509                "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1510            ))
1511        })
1512    }
1513
1514    fn operation_completion_feed_class(
1515        operation_id: &OperationId,
1516        kind: OperationKind,
1517    ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1518        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1519        let kind = mm_dsl::OperationKind::from(kind);
1520        let effects = Self::apply_stateless_classifier(
1521            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1522                operation_id: operation_id_key.clone(),
1523                kind,
1524            },
1525            "ClassifyOperationCompletionFeed",
1526        )?;
1527        let mut class = None;
1528        for effect in effects {
1529            match effect {
1530                mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1531                    operation_id,
1532                    result,
1533                } if operation_id == operation_id_key => {
1534                    class = Some(result);
1535                }
1536                other => {
1537                    return Err(OpsLifecycleError::Internal(format!(
1538                        "unexpected generated operation completion-feed effect: {other:?}"
1539                    )));
1540                }
1541            }
1542        }
1543        class.ok_or_else(|| {
1544            OpsLifecycleError::Internal(format!(
1545                "generated operation completion-feed authority emitted no effect for {operation_id}"
1546            ))
1547        })
1548    }
1549
1550    fn operation_completion_wake_class(
1551        operation_id: &OperationId,
1552        kind: OperationKind,
1553    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1554        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1555        let kind = mm_dsl::OperationKind::from(kind);
1556        let effects = Self::apply_stateless_classifier(
1557            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1558                operation_id: operation_id_key.clone(),
1559                kind,
1560            },
1561            "ClassifyOperationCompletionWake",
1562        )?;
1563        let mut class = None;
1564        for effect in effects {
1565            match effect {
1566                mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1567                    operation_id,
1568                    result,
1569                } if operation_id == operation_id_key => {
1570                    class = Some(OperationCompletionWakeClass::from(result));
1571                }
1572                other => {
1573                    return Err(OpsLifecycleError::Internal(format!(
1574                        "unexpected generated operation completion-wake effect: {other:?}"
1575                    )));
1576                }
1577            }
1578        }
1579        class.ok_or_else(|| {
1580            OpsLifecycleError::Internal(format!(
1581                "generated operation completion-wake authority emitted no effect for {operation_id}"
1582            ))
1583        })
1584    }
1585
1586    fn operation_durability_class(
1587        operation_id: &OperationId,
1588        kind: OperationKind,
1589    ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1590        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1591        let kind = mm_dsl::OperationKind::from(kind);
1592        let effects = Self::apply_stateless_classifier(
1593            mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1594                operation_id: operation_id_key.clone(),
1595                kind,
1596            },
1597            "ClassifyOperationDurability",
1598        )?;
1599        let mut class = None;
1600        for effect in effects {
1601            match effect {
1602                mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1603                    operation_id,
1604                    result,
1605                } if operation_id == operation_id_key => {
1606                    class = Some(result);
1607                }
1608                other => {
1609                    return Err(OpsLifecycleError::Internal(format!(
1610                        "unexpected generated operation durability effect: {other:?}"
1611                    )));
1612                }
1613            }
1614        }
1615        class.ok_or_else(|| {
1616            OpsLifecycleError::Internal(format!(
1617                "generated operation durability authority emitted no effect for {operation_id}"
1618            ))
1619        })
1620    }
1621
1622    fn recovered_operation_record_disposition(
1623        operation_id: &OperationId,
1624        status: OperationStatus,
1625        kind: OperationKind,
1626        terminal_outcome_present: bool,
1627        terminal_payload_present: bool,
1628        completion_sequence_present: bool,
1629    ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1630        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1631        let effects = Self::apply_stateless_classifier(
1632            mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1633                operation_id: operation_id_key.clone(),
1634                status: mm_dsl::OperationStatus::from(status),
1635                kind: mm_dsl::OperationKind::from(kind),
1636                terminal_outcome_present,
1637                terminal_payload_present,
1638                completion_sequence_present,
1639            },
1640            "ClassifyRecoveredOperationRecord",
1641        )?;
1642        let mut disposition = None;
1643        for effect in effects {
1644            match effect {
1645                mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1646                    if operation_id == operation_id_key =>
1647                {
1648                    disposition = Some(RecoveredOperationRecordDisposition::Retain);
1649                }
1650                mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1651                    if operation_id == operation_id_key =>
1652                {
1653                    disposition = Some(RecoveredOperationRecordDisposition::Discard);
1654                }
1655                other => {
1656                    return Err(OpsLifecycleError::Internal(format!(
1657                        "unexpected generated recovered-operation classification effect: {other:?}"
1658                    )));
1659                }
1660            }
1661        }
1662        disposition.ok_or_else(|| {
1663            OpsLifecycleError::Internal(format!(
1664                "generated recovered-operation classifier emitted no effect for {operation_id}"
1665            ))
1666        })
1667    }
1668
1669    fn apply_stateless_classifier(
1670        input: mm_dsl::MeerkatMachineInput,
1671        label: &'static str,
1672    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1673        let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1674            "ops stateless classifier Initialize must be accepted",
1675        );
1676        let transition =
1677            mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1678                OpsLifecycleError::Internal(format!(
1679                    "DSL rejected ops transition ({label}): {err:?}"
1680                ))
1681            })?;
1682        Ok(transition.into_effects())
1683    }
1684
1685    /// Check whether a pending barrier wait is now satisfied and resolve it.
1686    ///
1687    /// Barrier membership and the "all members terminal" decision both live
1688    /// in the DSL: `wait_operation_ids` carries the set, `wait_active`
1689    /// signals a pending barrier, and `SatisfyWaitAll`'s
1690    /// `all_members_terminal` guard owns the fixed-point test. The shell
1691    /// echoes the DSL-owned request id and typed operation tokens into
1692    /// `SatisfyWaitAll` so the transition can clear the barrier before
1693    /// rendering the handoff effect. It still fires idempotently on every
1694    /// terminal transition and lets the DSL guard reject early firings as a
1695    /// no-op. On acceptance (transition returns `Ok`), the shell selects the
1696    /// correlated oneshot and delivers the `WaitAllSatisfied` obligation
1697    /// token.
1698    ///
1699    /// `wait_request_id` is the shell-owned oneshot correlation id that
1700    /// selects which sender to notify; when the DSL barrier satisfies
1701    /// without a live correlation (post-recovery, or duplicate resolution),
1702    /// the oneshot simply remains pending. That benign no-correlation case is
1703    /// the only path that leaves the oneshot pending: on authority-invariant
1704    /// corruption the pending sender is dropped so the waiter's
1705    /// `WaitAllFuture` resolves to `Err` and the typed
1706    /// [`OpsLifecycleError`] propagates to the terminal transition caller
1707    /// instead of reporting the op complete with a silently-hung barrier.
1708    fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1709        let satisfied = match self.try_satisfy_wait_all_authority() {
1710            Ok(Some(satisfied)) => satisfied,
1711            Ok(None) => return Ok(()),
1712            Err(err) => {
1713                // Authority-invariant corruption: do NOT report the op complete
1714                // with a silently-pending barrier oneshot. Drop the sender so
1715                // the waiter's `WaitAllFuture` resolves to `Err` via its
1716                // `Poll::Ready(Err(_))` arm (the same mechanism
1717                // `cancel_wait_all_internal` uses), then propagate the typed
1718                // fault. The invariant is already broken, so we do not attempt
1719                // a `CancelWaitAll` rollback on the corrupt machine.
1720                if let Some(pending) = self.pending_wait.take() {
1721                    drop(pending.sender);
1722                }
1723                self.wait_request_id = None;
1724                return Err(err);
1725            }
1726        };
1727        let shell_wait_id = self.wait_request_id.take();
1728        if shell_wait_id
1729            .as_ref()
1730            .is_some_and(|id| id != &satisfied.wait_request_id)
1731        {
1732            tracing::error!(
1733                shell_wait_request_id = ?shell_wait_id,
1734                authority_wait_request_id = %satisfied.wait_request_id,
1735                "generated wait_all authority satisfied a different wait request"
1736            );
1737        }
1738        if let Some(pending) = self.pending_wait.take() {
1739            if pending.wait_request_id == satisfied.wait_request_id {
1740                let _ = pending.sender.send(satisfied);
1741            } else if let Some(shell_wait_id) = shell_wait_id {
1742                tracing::error!(
1743                    shell_wait_request_id = %shell_wait_id,
1744                    pending_wait_request_id = %pending.wait_request_id,
1745                    authority_wait_request_id = %satisfied.wait_request_id,
1746                    "generated wait_all authority satisfied without a matching pending waiter"
1747                );
1748            }
1749        }
1750        Ok(())
1751    }
1752
1753    /// Persist a terminal snapshot if a persistence channel is wired.
1754    ///
1755    /// Called after terminal transitions. Captures authority + entries + cursors under the write
1756    /// lock (caller already holds it), submits a persistence request, and waits for the worker's
1757    /// durable-store result. Returning success only means the snapshot write itself succeeded.
1758    fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1759        let (tx, epoch_id, cursor_state) = match (
1760            &self.persist_tx,
1761            &self.persist_epoch_id,
1762            &self.persist_cursor_state,
1763        ) {
1764            (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1765            _ => return Ok(()),
1766        };
1767
1768        let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1769        let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1770        let request = OpsLifecyclePersistenceRequest {
1771            snapshot,
1772            result_tx,
1773        };
1774
1775        tx.send(request).map_err(|_| {
1776            OpsLifecycleError::Internal(
1777                "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1778                    .into(),
1779            )
1780        })?;
1781        result_rx.recv().map_err(|_| {
1782            OpsLifecycleError::Internal(
1783                "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1784                    .into(),
1785            )
1786        })?
1787    }
1788
1789    /// Capture the full persisted snapshot for the current state.
1790    fn capture_snapshot(
1791        &self,
1792        epoch_id: meerkat_core::RuntimeEpochId,
1793        _cursor_state: &meerkat_core::EpochCursorState,
1794    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1795        let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1796        for op_id in self.operation_ids()? {
1797            let status = self.require_status(&op_id)?;
1798            let kind = self.require_kind(&op_id)?;
1799            if Self::operation_durability_class(&op_id, kind)?
1800                != mm_dsl::OperationDurabilityClass::Retain
1801            {
1802                continue;
1803            }
1804            let peer_ready = self.require_peer_ready(&op_id)?;
1805            let progress_count = self.require_progress_count(&op_id)?;
1806            let operation_source = self.operation_source(&op_id)?;
1807            let terminal_outcome = self.terminal_outcome(&op_id)?;
1808            let completion_sequence = self.completion_sequence(&op_id);
1809            if terminal_outcome.is_some() && completion_sequence.is_none() {
1810                return Err(OpsLifecycleError::Internal(format!(
1811                    "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1812                )));
1813            }
1814            if terminal_outcome.is_none() && completion_sequence.is_some() {
1815                return Err(OpsLifecycleError::Internal(format!(
1816                    "generated op terminal authority has completion sequence for non-terminal {op_id}"
1817                )));
1818            }
1819            let terminal_buffered = terminal_outcome.is_some();
1820            let watcher_count = self
1821                .records
1822                .get(&op_id)
1823                .map(|r| r.watchers.len() as u32)
1824                .unwrap_or(0);
1825            operations.insert(
1826                op_id,
1827                OperationCanonicalState {
1828                    status,
1829                    kind,
1830                    operation_source,
1831                    peer_ready,
1832                    progress_count,
1833                    watcher_count,
1834                    terminal_outcome,
1835                    completion_sequence,
1836                    terminal_buffered,
1837                },
1838            );
1839        }
1840        let operation_specs: HashMap<OperationId, OperationSpec> = self
1841            .records
1842            .iter()
1843            .filter(|(id, _)| operations.contains_key(*id))
1844            .map(|(id, record)| {
1845                let mut spec = record.spec.clone();
1846                let operation_source = operations
1847                    .get(id)
1848                    .and_then(|state| state.operation_source.as_ref());
1849                Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1850                (id.clone(), spec)
1851            })
1852            .collect();
1853        let completed_order: VecDeque<OperationId> = self
1854            .completed_order
1855            .iter()
1856            .filter(|id| operations.contains_key(*id))
1857            .cloned()
1858            .collect();
1859        let active_count = operations
1860            .iter()
1861            .filter(|(id, state)| {
1862                matches!(
1863                    Self::operation_status_is_terminal(id, state.status),
1864                    Ok(false)
1865                )
1866            })
1867            .count();
1868        let authority_completion_entries = self.completion_feed_authority_entries()?;
1869        let published_completion_entries_by_id: HashMap<OperationId, CompletionEntry> = {
1870            let inner = self
1871                .feed_buffer
1872                .inner
1873                .read()
1874                .unwrap_or_else(std::sync::PoisonError::into_inner);
1875            let mut entries = HashMap::new();
1876            for entry in &inner.entries {
1877                if !authority_completion_entries.contains_key(&entry.operation_id) {
1878                    return Err(OpsLifecycleError::Internal(format!(
1879                        "public completion feed projection for {} has no generated authority",
1880                        entry.operation_id
1881                    )));
1882                }
1883                if entries
1884                    .insert(entry.operation_id.clone(), entry.clone())
1885                    .is_some()
1886                {
1887                    return Err(OpsLifecycleError::Internal(format!(
1888                        "public completion feed projection for {} appeared more than once",
1889                        entry.operation_id
1890                    )));
1891                }
1892            }
1893            entries
1894        };
1895        let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
1896            .iter()
1897            .map(|(operation_id, authority_entry)| {
1898                if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
1899                    if projection.seq != authority_entry.seq
1900                        || projection.kind != authority_entry.kind
1901                        || projection.terminal_outcome != authority_entry.terminal_outcome
1902                    {
1903                        return Err(OpsLifecycleError::Internal(format!(
1904                            "public completion feed projection for {operation_id} drifted from generated authority"
1905                        )));
1906                    }
1907                    return Ok(projection.clone());
1908                }
1909
1910                let display_name = self
1911                    .records
1912                    .get(operation_id)
1913                    .map(|record| record.spec.display_name.clone())
1914                    .unwrap_or_default();
1915                let completed_at_ms = self.records.get(operation_id).and_then(|record| {
1916                    record
1917                        .completed_at
1918                        .map(|completed_at| record.epoch_millis_for_instant(completed_at))
1919                });
1920
1921                Ok(CompletionEntry {
1922                    seq: authority_entry.seq,
1923                    operation_id: operation_id.clone(),
1924                    kind: authority_entry.kind,
1925                    display_name,
1926                    terminal_outcome: authority_entry.terminal_outcome.clone(),
1927                    completed_at_ms,
1928                })
1929            })
1930            .collect::<Result<_, _>>()?;
1931        completion_entries.sort_by_key(|entry| entry.seq);
1932
1933        let authority_state = RegistryCanonicalState {
1934            operations,
1935            completion_feed_entries: authority_completion_entries,
1936            completed_order,
1937            max_completed: self.max_completed,
1938            max_concurrent: self.max_concurrent,
1939            active_count,
1940            wait_request_id: self.wait_request_id.clone(),
1941            wait_operation_ids: self.wait_operation_ids()?,
1942            next_completion_seq: self.dsl.0.state().next_completion_seq,
1943        };
1944
1945        Ok(PersistedOpsSnapshot {
1946            epoch_id,
1947            authority_state,
1948            operation_specs,
1949            completion_entries,
1950            cursors: self.completion_cursor_snapshot(),
1951        })
1952    }
1953
1954    fn shell_record_mut(
1955        &mut self,
1956        id: &OperationId,
1957    ) -> Result<&mut ShellRecord, OpsLifecycleError> {
1958        self.records
1959            .get_mut(id)
1960            .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
1961    }
1962
1963    fn collect_wait_outcomes(
1964        &self,
1965        operation_ids: &[OperationId],
1966    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
1967        operation_ids
1968            .iter()
1969            .map(|operation_id| {
1970                let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
1971                    OpsLifecycleError::Internal(format!(
1972                        "wait_all completed without terminal outcome for {operation_id}"
1973                    ))
1974                })?;
1975                Ok((operation_id.clone(), outcome))
1976            })
1977            .collect()
1978    }
1979}
1980
1981impl Default for ShellState {
1982    fn default() -> Self {
1983        Self::new(DEFAULT_MAX_COMPLETED, None)
1984    }
1985}
1986
1987// ---------------------------------------------------------------------------
1988// Public configuration & registry
1989// ---------------------------------------------------------------------------
1990
1991/// Configuration for [`RuntimeOpsLifecycleRegistry`].
1992#[derive(Debug, Clone)]
1993pub struct OpsLifecycleConfig {
1994    /// Maximum number of completed operations to retain (default: 256).
1995    pub max_completed: usize,
1996    /// Maximum concurrent non-terminal operations (None = unlimited).
1997    pub max_concurrent: Option<usize>,
1998}
1999
2000impl Default for OpsLifecycleConfig {
2001    fn default() -> Self {
2002        Self {
2003            max_completed: DEFAULT_MAX_COMPLETED,
2004            max_concurrent: None,
2005        }
2006    }
2007}
2008
2009/// Per-runtime shared registry for async operation lifecycle truth.
2010///
2011/// Per-operation canonical lifecycle state is owned by the DSL authority
2012/// embedded in the shell. This struct manages I/O concerns: watcher
2013/// channels, timestamps, peer handles, snapshot assembly, FIFO eviction,
2014/// and the completion feed buffer.
2015#[derive(Debug)]
2016pub struct RuntimeOpsLifecycleRegistry {
2017    state: RwLock<ShellState>,
2018}
2019
2020#[derive(Debug, Clone)]
2021pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2022    pub operation_count: usize,
2023    pub active_count: usize,
2024    pub wait_request_id: Option<WaitRequestId>,
2025    pub pending_wait_present: bool,
2026    pub pending_wait_request_id: Option<WaitRequestId>,
2027    pub wait_operation_ids: Vec<OperationId>,
2028    pub operations: Vec<OperationLifecycleSnapshot>,
2029}
2030
2031impl Default for RuntimeOpsLifecycleRegistry {
2032    fn default() -> Self {
2033        Self {
2034            state: RwLock::new(ShellState::default()),
2035        }
2036    }
2037}
2038
2039impl RuntimeOpsLifecycleRegistry {
2040    pub fn new() -> Self {
2041        let dsl = new_ops_dsl_authority();
2042        let feed_capacity = DEFAULT_MAX_COMPLETED.saturating_mul(4).max(1024);
2043        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2044        Self {
2045            state: RwLock::new(ShellState {
2046                dsl,
2047                records: HashMap::new(),
2048                pending_wait: None,
2049                completed_order: VecDeque::new(),
2050                max_completed: DEFAULT_MAX_COMPLETED,
2051                max_concurrent: None,
2052                wait_request_id: None,
2053                feed_buffer,
2054                persist_tx: None,
2055                persistence_sealed: false,
2056                persist_epoch_id: None,
2057                persist_cursor_state: None,
2058                owner_retired: false,
2059            }),
2060        }
2061    }
2062
2063    pub fn with_config(config: OpsLifecycleConfig) -> Self {
2064        Self {
2065            state: RwLock::new(ShellState::new(config.max_completed, config.max_concurrent)),
2066        }
2067    }
2068
2069    fn recover_completion_feed_entry(
2070        shell: &mut ShellState,
2071        operation_id: &OperationId,
2072        entry: &CompletionFeedCanonicalState,
2073    ) -> Result<(), OpsLifecycleError> {
2074        let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2075        let terminal_outcome_kind =
2076            mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2077        let effects = shell.dsl_apply_with_effects(
2078            mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2079                operation_id: expected_operation_id.clone(),
2080                kind: mm_dsl::OperationKind::from(entry.kind),
2081                terminal_outcome: terminal_outcome_kind,
2082                terminal_payload: entry.terminal_outcome.clone(),
2083                completion_sequence: entry.seq,
2084            },
2085            "RecoverCompletionFeedEntry",
2086        )?;
2087        let recovered = effects.iter().find_map(|effect| match effect {
2088            mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2089                operation_id,
2090                seq,
2091                kind,
2092                terminal_outcome,
2093                terminal_payload,
2094            } => Some((
2095                operation_id,
2096                *seq,
2097                OperationKind::from(*kind),
2098                *terminal_outcome,
2099                terminal_payload,
2100            )),
2101            _ => None,
2102        });
2103        let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2104            return Err(OpsLifecycleError::Internal(
2105                "generated completion-feed recovery emitted no recovered entry".into(),
2106            ));
2107        };
2108        if operation_id != &expected_operation_id
2109            || seq != entry.seq
2110            || kind != entry.kind
2111            || terminal_outcome != terminal_outcome_kind
2112            || terminal_payload != &entry.terminal_outcome
2113        {
2114            return Err(OpsLifecycleError::Internal(format!(
2115                "generated completion-feed recovery drifted for {operation_id}"
2116            )));
2117        }
2118        Ok(())
2119    }
2120
2121    /// Wire a persistence channel for durable snapshot writes.
2122    ///
2123    /// After this call, terminal transitions (complete/fail/cancel/abort)
2124    /// capture a snapshot and queue it to the channel. A dedicated
2125    /// persistence task should drain the channel and write to the store.
2126    pub fn set_persistence_channel(
2127        &self,
2128        tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2129        epoch_id: meerkat_core::RuntimeEpochId,
2130        cursor_state: Arc<meerkat_core::EpochCursorState>,
2131    ) {
2132        if let Ok(mut state) = self.state.write() {
2133            if state.owner_retired || state.persistence_sealed {
2134                tracing::warn!(
2135                    "ignored attempt to rewire a terminally sealed ops persistence channel"
2136                );
2137                return;
2138            }
2139            state.persist_tx = Some(tx);
2140            state.persist_epoch_id = Some(epoch_id);
2141            state.persist_cursor_state = Some(cursor_state);
2142        }
2143    }
2144
2145    /// Terminalize every live operation, persist those generated transitions,
2146    /// then close both lifecycle admission and the persistence producer.
2147    ///
2148    /// The write lock excludes every callback that retained this registry.
2149    /// Each terminal transition waits for its durable worker acknowledgement;
2150    /// taking the sender afterward therefore leaves no queued write behind.
2151    /// The unregister coordinator must join the owned worker before publishing
2152    /// final lifecycle deletion in the RuntimeStore.
2153    pub(crate) fn retire_owner_for_unregister(
2154        &self,
2155        reason: String,
2156    ) -> Result<(), OpsLifecycleError> {
2157        let closed_sender = {
2158            let mut state = self.write_state()?;
2159            if state.owner_retired {
2160                return Ok(());
2161            }
2162            terminate_owner_locked(&mut state, &reason)?;
2163            state.owner_retired = true;
2164            state.persistence_sealed = true;
2165            state.persist_epoch_id = None;
2166            state.persist_cursor_state = None;
2167            state.persist_tx.take()
2168        };
2169        drop(closed_sender);
2170        Ok(())
2171    }
2172
2173    /// Recover from a persisted snapshot.
2174    ///
2175    /// Rebuilds DSL state, retaining terminal operations and running
2176    /// `DetachedJobWait` bindings while stripping every other non-terminal
2177    /// operation. Fresh shell records come from specs; completion entries are
2178    /// seeded only after generated recovery authority accepts them.
2179    pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2180        let PersistedOpsSnapshot {
2181            authority_state,
2182            operation_specs,
2183            completion_entries,
2184            cursors,
2185            ..
2186        } = snapshot;
2187        let max_completed = authority_state.max_completed;
2188        let max_concurrent = authority_state.max_concurrent;
2189        let next_completion_seq = authority_state.next_completion_seq;
2190        let authority_completion_entries = authority_state.completion_feed_entries;
2191        let authority_operations = authority_state.operations;
2192        let mut shell = ShellState::new(max_completed, max_concurrent);
2193
2194        // Replay every persisted op through generated recovery authority.
2195        // The transition accepts terminal records with outcome/sequence
2196        // witnesses plus running DetachedJobWait records with their typed
2197        // source. Other non-terminal rows remain volatile.
2198        let mut retained_ids: HashSet<OperationId> = HashSet::new();
2199        for (op_id, op_state) in authority_operations {
2200            let terminal_outcome = op_state
2201                .terminal_outcome
2202                .as_ref()
2203                .map(mm_dsl::OperationTerminalOutcomeKind::from);
2204            let terminal_payload = op_state.terminal_outcome.clone();
2205            let disposition = ShellState::recovered_operation_record_disposition(
2206                &op_id,
2207                op_state.status,
2208                op_state.kind,
2209                terminal_outcome.is_some(),
2210                terminal_payload.is_some(),
2211                op_state.completion_sequence.is_some(),
2212            )?;
2213            if disposition == RecoveredOperationRecordDisposition::Discard {
2214                continue;
2215            }
2216            if let Some(spec_source) = operation_specs
2217                .get(&op_id)
2218                .and_then(|spec| spec.operation_source.as_ref())
2219                && op_state.operation_source.as_ref() != Some(spec_source)
2220            {
2221                return Err(OpsLifecycleError::Internal(format!(
2222                    "persisted operation source mirror for {op_id} drifted from generated authority"
2223                )));
2224            }
2225            let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2226                operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2227                status: mm_dsl::OperationStatus::from(op_state.status),
2228                kind: mm_dsl::OperationKind::from(op_state.kind),
2229                source: op_state
2230                    .operation_source
2231                    .as_ref()
2232                    .map(mm_dsl::OperationSource::from_domain),
2233                peer_ready: op_state.peer_ready,
2234                progress_count: u64::from(op_state.progress_count),
2235                terminal_outcome,
2236                terminal_payload,
2237                completion_sequence: op_state.completion_sequence,
2238            };
2239            shell.dsl_apply(recovery, "RecoverOpRecord")?;
2240            let recovered_seq = shell.completion_sequence(&op_id);
2241            if op_state.completion_sequence != recovered_seq {
2242                return Err(OpsLifecycleError::Internal(format!(
2243                    "generated op recovery completion sequence mismatch for {op_id}"
2244                )));
2245            }
2246            retained_ids.insert(op_id);
2247        }
2248        shell.dsl_apply(
2249            mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2250                next_completion_seq,
2251            },
2252            "RecoverOpsCompletionCursor",
2253        )?;
2254        shell.dsl_apply(
2255            mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2256                agent_applied_cursor: cursors.agent_applied_cursor,
2257                runtime_observed_cursor: cursors.runtime_observed_seq,
2258                runtime_injected_cursor: cursors.runtime_last_injected_seq,
2259            },
2260            "RecoverCompletionConsumerCursors",
2261        )?;
2262
2263        // Rebuild completed_order from generated completion-sequence truth,
2264        // never from the persisted shell ordering mirror.
2265        let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2266            .iter()
2267            .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2268            .collect();
2269        recovered_completed.sort_by_key(|(seq, _)| *seq);
2270        shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2271
2272        // Recover generated-owned feed authority for entries whose operation
2273        // record is no longer retained. Retained records already wrote their
2274        // feed authority through RecoverOpRecord above.
2275        for (operation_id, entry) in &authority_completion_entries {
2276            if !retained_ids.contains(operation_id) {
2277                Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2278            }
2279        }
2280
2281        let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2282        for (operation_id, entry) in &authority_completion_entries {
2283            let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2284                return Err(OpsLifecycleError::Internal(format!(
2285                    "persisted completion feed authority for {operation_id} was not recovered"
2286                )));
2287            };
2288            if recovered_entry != entry {
2289                return Err(OpsLifecycleError::Internal(format!(
2290                    "persisted completion feed authority drifted from generated recovery for {operation_id}"
2291                )));
2292            }
2293        }
2294
2295        // Projection rows may carry display metadata only after generated
2296        // feed authority has decided the operation id, sequence, kind, and
2297        // terminal outcome. Any semantic drift fails closed.
2298        let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2299        for entry in completion_entries {
2300            let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2301                return Err(OpsLifecycleError::Internal(format!(
2302                    "persisted completion feed projection for {} has no generated feed authority",
2303                    entry.operation_id
2304                )));
2305            };
2306            if authority_entry.seq != entry.seq
2307                || authority_entry.kind != entry.kind
2308                || authority_entry.terminal_outcome != entry.terminal_outcome
2309            {
2310                return Err(OpsLifecycleError::Internal(format!(
2311                    "persisted completion feed projection for {} drifted from generated feed authority",
2312                    entry.operation_id
2313                )));
2314            }
2315            projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2316        }
2317
2318        let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2319            canonical_feed_entries.into_iter().collect();
2320        recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2321        for (operation_id, entry) in recovered_entries {
2322            let projection = projection_entries_by_id.get(&operation_id);
2323            let display_name = operation_specs
2324                .get(&operation_id)
2325                .map(|spec| spec.display_name.clone())
2326                .or_else(|| projection.map(|entry| entry.display_name.clone()))
2327                .unwrap_or_default();
2328            let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2329            shell.feed_buffer.push(CompletionEntry {
2330                seq: entry.seq,
2331                operation_id,
2332                kind: entry.kind,
2333                display_name,
2334                terminal_outcome: entry.terminal_outcome,
2335                completed_at_ms,
2336            });
2337        }
2338
2339        // Rebuild shell records from specs (fresh timestamps, no watchers)
2340        // — only for operations still retained in the DSL state.
2341        for (op_id, spec) in operation_specs {
2342            if retained_ids.contains(&op_id) {
2343                let mut spec = spec;
2344                let operation_source = shell.operation_source(&op_id)?;
2345                ShellState::align_spec_child_session_id_to_source(
2346                    &mut spec,
2347                    operation_source.as_ref(),
2348                );
2349                shell.records.insert(
2350                    op_id,
2351                    ShellRecord {
2352                        spec,
2353                        peer_handle: None,
2354                        watchers: Vec::new(),
2355                        created_at: Instant::now(),
2356                        started_at: None,
2357                        completed_at: None,
2358                        created_at_wall: SystemTime::now(),
2359                    },
2360                );
2361            }
2362        }
2363
2364        Ok(Self {
2365            state: RwLock::new(shell),
2366        })
2367    }
2368
2369    /// Capture a serializable snapshot of the current state for persistence.
2370    ///
2371    /// Includes authority state, operation specs, completion entries, and
2372    /// generated completion-consumer cursor values.
2373    pub fn capture_persistence_snapshot(
2374        &self,
2375        epoch_id: meerkat_core::RuntimeEpochId,
2376        cursor_state: &meerkat_core::EpochCursorState,
2377    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2378        let state = self
2379            .state
2380            .read()
2381            .unwrap_or_else(std::sync::PoisonError::into_inner);
2382        state.capture_snapshot(epoch_id, cursor_state)
2383    }
2384
2385    /// Snapshot generated completion-consumer cursor state.
2386    pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2387        let state = self
2388            .state
2389            .read()
2390            .unwrap_or_else(std::sync::PoisonError::into_inner);
2391        state.completion_cursor_snapshot()
2392    }
2393
2394    /// Return a read handle to the completion feed.
2395    pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2396        let state = self
2397            .state
2398            .read()
2399            .unwrap_or_else(std::sync::PoisonError::into_inner);
2400        Arc::new(RuntimeCompletionFeed {
2401            buffer: Arc::clone(&state.feed_buffer),
2402        })
2403    }
2404
2405    /// Capture a stable diagnostic snapshot of the canonical ops lifecycle state.
2406    pub(crate) fn diagnostic_snapshot(
2407        &self,
2408    ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2409        let state = self
2410            .state
2411            .read()
2412            .unwrap_or_else(std::sync::PoisonError::into_inner);
2413        let mut operations = state
2414            .operation_ids()?
2415            .into_iter()
2416            .map(|id| state.snapshot(&id))
2417            .collect::<Result<Vec<_>, _>>()?
2418            .into_iter()
2419            .flatten()
2420            .collect::<Vec<_>>();
2421        operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2422        Ok(RuntimeOpsDiagnosticSnapshot {
2423            operation_count: state.operation_count(),
2424            active_count: state.active_count(),
2425            wait_request_id: state.wait_request_id.clone(),
2426            pending_wait_present: state.pending_wait.is_some(),
2427            pending_wait_request_id: state
2428                .pending_wait
2429                .as_ref()
2430                .map(|pending_wait| pending_wait.wait_request_id.clone()),
2431            wait_operation_ids: state.wait_operation_ids()?,
2432            operations,
2433        })
2434    }
2435
2436    fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2437        self.state
2438            .read()
2439            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2440    }
2441
2442    fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2443        self.state
2444            .write()
2445            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2446    }
2447
2448    fn cancel_wait_all_internal(
2449        &self,
2450        wait_request_id: &WaitRequestId,
2451    ) -> Result<(), OpsLifecycleError> {
2452        let mut state = self.write_state()?;
2453        match state.wait_request_id.as_ref() {
2454            Some(active) if active == wait_request_id => {
2455                // Clear the DSL barrier via the dedicated `CancelWaitAll`
2456                // transition. Unlike `SatisfyWaitAll`, it does not require
2457                // every member to be terminal (the request was dropped, not
2458                // resolved) and does not emit the `WaitAllSatisfied`.
2459                state.dsl_apply(
2460                    mm_dsl::MeerkatMachineInput::CancelWaitAll,
2461                    "CancelWaitAll(cancel)",
2462                )?;
2463                state.wait_request_id = None;
2464                if state
2465                    .pending_wait
2466                    .as_ref()
2467                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2468                {
2469                    state.pending_wait = None;
2470                }
2471                Ok(())
2472            }
2473            _ => {
2474                if state
2475                    .pending_wait
2476                    .as_ref()
2477                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2478                {
2479                    state.pending_wait = None;
2480                }
2481                Ok(())
2482            }
2483        }
2484    }
2485}
2486
2487enum WaitAllFutureState {
2488    Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2489    Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2490    Done,
2491}
2492
2493struct WaitAllFuture<'a> {
2494    registry: &'a RuntimeOpsLifecycleRegistry,
2495    wait_request_id: WaitRequestId,
2496    state: WaitAllFutureState,
2497}
2498
2499impl Future for WaitAllFuture<'_> {
2500    type Output = Result<WaitAllResult, OpsLifecycleError>;
2501
2502    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2503        match &mut self.state {
2504            WaitAllFutureState::Ready(result) => {
2505                let ready = result.take().unwrap_or_else(|| {
2506                    Err(OpsLifecycleError::Internal(
2507                        "wait_all future polled after completion".into(),
2508                    ))
2509                });
2510                self.state = WaitAllFutureState::Done;
2511                Poll::Ready(ready)
2512            }
2513            WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2514                Poll::Pending => Poll::Pending,
2515                Poll::Ready(Ok(satisfied)) => {
2516                    let outcomes = match self.registry.read_state() {
2517                        Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2518                        Err(err) => Err(err),
2519                    };
2520                    self.state = WaitAllFutureState::Done;
2521                    Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2522                        outcomes,
2523                        satisfied,
2524                    }))
2525                }
2526                Poll::Ready(Err(_)) => {
2527                    self.state = WaitAllFutureState::Done;
2528                    Poll::Ready(Err(OpsLifecycleError::Internal(
2529                        "wait_all completion channel dropped".into(),
2530                    )))
2531                }
2532            },
2533            WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2534                "wait_all future polled after completion".into(),
2535            ))),
2536        }
2537    }
2538}
2539
2540impl Drop for WaitAllFuture<'_> {
2541    fn drop(&mut self) {
2542        if matches!(self.state, WaitAllFutureState::Waiting(_))
2543            && let Err(err) = self
2544                .registry
2545                .cancel_wait_all_internal(&self.wait_request_id)
2546        {
2547            tracing::error!(
2548                wait_request_id = %self.wait_request_id,
2549                error = %err,
2550                "generated wait_all authority rejected cancellation during drop"
2551            );
2552        }
2553    }
2554}
2555
2556// ---------------------------------------------------------------------------
2557// Generated op lifecycle result-class feedback
2558// ---------------------------------------------------------------------------
2559
2560fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2561    match action {
2562        mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2563        mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2564        mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2565        mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2566        mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2567        mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2568        mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2569        mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2570        mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2571        mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2572    }
2573}
2574
2575fn op_lifecycle_rejection_error_from_effects(
2576    id: &OperationId,
2577    requested_action: mm_dsl::OpLifecycleActionKind,
2578    effects: &[mm_dsl::MeerkatMachineEffect],
2579) -> Result<OpsLifecycleError, OpsLifecycleError> {
2580    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2581    let mut rejection = None;
2582    for effect in effects {
2583        let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2584            operation_id,
2585            action,
2586            reason,
2587            status,
2588        } = effect
2589        else {
2590            continue;
2591        };
2592        if rejection.is_some() {
2593            return Err(OpsLifecycleError::Internal(
2594                "generated op lifecycle authority emitted multiple rejection results".into(),
2595            ));
2596        }
2597        if operation_id != &expected_id || *action != requested_action {
2598            return Err(OpsLifecycleError::Internal(format!(
2599                "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2600            )));
2601        }
2602        let error = match reason {
2603            mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2604                if status.is_some() {
2605                    return Err(OpsLifecycleError::Internal(
2606                        "generated op lifecycle authority emitted not-found with status".into(),
2607                    ));
2608                }
2609                OpsLifecycleError::NotFound(id.clone())
2610            }
2611            mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2612                let status = status.ok_or_else(|| {
2613                    OpsLifecycleError::Internal(
2614                        "generated op lifecycle authority emitted invalid-transition without status"
2615                            .into(),
2616                    )
2617                })?;
2618                OpsLifecycleError::InvalidTransition {
2619                    id: id.clone(),
2620                    status: OperationStatus::from(status),
2621                    action: op_lifecycle_action_label(requested_action),
2622                }
2623            }
2624            mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2625                if status.is_none() {
2626                    return Err(OpsLifecycleError::Internal(
2627                        "generated op lifecycle authority emitted peer-not-expected without status"
2628                            .into(),
2629                    ));
2630                }
2631                OpsLifecycleError::PeerNotExpected(id.clone())
2632            }
2633            mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2634                if status.is_none() {
2635                    return Err(OpsLifecycleError::Internal(
2636                        "generated op lifecycle authority emitted already-peer-ready without status"
2637                            .into(),
2638                    ));
2639                }
2640                OpsLifecycleError::AlreadyPeerReady(id.clone())
2641            }
2642        };
2643        rejection = Some(error);
2644    }
2645    rejection.ok_or_else(|| {
2646        OpsLifecycleError::Internal(
2647            "generated op lifecycle authority emitted no rejection result".into(),
2648        )
2649    })
2650}
2651
2652fn classify_generated_op_rejection(
2653    state: &mut ShellState,
2654    err: mm_dsl::MeerkatMachineTransitionError,
2655    id: &OperationId,
2656    action: mm_dsl::OpLifecycleActionKind,
2657) -> OpsLifecycleError {
2658    match err {
2659        mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2660            match state.dsl_apply_with_effects(
2661                mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2662                    operation_id: mm_dsl::OperationId::from_domain(id).0,
2663                    action,
2664                },
2665                "ResolveOpLifecycleTransitionRejection",
2666            ) {
2667                Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2668                    .unwrap_or_else(|err| err),
2669                Err(err) => err,
2670            }
2671        }
2672        other => OpsLifecycleError::Internal(format!(
2673            "DSL rejected ops transition ({}): {other:?}",
2674            op_lifecycle_action_label(action)
2675        )),
2676    }
2677}
2678
2679fn apply_op_transition(
2680    state: &mut ShellState,
2681    id: &OperationId,
2682    input: mm_dsl::MeerkatMachineInput,
2683    action: mm_dsl::OpLifecycleActionKind,
2684) -> Result<(), OpsLifecycleError> {
2685    state.ensure_owner_active()?;
2686    state
2687        .dsl_apply_raw(input)
2688        .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2689}
2690
2691fn apply_terminal_op_transition_and_persist(
2692    state: &mut ShellState,
2693    id: &OperationId,
2694    input: mm_dsl::MeerkatMachineInput,
2695    action: mm_dsl::OpLifecycleActionKind,
2696) -> Result<(), OpsLifecycleError> {
2697    let previous_snapshot = state.dsl.0.snapshot();
2698    apply_op_transition(state, id, input, action)?;
2699    if let Err(err) = state.maybe_persist() {
2700        state.dsl.0.restore_snapshot(previous_snapshot);
2701        return Err(err);
2702    }
2703    Ok(())
2704}
2705
2706fn terminate_owner_locked(state: &mut ShellState, reason: &str) -> Result<(), OpsLifecycleError> {
2707    state.ensure_owner_active()?;
2708    let to_terminate = state.owner_termination_targets()?;
2709
2710    for (op_id, _status) in &to_terminate {
2711        let terminal_outcome = OperationTerminalOutcome::Terminated {
2712            reason: reason.to_owned(),
2713        };
2714        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2715
2716        apply_terminal_op_transition_and_persist(
2717            state,
2718            op_id,
2719            mm_dsl::MeerkatMachineInput::TerminateOp {
2720                operation_id: mm_dsl::OperationId::from_domain(op_id).0,
2721                outcome: outcome_kind,
2722                payload: terminal_outcome,
2723            },
2724            mm_dsl::OpLifecycleActionKind::Terminate,
2725        )?;
2726
2727        if let Some(entry) = state.finalize_terminal(op_id)? {
2728            state.publish_completion_entry(Some(entry));
2729        }
2730    }
2731    Ok(())
2732}
2733
2734fn op_registration_error_from_effects(
2735    id: &OperationId,
2736    effects: &[mm_dsl::MeerkatMachineEffect],
2737) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2738    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2739    let mut admission = None;
2740    for effect in effects {
2741        let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2742            operation_id,
2743            result,
2744            reject_reason,
2745            max_concurrent_limit,
2746            active_op_count,
2747        } = effect
2748        else {
2749            continue;
2750        };
2751        if admission.is_some() {
2752            return Err(OpsLifecycleError::Internal(
2753                "generated op registration authority emitted multiple admission results".into(),
2754            ));
2755        }
2756        if operation_id != &expected_id {
2757            return Err(OpsLifecycleError::Internal(format!(
2758                "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2759            )));
2760        }
2761        admission = Some(match result {
2762            mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2763                if reject_reason.is_some() {
2764                    return Err(OpsLifecycleError::Internal(
2765                        "generated op registration authority accepted with rejection reason".into(),
2766                    ));
2767                }
2768                None
2769            }
2770            mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2771                let reason = reject_reason.ok_or_else(|| {
2772                    OpsLifecycleError::Internal(
2773                        "generated op registration authority rejected without reason".into(),
2774                    )
2775                })?;
2776                let error = match reason {
2777                    mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2778                        OpsLifecycleError::AlreadyRegistered(id.clone())
2779                    }
2780                    mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2781                        let limit = max_concurrent_limit.ok_or_else(|| {
2782                            OpsLifecycleError::Internal(
2783                                "generated op registration authority rejected capacity without limit"
2784                                    .into(),
2785                            )
2786                        })?;
2787                        OpsLifecycleError::MaxConcurrentExceeded {
2788                            limit: limit as usize,
2789                            active: *active_op_count as usize,
2790                        }
2791                    }
2792                };
2793                Some(error)
2794            }
2795        });
2796    }
2797    admission.ok_or_else(|| {
2798        OpsLifecycleError::Internal(
2799            "generated op registration authority emitted no admission result".into(),
2800        )
2801    })
2802}
2803
2804impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2805    fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2806        self.register_operation_with_admission_limit(spec, None)
2807    }
2808
2809    fn register_operation_with_admission_limit(
2810        &self,
2811        mut spec: OperationSpec,
2812        max_concurrent: Option<usize>,
2813    ) -> Result<(), OpsLifecycleError> {
2814        let mut state = self.write_state()?;
2815        state.ensure_owner_active()?;
2816        let operation_id = spec.id.clone();
2817        let kind = spec.kind;
2818        let max_concurrent = max_concurrent
2819            .or(state.max_concurrent)
2820            .map(|limit| limit as u64);
2821
2822        let effects = state.dsl_apply_with_effects(
2823            mm_dsl::MeerkatMachineInput::RegisterOp {
2824                operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2825                kind: mm_dsl::OperationKind::from_domain(&kind),
2826                source: spec
2827                    .operation_source
2828                    .as_ref()
2829                    .map(mm_dsl::OperationSource::from_domain),
2830                max_concurrent,
2831            },
2832            "RegisterOp",
2833        )?;
2834        if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
2835            return Err(error);
2836        }
2837
2838        let authority_operation_source = state.operation_source(&operation_id)?;
2839        ShellState::align_spec_child_session_id_to_source(
2840            &mut spec,
2841            authority_operation_source.as_ref(),
2842        );
2843
2844        // Insert shell record.
2845        state.records.insert(operation_id, ShellRecord::new(spec));
2846        Ok(())
2847    }
2848
2849    fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2850        let mut state = self.write_state()?;
2851
2852        apply_op_transition(
2853            &mut state,
2854            id,
2855            mm_dsl::MeerkatMachineInput::StartOp {
2856                operation_id: mm_dsl::OperationId::from_domain(id).0,
2857            },
2858            mm_dsl::OpLifecycleActionKind::Start,
2859        )?;
2860
2861        // Shell concern: record the started timestamp.
2862        if let Some(shell) = state.records.get_mut(id) {
2863            shell.started_at = Some(Instant::now());
2864        }
2865        Ok(())
2866    }
2867
2868    fn provisioning_failed(
2869        &self,
2870        id: &OperationId,
2871        error: String,
2872    ) -> Result<(), OpsLifecycleError> {
2873        let mut state = self.write_state()?;
2874
2875        let terminal_outcome = OperationTerminalOutcome::Failed { error };
2876        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2877
2878        apply_terminal_op_transition_and_persist(
2879            &mut state,
2880            id,
2881            mm_dsl::MeerkatMachineInput::FailOp {
2882                operation_id: mm_dsl::OperationId::from_domain(id).0,
2883                outcome: outcome_kind,
2884                payload: terminal_outcome,
2885            },
2886            mm_dsl::OpLifecycleActionKind::Fail,
2887        )?;
2888
2889        let completion_entry = state.finalize_terminal(id)?;
2890        state.publish_completion_entry(completion_entry);
2891        Ok(())
2892    }
2893
2894    fn peer_ready(
2895        &self,
2896        id: &OperationId,
2897        peer: OperationPeerHandle,
2898    ) -> Result<(), OpsLifecycleError> {
2899        let mut state = self.write_state()?;
2900
2901        apply_op_transition(
2902            &mut state,
2903            id,
2904            mm_dsl::MeerkatMachineInput::PeerReadyOp {
2905                operation_id: mm_dsl::OperationId::from_domain(id).0,
2906            },
2907            mm_dsl::OpLifecycleActionKind::PeerReady,
2908        )?;
2909
2910        // Shell concern: store the peer handle.
2911        if let Some(shell) = state.records.get_mut(id) {
2912            shell.peer_handle = Some(peer);
2913        }
2914        Ok(())
2915    }
2916
2917    fn register_watcher(
2918        &self,
2919        id: &OperationId,
2920    ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
2921        let mut state = self.write_state()?;
2922
2923        if !state.contains(id) {
2924            return Err(OpsLifecycleError::NotFound(id.clone()));
2925        }
2926
2927        // If already terminal, return an already-resolved watch.
2928        if let Some(outcome) = state.terminal_outcome(id)? {
2929            return Ok(resolved_operation_completion_watch(outcome));
2930        }
2931
2932        // Shell concern: create the channel and store the sender.
2933        let shell = state.shell_record_mut(id)?;
2934        let (tx, rx) = tokio::sync::oneshot::channel();
2935        let watch = operation_completion_watch_from_receiver(rx);
2936        shell.watchers.push(OperationCompletionNotifier::new(tx));
2937        Ok(watch)
2938    }
2939
2940    fn report_progress(
2941        &self,
2942        id: &OperationId,
2943        _update: OperationProgressUpdate,
2944    ) -> Result<(), OpsLifecycleError> {
2945        let mut state = self.write_state()?;
2946
2947        apply_op_transition(
2948            &mut state,
2949            id,
2950            mm_dsl::MeerkatMachineInput::ProgressReportedOp {
2951                operation_id: mm_dsl::OperationId::from_domain(id).0,
2952            },
2953            mm_dsl::OpLifecycleActionKind::ProgressReported,
2954        )?;
2955        Ok(())
2956    }
2957
2958    fn complete_operation(
2959        &self,
2960        id: &OperationId,
2961        result: OperationResult,
2962    ) -> Result<(), OpsLifecycleError> {
2963        let mut state = self.write_state()?;
2964
2965        let terminal_outcome = OperationTerminalOutcome::Completed(result);
2966        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2967
2968        apply_terminal_op_transition_and_persist(
2969            &mut state,
2970            id,
2971            mm_dsl::MeerkatMachineInput::CompleteOp {
2972                operation_id: mm_dsl::OperationId::from_domain(id).0,
2973                outcome: outcome_kind,
2974                payload: terminal_outcome,
2975            },
2976            mm_dsl::OpLifecycleActionKind::Complete,
2977        )?;
2978
2979        let completion_entry = state.finalize_terminal(id)?;
2980        state.publish_completion_entry(completion_entry);
2981        Ok(())
2982    }
2983
2984    fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
2985        let mut state = self.write_state()?;
2986
2987        let terminal_outcome = OperationTerminalOutcome::Failed { error };
2988        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2989
2990        apply_terminal_op_transition_and_persist(
2991            &mut state,
2992            id,
2993            mm_dsl::MeerkatMachineInput::FailOp {
2994                operation_id: mm_dsl::OperationId::from_domain(id).0,
2995                outcome: outcome_kind,
2996                payload: terminal_outcome,
2997            },
2998            mm_dsl::OpLifecycleActionKind::Fail,
2999        )?;
3000
3001        let completion_entry = state.finalize_terminal(id)?;
3002        state.publish_completion_entry(completion_entry);
3003        Ok(())
3004    }
3005
3006    fn abort_provisioning(
3007        &self,
3008        id: &OperationId,
3009        reason: Option<String>,
3010    ) -> Result<(), OpsLifecycleError> {
3011        let mut state = self.write_state()?;
3012
3013        let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
3014        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3015
3016        apply_terminal_op_transition_and_persist(
3017            &mut state,
3018            id,
3019            mm_dsl::MeerkatMachineInput::AbortOp {
3020                operation_id: mm_dsl::OperationId::from_domain(id).0,
3021                outcome: outcome_kind,
3022                payload: terminal_outcome,
3023            },
3024            mm_dsl::OpLifecycleActionKind::Abort,
3025        )?;
3026
3027        let completion_entry = state.finalize_terminal(id)?;
3028        state.publish_completion_entry(completion_entry);
3029        Ok(())
3030    }
3031
3032    fn rollback_unreturned_operation(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3033        let mut state = self.write_state()?;
3034        state.ensure_owner_active()?;
3035        let expected_id = mm_dsl::OperationId::from_domain(id).0;
3036        let effects = state.dsl_apply_with_effects(
3037            mm_dsl::MeerkatMachineInput::RollbackUnreturnedOp {
3038                operation_id: expected_id.clone(),
3039            },
3040            "RollbackUnreturnedOp",
3041        )?;
3042        let mut acknowledgements = effects.iter().filter(|effect| {
3043            matches!(
3044                effect,
3045                mm_dsl::MeerkatMachineEffect::UnreturnedOpRolledBack { operation_id }
3046                    if operation_id == &expected_id
3047            )
3048        });
3049        if acknowledgements.next().is_none() || acknowledgements.next().is_some() {
3050            return Err(OpsLifecycleError::Internal(format!(
3051                "generated rollback authority did not emit exactly one matching acknowledgement for {id}"
3052            )));
3053        }
3054        state.records.remove(id);
3055        state.completed_order.retain(|queued| queued != id);
3056        Ok(())
3057    }
3058
3059    fn cancel_operation(
3060        &self,
3061        id: &OperationId,
3062        reason: Option<String>,
3063    ) -> Result<(), OpsLifecycleError> {
3064        let mut state = self.write_state()?;
3065
3066        let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
3067        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3068
3069        apply_terminal_op_transition_and_persist(
3070            &mut state,
3071            id,
3072            mm_dsl::MeerkatMachineInput::CancelOp {
3073                operation_id: mm_dsl::OperationId::from_domain(id).0,
3074                outcome: outcome_kind,
3075                payload: terminal_outcome,
3076            },
3077            mm_dsl::OpLifecycleActionKind::Cancel,
3078        )?;
3079
3080        let completion_entry = state.finalize_terminal(id)?;
3081        state.publish_completion_entry(completion_entry);
3082        Ok(())
3083    }
3084
3085    fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3086        let mut state = self.write_state()?;
3087
3088        apply_op_transition(
3089            &mut state,
3090            id,
3091            mm_dsl::MeerkatMachineInput::RetireRequestedOp {
3092                operation_id: mm_dsl::OperationId::from_domain(id).0,
3093            },
3094            mm_dsl::OpLifecycleActionKind::RetireRequested,
3095        )?;
3096        Ok(())
3097    }
3098
3099    fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3100        let mut state = self.write_state()?;
3101
3102        let terminal_outcome = OperationTerminalOutcome::Retired;
3103        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3104
3105        apply_terminal_op_transition_and_persist(
3106            &mut state,
3107            id,
3108            mm_dsl::MeerkatMachineInput::RetireCompletedOp {
3109                operation_id: mm_dsl::OperationId::from_domain(id).0,
3110                outcome: outcome_kind,
3111                payload: terminal_outcome,
3112            },
3113            mm_dsl::OpLifecycleActionKind::RetireCompleted,
3114        )?;
3115
3116        let completion_entry = state.finalize_terminal(id)?;
3117        state.publish_completion_entry(completion_entry);
3118        Ok(())
3119    }
3120
3121    fn snapshot(
3122        &self,
3123        id: &OperationId,
3124    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3125        let state = self.read_state()?;
3126        state.snapshot(id)
3127    }
3128
3129    fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3130        let state = self.read_state()?;
3131        let mut snapshots = Vec::new();
3132        for id in state.operation_ids()? {
3133            let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3134                OpsLifecycleError::Internal(format!(
3135                    "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3136                ))
3137            })?;
3138            snapshots.push(snapshot);
3139        }
3140        snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3141        Ok(snapshots)
3142    }
3143
3144    fn classify_operation_terminality(
3145        &self,
3146        id: &OperationId,
3147        status: OperationStatus,
3148    ) -> Result<bool, OpsLifecycleError> {
3149        ShellState::operation_status_is_terminal(id, status)
3150    }
3151
3152    fn classify_operation_public_result(
3153        &self,
3154        id: &OperationId,
3155    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3156        let state = self.read_state()?;
3157        let status = match state.status(id) {
3158            Some(status) => status,
3159            None if state.records.contains_key(id)
3160                || state.has_generated_operation_record_fact(id) =>
3161            {
3162                return Err(OpsLifecycleError::Internal(format!(
3163                    "generated op lifecycle authority missing status for {id}"
3164                )));
3165            }
3166            None => OperationStatus::Absent,
3167        };
3168        ShellState::operation_public_result_class(id, status)
3169    }
3170
3171    fn classify_operation_completion_wake(
3172        &self,
3173        id: &OperationId,
3174        kind: OperationKind,
3175    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3176        ShellState::operation_completion_wake_class(id, kind)
3177    }
3178
3179    fn classify_operation_transition_idempotence(
3180        &self,
3181        id: &OperationId,
3182        action: OperationLifecycleAction,
3183    ) -> Result<bool, OpsLifecycleError> {
3184        let state = self.read_state()?;
3185        let status = match state.status(id) {
3186            Some(status) => status,
3187            None if state.records.contains_key(id)
3188                || state.has_generated_operation_record_fact(id) =>
3189            {
3190                return Err(OpsLifecycleError::Internal(format!(
3191                    "generated op lifecycle authority missing status for {id}"
3192                )));
3193            }
3194            None => OperationStatus::Absent,
3195        };
3196        ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3197    }
3198
3199    fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3200        let mut state = self.write_state()?;
3201        if state.owner_retired {
3202            return Ok(());
3203        }
3204        terminate_owner_locked(&mut state, &reason)
3205    }
3206
3207    fn collect_completed(
3208        &self,
3209    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3210        let mut state = self.write_state()?;
3211        state.ensure_owner_active()?;
3212
3213        let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3214        let mut collected = Vec::with_capacity(ids.len());
3215        for id in ids {
3216            let outcome = state.terminal_outcome(&id)?;
3217            state.dsl_apply(
3218                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3219                    operation_id: mm_dsl::OperationId::from_domain(&id).0,
3220                },
3221                "CollectCompletedOp",
3222            )?;
3223            state.completed_order.retain(|queued| queued != &id);
3224            state.records.remove(&id);
3225            if let Some(outcome) = outcome {
3226                collected.push((id, outcome));
3227            }
3228        }
3229        Ok(collected)
3230    }
3231
3232    fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3233        Some(self.completion_feed_handle())
3234    }
3235
3236    fn completion_cursor(
3237        &self,
3238        consumer: CompletionCursorConsumer,
3239    ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3240        let state = self.read_state()?;
3241        Ok(Some(state.completion_cursor(consumer)))
3242    }
3243
3244    fn advance_completion_cursor(
3245        &self,
3246        consumer: CompletionCursorConsumer,
3247        cursor: CompletionSeq,
3248        projection: Option<&meerkat_core::EpochCursorState>,
3249    ) -> Result<CompletionSeq, OpsLifecycleError> {
3250        let mut state = self.write_state()?;
3251        state.ensure_owner_active()?;
3252        let input = match consumer {
3253            CompletionCursorConsumer::AgentApplied => {
3254                mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3255            }
3256            CompletionCursorConsumer::RuntimeObserved => {
3257                mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3258            }
3259            CompletionCursorConsumer::RuntimeInjected => {
3260                mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3261            }
3262        };
3263        let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3264        let advanced = effects
3265            .iter()
3266            .find_map(|effect| match (consumer, effect) {
3267                (
3268                    CompletionCursorConsumer::AgentApplied,
3269                    mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3270                ) => Some(*cursor),
3271                (
3272                    CompletionCursorConsumer::RuntimeObserved,
3273                    mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3274                        cursor,
3275                    },
3276                ) => Some(*cursor),
3277                (
3278                    CompletionCursorConsumer::RuntimeInjected,
3279                    mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3280                        cursor,
3281                    },
3282                ) => Some(*cursor),
3283                _ => None,
3284            })
3285            .ok_or_else(|| {
3286                OpsLifecycleError::Internal(format!(
3287                    "generated completion cursor transition emitted no feedback for {consumer:?}"
3288                ))
3289            })?;
3290        if let Some(projection) = projection {
3291            projection.project_authorized_completion_cursor(consumer, advanced);
3292        }
3293        Ok(advanced)
3294    }
3295
3296    fn wait_all(
3297        &self,
3298        run_id: &RunId,
3299        ids: &[OperationId],
3300    ) -> std::pin::Pin<
3301        Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3302    > {
3303        let wait_request_id = WaitRequestId::new();
3304        let owned_ids = ids.to_vec();
3305
3306        let state = match self.write_state() {
3307            Ok(mut state) => {
3308                if let Err(error) = state.ensure_owner_active() {
3309                    return Box::pin(WaitAllFuture {
3310                        registry: self,
3311                        wait_request_id,
3312                        state: WaitAllFutureState::Ready(Some(Err(error))),
3313                    });
3314                }
3315                match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3316                    Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3317                        let outcomes =
3318                            state
3319                                .collect_wait_outcomes(&satisfied.operation_ids)
3320                                .map(|outcomes| WaitAllResult {
3321                                    outcomes,
3322                                    satisfied,
3323                                });
3324                        WaitAllFutureState::Ready(Some(outcomes))
3325                    }
3326                    Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3327                        if state.pending_wait.is_some() {
3328                            // Roll back the DSL barrier we just activated so the
3329                            // registry is not stuck in a wait-active state with
3330                            // no correlation oneshot to resolve. `CancelWaitAll`
3331                            // is the no-obligation clearer (members need not be
3332                            // terminal).
3333                            let rollback = state.dsl_apply(
3334                                mm_dsl::MeerkatMachineInput::CancelWaitAll,
3335                                "CancelWaitAll(rollback)",
3336                            );
3337                            return Box::pin(WaitAllFuture {
3338                                registry: self,
3339                                wait_request_id,
3340                                state: WaitAllFutureState::Ready(Some(Err(match rollback {
3341                                    Ok(()) => OpsLifecycleError::Internal(
3342                                        "wait_all started while a pending wait sender already existed"
3343                                            .into(),
3344                                    ),
3345                                    Err(err) => err,
3346                                }))),
3347                            });
3348                        }
3349                        state.wait_request_id = Some(wait_request_id.clone());
3350                        let (sender, receiver) = tokio::sync::oneshot::channel();
3351                        state.pending_wait = Some(PendingWaitState {
3352                            wait_request_id: wait_request_id.clone(),
3353                            sender,
3354                        });
3355                        WaitAllFutureState::Waiting(receiver)
3356                    }
3357                    Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3358                }
3359            }
3360            Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3361        };
3362
3363        Box::pin(WaitAllFuture {
3364            registry: self,
3365            wait_request_id,
3366            state,
3367        })
3368    }
3369}
3370
3371#[cfg(test)]
3372#[allow(clippy::unwrap_used, clippy::panic)]
3373mod tests {
3374    use super::*;
3375    use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3376    use meerkat_core::lifecycle::RunId;
3377    use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3378    use meerkat_core::types::SessionId;
3379    use std::sync::atomic::Ordering;
3380    use uuid::Uuid;
3381
3382    fn test_run_id() -> RunId {
3383        RunId(Uuid::from_u128(1))
3384    }
3385
3386    fn background_spec(name: &str) -> OperationSpec {
3387        OperationSpec {
3388            id: OperationId::new(),
3389            kind: OperationKind::BackgroundToolOp,
3390            owner_session_id: SessionId::new(),
3391            display_name: name.into(),
3392            source_label: "test".into(),
3393            operation_source: None,
3394            child_session_id: None,
3395            expect_peer_channel: false,
3396        }
3397    }
3398
3399    #[tokio::test]
3400    async fn late_watchers_resolve_immediately() {
3401        let registry = RuntimeOpsLifecycleRegistry::new();
3402        let spec = background_spec("late");
3403        let op_id = spec.id.clone();
3404        registry.register_operation(spec).unwrap();
3405        registry.provisioning_succeeded(&op_id).unwrap();
3406        registry
3407            .complete_operation(
3408                &op_id,
3409                OperationResult {
3410                    id: op_id.clone(),
3411                    content: "done".into(),
3412                    is_error: false,
3413                    duration_ms: 1,
3414                    tokens_used: 0,
3415                },
3416            )
3417            .unwrap();
3418
3419        let watch = registry.register_watcher(&op_id).unwrap();
3420        match watch
3421            .await
3422            .expect("operation completion watch should resolve")
3423        {
3424            OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3425            other => panic!("expected completed outcome, got {other:?}"),
3426        }
3427    }
3428
3429    #[tokio::test]
3430    async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3431        let (tx, rx) = tokio::sync::oneshot::channel();
3432        let watch = operation_completion_watch_from_receiver(rx);
3433        drop(tx);
3434
3435        assert_eq!(
3436            watch.await,
3437            Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3438        );
3439    }
3440
3441    #[test]
3442    fn peer_ready_requires_peer_expectation() {
3443        let registry = RuntimeOpsLifecycleRegistry::new();
3444        let spec = background_spec("no-peer");
3445        let op_id = spec.id.clone();
3446        registry.register_operation(spec).unwrap();
3447        registry.provisioning_succeeded(&op_id).unwrap();
3448
3449        let result = registry.peer_ready(
3450            &op_id,
3451            OperationPeerHandle {
3452                peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3453                trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3454                    "peer",
3455                    PeerId::new(),
3456                    "inproc://peer",
3457                )
3458                .unwrap(),
3459            },
3460        );
3461        assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3462    }
3463
3464    /// K8b pinning test: the terminal payload carried through the generated
3465    /// machine IS the typed domain outcome — no JSON codec exists in either
3466    /// direction. Every variant classifies to its matching discriminant, the
3467    /// fail-closed read returns the exact payload when the discriminant
3468    /// matches, and rejects any discriminant/variant disagreement.
3469    #[test]
3470    fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3471        let op_id = OperationId::new();
3472        let outcomes = vec![
3473            (
3474                OperationTerminalOutcome::Completed(OperationResult {
3475                    id: op_id.clone(),
3476                    content: "done".into(),
3477                    is_error: false,
3478                    duration_ms: 7,
3479                    tokens_used: 42,
3480                }),
3481                mm_dsl::OperationTerminalOutcomeKind::Completed,
3482            ),
3483            (
3484                OperationTerminalOutcome::Failed {
3485                    error: "boom".into(),
3486                },
3487                mm_dsl::OperationTerminalOutcomeKind::Failed,
3488            ),
3489            (
3490                OperationTerminalOutcome::Aborted {
3491                    reason: Some("user aborted".into()),
3492                },
3493                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3494            ),
3495            (
3496                OperationTerminalOutcome::Aborted { reason: None },
3497                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3498            ),
3499            (
3500                OperationTerminalOutcome::Cancelled {
3501                    reason: Some("cancelled".into()),
3502                },
3503                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3504            ),
3505            (
3506                OperationTerminalOutcome::Cancelled { reason: None },
3507                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3508            ),
3509            (
3510                OperationTerminalOutcome::Retired,
3511                mm_dsl::OperationTerminalOutcomeKind::Retired,
3512            ),
3513            (
3514                OperationTerminalOutcome::Terminated {
3515                    reason: "owner stopped".into(),
3516                },
3517                mm_dsl::OperationTerminalOutcomeKind::Terminated,
3518            ),
3519        ];
3520
3521        for (outcome, expected_kind) in &outcomes {
3522            assert_eq!(
3523                mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3524                *expected_kind,
3525                "typed payload {outcome:?} must classify to {expected_kind:?}"
3526            );
3527            let read = ShellState::checked_terminal_payload(
3528                *expected_kind,
3529                outcome,
3530                "test authority",
3531                "test-op",
3532            )
3533            .expect("matching discriminant must read back the exact payload");
3534            assert_eq!(&read, outcome);
3535        }
3536
3537        // Discriminant/variant disagreement fails closed.
3538        let err = ShellState::checked_terminal_payload(
3539            mm_dsl::OperationTerminalOutcomeKind::Completed,
3540            &OperationTerminalOutcome::Retired,
3541            "test authority",
3542            "test-op",
3543        )
3544        .expect_err("variant mismatch must be rejected");
3545        assert!(matches!(err, OpsLifecycleError::Internal(_)));
3546    }
3547
3548    /// K8b machine-ownership pin: a terminal payload whose variant disagrees
3549    /// with the transition's terminal kind is rejected by the generated
3550    /// machine guard (`payload_variant_matches_kind`) — not by any shell
3551    /// decode step. Drives the DSL input directly to prove the guard owns
3552    /// the invariant.
3553    #[test]
3554    fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3555        let registry = RuntimeOpsLifecycleRegistry::new();
3556        let spec = background_spec("variant-mismatch");
3557        let op_id = spec.id.clone();
3558        registry.register_operation(spec).unwrap();
3559        registry.provisioning_succeeded(&op_id).unwrap();
3560
3561        let mut state = registry.write_state().unwrap();
3562        let err = state
3563            .dsl_apply(
3564                mm_dsl::MeerkatMachineInput::CompleteOp {
3565                    operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3566                    outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3567                    // Variant mismatch: Completed kind with a Retired payload.
3568                    payload: OperationTerminalOutcome::Retired,
3569                },
3570                "CompleteOp",
3571            )
3572            .expect_err("machine must reject payload variant mismatch");
3573        // The kernel reports guard rejections without naming the guard; the
3574        // discriminating pin is the pair: identical input EXCEPT the payload
3575        // variant is guard-rejected here, then accepted below. Status
3576        // (`Running`) and discriminant (`Completed`) are identical in both
3577        // calls, so `payload_variant_matches_kind` is the only differing
3578        // guard.
3579        assert!(
3580            matches!(
3581                &err,
3582                OpsLifecycleError::Internal(message)
3583                    if message.contains("GuardRejected") && message.contains("CompleteOp")
3584            ),
3585            "expected generated guard rejection for CompleteOp, got: {err:?}"
3586        );
3587        drop(state);
3588
3589        // RetireCompletedOp requires the unit Retired payload; a data-carrying
3590        // payload is rejected by the same guard shape.
3591        registry
3592            .complete_operation(
3593                &op_id,
3594                OperationResult {
3595                    id: op_id.clone(),
3596                    content: "done".into(),
3597                    is_error: false,
3598                    duration_ms: 1,
3599                    tokens_used: 0,
3600                },
3601            )
3602            .expect("matching variant must complete");
3603    }
3604
3605    #[test]
3606    fn duplicate_registration_rejection_is_generated() {
3607        let registry = RuntimeOpsLifecycleRegistry::new();
3608        let spec = background_spec("duplicate");
3609        let op_id = spec.id.clone();
3610
3611        registry.register_operation(spec.clone()).unwrap();
3612        let result = registry.register_operation(spec);
3613
3614        assert!(matches!(
3615            result,
3616            Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3617        ));
3618    }
3619
3620    #[test]
3621    fn invalid_transition_rejection_is_generated() {
3622        let registry = RuntimeOpsLifecycleRegistry::new();
3623        let spec = background_spec("invalid-transition");
3624        let op_id = spec.id.clone();
3625        registry.register_operation(spec).unwrap();
3626
3627        let result = registry.complete_operation(
3628            &op_id,
3629            OperationResult {
3630                id: op_id.clone(),
3631                content: "too-early".into(),
3632                is_error: false,
3633                duration_ms: 1,
3634                tokens_used: 0,
3635            },
3636        );
3637
3638        assert!(matches!(
3639            result,
3640            Err(OpsLifecycleError::InvalidTransition {
3641                id,
3642                status: OperationStatus::Provisioning,
3643                action: "complete_operation",
3644            }) if id == op_id
3645        ));
3646    }
3647
3648    #[tokio::test]
3649    async fn multi_listener_completion() {
3650        let registry = RuntimeOpsLifecycleRegistry::new();
3651        let spec = background_spec("multi");
3652        let op_id = spec.id.clone();
3653        registry.register_operation(spec).unwrap();
3654        registry.provisioning_succeeded(&op_id).unwrap();
3655
3656        let watch1 = registry.register_watcher(&op_id).unwrap();
3657        let watch2 = registry.register_watcher(&op_id).unwrap();
3658        let watch3 = registry.register_watcher(&op_id).unwrap();
3659
3660        registry
3661            .complete_operation(
3662                &op_id,
3663                OperationResult {
3664                    id: op_id.clone(),
3665                    content: "multi-done".into(),
3666                    is_error: false,
3667                    duration_ms: 1,
3668                    tokens_used: 0,
3669                },
3670            )
3671            .unwrap();
3672
3673        for watch in [watch1, watch2, watch3] {
3674            match watch
3675                .await
3676                .expect("operation completion watch should resolve")
3677            {
3678                OperationTerminalOutcome::Completed(result) => {
3679                    assert_eq!(result.content, "multi-done");
3680                }
3681                other => panic!("expected completed, got {other:?}"),
3682            }
3683        }
3684    }
3685
3686    #[tokio::test]
3687    async fn wait_all_returns_all_outcomes() {
3688        let registry = RuntimeOpsLifecycleRegistry::new();
3689
3690        let spec_a = background_spec("a");
3691        let id_a = spec_a.id.clone();
3692        registry.register_operation(spec_a).unwrap();
3693        registry.provisioning_succeeded(&id_a).unwrap();
3694
3695        let spec_b = background_spec("b");
3696        let id_b = spec_b.id.clone();
3697        registry.register_operation(spec_b).unwrap();
3698        registry.provisioning_succeeded(&id_b).unwrap();
3699
3700        registry
3701            .complete_operation(
3702                &id_a,
3703                OperationResult {
3704                    id: id_a.clone(),
3705                    content: "a-done".into(),
3706                    is_error: false,
3707                    duration_ms: 1,
3708                    tokens_used: 0,
3709                },
3710            )
3711            .unwrap();
3712        registry.fail_operation(&id_b, "b-error".into()).unwrap();
3713
3714        let wait_result = registry
3715            .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3716            .await
3717            .unwrap();
3718        assert_eq!(wait_result.outcomes.len(), 2);
3719        assert_eq!(wait_result.outcomes[0].0, id_a);
3720        assert!(matches!(
3721            wait_result.outcomes[0].1,
3722            OperationTerminalOutcome::Completed(_)
3723        ));
3724        assert_eq!(wait_result.outcomes[1].0, id_b);
3725        assert!(matches!(
3726            wait_result.outcomes[1].1,
3727            OperationTerminalOutcome::Failed { .. }
3728        ));
3729        // Obligation carries the awaited IDs
3730        assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3731        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3732    }
3733
3734    /// Exercises the trait `wait_all` path (via `dyn OpsLifecycleRegistry`)
3735    /// which must submit WaitAll through the DSL for cross-machine handoff.
3736    #[tokio::test]
3737    async fn wait_all_trait_path_submits_through_authority() {
3738        let registry = RuntimeOpsLifecycleRegistry::new();
3739        let spec = background_spec("trait-wait");
3740        let op_id = spec.id.clone();
3741        registry.register_operation(spec).unwrap();
3742        registry.provisioning_succeeded(&op_id).unwrap();
3743        registry
3744            .complete_operation(
3745                &op_id,
3746                OperationResult {
3747                    id: op_id.clone(),
3748                    content: "done".into(),
3749                    is_error: false,
3750                    duration_ms: 1,
3751                    tokens_used: 0,
3752                },
3753            )
3754            .unwrap();
3755
3756        // Call through trait object to exercise the trait impl, not the inherent method.
3757        let trait_ref: &dyn OpsLifecycleRegistry = &registry;
3758        let wait_result = trait_ref
3759            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3760            .await
3761            .unwrap();
3762        assert_eq!(wait_result.outcomes.len(), 1);
3763        assert!(matches!(
3764            wait_result.outcomes[0].1,
3765            OperationTerminalOutcome::Completed(_)
3766        ));
3767        // Obligation carries the validated ID
3768        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3769        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3770        let state = registry.read_state().unwrap();
3771        assert!(
3772            !state.wait_active(),
3773            "already-satisfied wait_all must be cleared by generated satisfaction authority"
3774        );
3775        assert!(state.wait_operation_ids().unwrap().is_empty());
3776    }
3777
3778    #[tokio::test]
3779    async fn wait_all_duplicate_rejection_is_generated() {
3780        let registry = RuntimeOpsLifecycleRegistry::new();
3781        let spec = background_spec("duplicate-wait");
3782        let op_id = spec.id.clone();
3783        registry.register_operation(spec).unwrap();
3784        registry.provisioning_succeeded(&op_id).unwrap();
3785
3786        let result = registry
3787            .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3788            .await;
3789
3790        assert!(matches!(
3791            result,
3792            Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3793        ));
3794        let state = registry.read_state().unwrap();
3795        assert!(
3796            !state.wait_active(),
3797            "duplicate wait rejection must not create a shell or machine barrier"
3798        );
3799        assert!(state.wait_operation_ids().unwrap().is_empty());
3800    }
3801
3802    #[tokio::test]
3803    async fn wait_all_active_rejection_is_generated() {
3804        let registry = RuntimeOpsLifecycleRegistry::new();
3805        let spec = background_spec("active-wait");
3806        let op_id = spec.id.clone();
3807        registry.register_operation(spec).unwrap();
3808        registry.provisioning_succeeded(&op_id).unwrap();
3809
3810        let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3811        let result = registry
3812            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3813            .await;
3814
3815        assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3816        drop(active_wait);
3817        let state = registry.read_state().unwrap();
3818        assert!(!state.wait_active());
3819        assert!(state.wait_operation_ids().unwrap().is_empty());
3820    }
3821
3822    #[tokio::test]
3823    async fn wait_all_unknown_operation_rejection_is_generated() {
3824        let registry = RuntimeOpsLifecycleRegistry::new();
3825        let op_id = OperationId::new();
3826
3827        let result = registry
3828            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3829            .await;
3830
3831        assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3832        let state = registry.read_state().unwrap();
3833        assert!(!state.wait_active());
3834        assert!(state.wait_operation_ids().unwrap().is_empty());
3835    }
3836
3837    #[tokio::test]
3838    async fn wait_all_resolves_from_authority_owned_wait_request() {
3839        let registry = RuntimeOpsLifecycleRegistry::new();
3840        let run_id = test_run_id();
3841
3842        let spec = background_spec("pending");
3843        let op_id = spec.id.clone();
3844        registry.register_operation(spec).unwrap();
3845        registry.provisioning_succeeded(&op_id).unwrap();
3846
3847        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3848        tokio::pin!(wait_fut);
3849        assert!(
3850            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3851                .await
3852                .is_err()
3853        );
3854
3855        let active_wait_request_id = {
3856            let state = registry.read_state().unwrap();
3857            let wait_request_id = match state.wait_request_id.clone() {
3858                Some(wait_request_id) => wait_request_id,
3859                None => panic!("wait request should be active"),
3860            };
3861            assert_eq!(
3862                state.wait_operation_ids().unwrap().as_slice(),
3863                std::slice::from_ref(&op_id)
3864            );
3865            wait_request_id
3866        };
3867
3868        registry
3869            .complete_operation(
3870                &op_id,
3871                OperationResult {
3872                    id: op_id.clone(),
3873                    content: "done".into(),
3874                    is_error: false,
3875                    duration_ms: 1,
3876                    tokens_used: 0,
3877                },
3878            )
3879            .unwrap();
3880
3881        let wait_result = wait_fut.await.unwrap();
3882        assert_eq!(
3883            wait_result.satisfied.wait_request_id,
3884            active_wait_request_id
3885        );
3886        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
3887        assert!(matches!(
3888            wait_result.outcomes.as_slice(),
3889            [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
3890        ));
3891        assert!(registry.read_state().unwrap().wait_request_id.is_none());
3892    }
3893
3894    #[tokio::test]
3895    async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
3896        let registry = RuntimeOpsLifecycleRegistry::new();
3897        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3898        registry.set_persistence_channel(
3899            tx,
3900            meerkat_core::RuntimeEpochId::new(),
3901            Arc::new(meerkat_core::EpochCursorState::new()),
3902        );
3903        let worker = std::thread::spawn(move || {
3904            let request = rx.blocking_recv().expect("persistence request");
3905            let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
3906                "injected persistence failure".to_string(),
3907            )));
3908        });
3909
3910        let spec = background_spec("feed-after-persist");
3911        let op_id = spec.id.clone();
3912        registry.register_operation(spec).unwrap();
3913        registry.provisioning_succeeded(&op_id).unwrap();
3914        let watch = registry.register_watcher(&op_id).unwrap();
3915
3916        let err = registry
3917            .complete_operation(
3918                &op_id,
3919                OperationResult {
3920                    id: op_id.clone(),
3921                    content: "done".into(),
3922                    is_error: false,
3923                    duration_ms: 1,
3924                    tokens_used: 0,
3925                },
3926            )
3927            .expect_err("persistence failure must fail the terminal transition");
3928        assert!(
3929            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
3930            "unexpected persistence error: {err:?}"
3931        );
3932        worker.join().expect("persistence worker");
3933
3934        let feed = registry.completion_feed_handle();
3935        let batch = feed.list_since(0);
3936        assert!(
3937            batch.entries.is_empty(),
3938            "completion feed must not publish entries before durable snapshot success"
3939        );
3940        assert_eq!(batch.watermark, 0);
3941        assert!(
3942            tokio::time::timeout(std::time::Duration::from_millis(10), watch)
3943                .await
3944                .is_err(),
3945            "watchers must not resolve before durable terminal snapshot success"
3946        );
3947        assert_eq!(
3948            registry.snapshot(&op_id).unwrap().unwrap().status,
3949            OperationStatus::Running,
3950            "failed persistence must roll the generated terminal transition back"
3951        );
3952
3953        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3954        registry.set_persistence_channel(
3955            tx,
3956            meerkat_core::RuntimeEpochId::new(),
3957            Arc::new(meerkat_core::EpochCursorState::new()),
3958        );
3959        let worker = std::thread::spawn(move || {
3960            let request = rx.blocking_recv().expect("second persistence request");
3961            let _ = request.result_tx.send(Ok(()));
3962        });
3963        registry
3964            .complete_operation(
3965                &op_id,
3966                OperationResult {
3967                    id: op_id.clone(),
3968                    content: "done".into(),
3969                    is_error: false,
3970                    duration_ms: 1,
3971                    tokens_used: 0,
3972                },
3973            )
3974            .expect("rolled-back terminal transition should remain retryable");
3975        worker.join().expect("second persistence worker");
3976
3977        let batch = feed.list_since(0);
3978        assert_eq!(batch.entries.len(), 1);
3979        assert_eq!(batch.entries[0].operation_id, op_id);
3980    }
3981
3982    #[tokio::test]
3983    async fn dropping_wait_all_future_cancels_active_wait_request() {
3984        let registry = RuntimeOpsLifecycleRegistry::new();
3985        let run_id = test_run_id();
3986
3987        let spec = background_spec("cancelled-wait");
3988        let op_id = spec.id.clone();
3989        registry.register_operation(spec).unwrap();
3990        registry.provisioning_succeeded(&op_id).unwrap();
3991
3992        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3993        drop(wait_fut);
3994
3995        let state = registry.read_state().unwrap();
3996        assert!(state.wait_request_id.is_none());
3997        assert!(state.wait_operation_ids().unwrap().is_empty());
3998        assert!(!state.wait_active());
3999    }
4000
4001    /// id 101: an authority-invariant corruption surfaced during a terminal
4002    /// transition must NOT report the op terminal with a silently-hung barrier.
4003    /// The terminal call must return the typed `Internal` fault, AND the awaited
4004    /// `wait_all` future must resolve to `Err` (via the dropped-sender arm)
4005    /// instead of hanging forever.
4006    #[tokio::test]
4007    async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
4008        let registry = RuntimeOpsLifecycleRegistry::new();
4009        let run_id = test_run_id();
4010
4011        let spec = background_spec("corrupt-barrier");
4012        let op_id = spec.id.clone();
4013        registry.register_operation(spec).unwrap();
4014        registry.provisioning_succeeded(&op_id).unwrap();
4015
4016        // Activate a barrier whose waiter is still pending.
4017        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
4018        tokio::pin!(wait_fut);
4019        assert!(
4020            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
4021                .await
4022                .is_err(),
4023            "barrier waiter must still be pending before corruption"
4024        );
4025        assert!(registry.read_state().unwrap().wait_request_id.is_some());
4026
4027        // Corrupt the generated wait authority: an active wait_request_id with
4028        // no wait_run_id forces `try_satisfy_wait_all_authority` down its
4029        // non-GuardRejected `Internal` arm during the terminal transition.
4030        {
4031            let mut state = registry.write_state().unwrap();
4032            let mut machine_state = state.dsl.0.state().clone();
4033            machine_state.wait_run_id = None;
4034            state.dsl = DslAuthority(Box::new(
4035                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4036            ));
4037        }
4038
4039        // The terminal transition must FAIL with the typed authority fault, not
4040        // report the op complete.
4041        let err = registry
4042            .complete_operation(
4043                &op_id,
4044                OperationResult {
4045                    id: op_id.clone(),
4046                    content: "done".into(),
4047                    is_error: false,
4048                    duration_ms: 1,
4049                    tokens_used: 0,
4050                },
4051            )
4052            .expect_err("corrupt wait authority must fail the terminal transition");
4053        assert!(
4054            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
4055            "unexpected terminal error: {err:?}"
4056        );
4057
4058        // The waiter must resolve to Err (dropped sender), not hang.
4059        let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
4060            .await
4061            .expect("waiter must resolve, not hang, after authority corruption");
4062        match waiter_result {
4063            Err(OpsLifecycleError::Internal(message)) => assert!(
4064                message.contains("wait_all completion channel dropped"),
4065                "unexpected waiter error message: {message}"
4066            ),
4067            other => panic!("expected dropped-channel Internal error, got {other:?}"),
4068        }
4069    }
4070
4071    /// id 97: a poisoned registry lock must surface as a typed `Internal` fault
4072    /// from `completion_cursor`, NOT laundered into `Ok(None)` (which means "no
4073    /// generated cursor authority") or a bare `None`.
4074    #[test]
4075    fn completion_cursor_propagates_poison_not_none() {
4076        let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
4077
4078        // Poison the registry RwLock by panicking while holding the write guard.
4079        let poison_registry = std::sync::Arc::clone(&registry);
4080        let join = std::thread::spawn(move || {
4081            let _guard = poison_registry.write_state().unwrap();
4082            panic!("intentional panic to poison ops lifecycle registry lock");
4083        });
4084        assert!(
4085            join.join().is_err(),
4086            "poisoning thread must have panicked while holding the write guard"
4087        );
4088
4089        let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
4090        let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
4091        match result {
4092            Err(OpsLifecycleError::Internal(message)) => assert!(
4093                message.contains("ops lifecycle registry poisoned"),
4094                "unexpected cursor error message: {message}"
4095            ),
4096            other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
4097        }
4098    }
4099
4100    #[test]
4101    fn terminate_owner_only_targets_non_terminal_operations() {
4102        let registry = RuntimeOpsLifecycleRegistry::new();
4103
4104        let running_spec = background_spec("running");
4105        let running_id = running_spec.id.clone();
4106        registry.register_operation(running_spec).unwrap();
4107        registry.provisioning_succeeded(&running_id).unwrap();
4108
4109        let completed_spec = background_spec("completed");
4110        let completed_id = completed_spec.id.clone();
4111        registry.register_operation(completed_spec).unwrap();
4112        registry.provisioning_succeeded(&completed_id).unwrap();
4113        registry
4114            .complete_operation(
4115                &completed_id,
4116                OperationResult {
4117                    id: completed_id.clone(),
4118                    content: "done".into(),
4119                    is_error: false,
4120                    duration_ms: 1,
4121                    tokens_used: 0,
4122                },
4123            )
4124            .unwrap();
4125
4126        registry.terminate_owner("shutdown".into()).unwrap();
4127
4128        assert!(matches!(
4129            registry.snapshot(&running_id).unwrap().unwrap().status,
4130            OperationStatus::Terminated
4131        ));
4132        assert!(matches!(
4133            registry.snapshot(&completed_id).unwrap().unwrap().status,
4134            OperationStatus::Completed
4135        ));
4136    }
4137
4138    #[test]
4139    fn unregister_retirement_fences_detached_late_callbacks() {
4140        let registry = RuntimeOpsLifecycleRegistry::new();
4141        let running_spec = background_spec("detached-running");
4142        let running_id = running_spec.id.clone();
4143        registry.register_operation(running_spec).unwrap();
4144        registry.provisioning_succeeded(&running_id).unwrap();
4145
4146        registry
4147            .retire_owner_for_unregister("owner unregistered".into())
4148            .unwrap();
4149        assert!(matches!(
4150            registry.snapshot(&running_id).unwrap().unwrap().status,
4151            OperationStatus::Terminated
4152        ));
4153
4154        let late_result = OperationResult {
4155            id: running_id.clone(),
4156            content: "late detached completion".into(),
4157            is_error: false,
4158            duration_ms: 1,
4159            tokens_used: 0,
4160        };
4161        assert_eq!(
4162            registry.complete_operation(&running_id, late_result),
4163            Err(OpsLifecycleError::OwnerRetired)
4164        );
4165        assert_eq!(
4166            registry.register_operation(background_spec("late-new-op")),
4167            Err(OpsLifecycleError::OwnerRetired)
4168        );
4169        assert_eq!(
4170            registry.report_progress(
4171                &running_id,
4172                OperationProgressUpdate {
4173                    message: "late progress".into(),
4174                    percent: None,
4175                },
4176            ),
4177            Err(OpsLifecycleError::OwnerRetired)
4178        );
4179    }
4180
4181    #[test]
4182    fn collect_completed_drains_terminal_operations() {
4183        let registry = RuntimeOpsLifecycleRegistry::new();
4184
4185        let spec_a = background_spec("a");
4186        let id_a = spec_a.id.clone();
4187        registry.register_operation(spec_a).unwrap();
4188        registry.provisioning_succeeded(&id_a).unwrap();
4189        registry
4190            .complete_operation(
4191                &id_a,
4192                OperationResult {
4193                    id: id_a.clone(),
4194                    content: "done".into(),
4195                    is_error: false,
4196                    duration_ms: 1,
4197                    tokens_used: 0,
4198                },
4199            )
4200            .unwrap();
4201
4202        let spec_b = background_spec("b");
4203        let id_b = spec_b.id.clone();
4204        registry.register_operation(spec_b).unwrap();
4205
4206        let collected = registry.collect_completed().unwrap();
4207        assert_eq!(collected.len(), 1);
4208        assert_eq!(collected[0].0, id_a);
4209
4210        assert!(registry.snapshot(&id_a).unwrap().is_none());
4211        assert!(registry.snapshot(&id_b).unwrap().is_some());
4212
4213        let collected2 = registry.collect_completed().unwrap();
4214        assert!(collected2.is_empty());
4215    }
4216
4217    #[test]
4218    fn bounded_completed_retention_evicts_oldest() {
4219        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4220            max_completed: 3,
4221            max_concurrent: None,
4222        });
4223
4224        let mut ids = Vec::new();
4225        for i in 0..5 {
4226            let spec = background_spec(&format!("op-{i}"));
4227            let id = spec.id.clone();
4228            registry.register_operation(spec).unwrap();
4229            registry.provisioning_succeeded(&id).unwrap();
4230            registry
4231                .complete_operation(
4232                    &id,
4233                    OperationResult {
4234                        id: id.clone(),
4235                        content: format!("done-{i}"),
4236                        is_error: false,
4237                        duration_ms: 1,
4238                        tokens_used: 0,
4239                    },
4240                )
4241                .unwrap();
4242            ids.push(id);
4243        }
4244
4245        assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4246        assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4247        assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4248        assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4249        assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4250    }
4251
4252    #[test]
4253    fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4254        let registry = RuntimeOpsLifecycleRegistry::new();
4255
4256        let completed_spec = background_spec("completed");
4257        let completed_id = completed_spec.id.clone();
4258        registry.register_operation(completed_spec).unwrap();
4259        registry.provisioning_succeeded(&completed_id).unwrap();
4260        registry
4261            .complete_operation(
4262                &completed_id,
4263                OperationResult {
4264                    id: completed_id.clone(),
4265                    content: "done".into(),
4266                    is_error: false,
4267                    duration_ms: 1,
4268                    tokens_used: 0,
4269                },
4270            )
4271            .unwrap();
4272
4273        let running_spec = background_spec("running");
4274        let running_id = running_spec.id.clone();
4275        registry.register_operation(running_spec).unwrap();
4276        registry.provisioning_succeeded(&running_id).unwrap();
4277
4278        let cursor_state = meerkat_core::EpochCursorState::new();
4279        let snapshot = registry
4280            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4281            .unwrap();
4282        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4283
4284        assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4285        assert!(recovered.snapshot(&running_id).unwrap().is_none());
4286
4287        let collected = recovered.collect_completed().unwrap();
4288        assert_eq!(collected.len(), 1);
4289        assert_eq!(collected[0].0, completed_id);
4290    }
4291
4292    #[test]
4293    fn recovered_snapshot_retains_running_detached_job_wait_binding() {
4294        let registry = RuntimeOpsLifecycleRegistry::new();
4295        let owner_session_id = SessionId::new();
4296        let job_id = "job_recovery_wait";
4297        let operation_id = OperationId::for_detached_job_wait(&owner_session_id, "realm-a", job_id);
4298        let source = OperationSource::detached_job("realm-a", job_id);
4299        registry
4300            .register_operation(OperationSpec {
4301                id: operation_id.clone(),
4302                kind: OperationKind::DetachedJobWait,
4303                owner_session_id: owner_session_id.clone(),
4304                display_name: "await detached job".into(),
4305                source_label: "await_job".into(),
4306                operation_source: Some(source.clone()),
4307                child_session_id: None,
4308                expect_peer_channel: false,
4309            })
4310            .unwrap();
4311        registry.provisioning_succeeded(&operation_id).unwrap();
4312
4313        let cursor_state = meerkat_core::EpochCursorState::new();
4314        let snapshot = registry
4315            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4316            .unwrap();
4317        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4318        let wait = recovered
4319            .snapshot(&operation_id)
4320            .unwrap()
4321            .expect("running await binding survives recovery");
4322
4323        assert_eq!(wait.status, OperationStatus::Running);
4324        assert_eq!(wait.owner_session_id, owner_session_id);
4325        assert_eq!(wait.operation_source, Some(source));
4326        assert!(wait.terminal_outcome.is_none());
4327        assert!(
4328            recovered
4329                .completion_feed()
4330                .expect("completion feed")
4331                .list_since(0)
4332                .entries
4333                .is_empty(),
4334            "recovery of a wait binding must not mint completion"
4335        );
4336    }
4337
4338    #[test]
4339    fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4340        let registry = RuntimeOpsLifecycleRegistry::new();
4341
4342        let mut spec = background_spec("capacity");
4343        spec.kind = OperationKind::BackgroundToolCapacitySlot;
4344        let operation_id = spec.id.clone();
4345        registry.register_operation(spec).unwrap();
4346        registry.provisioning_succeeded(&operation_id).unwrap();
4347        registry.mark_retired(&operation_id).unwrap();
4348
4349        assert!(registry.snapshot(&operation_id).unwrap().is_none());
4350
4351        let cursor_state = meerkat_core::EpochCursorState::new();
4352        let snapshot = registry
4353            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4354            .unwrap();
4355        assert!(
4356            !snapshot
4357                .authority_state
4358                .operations
4359                .contains_key(&operation_id)
4360        );
4361        assert!(!snapshot.operation_specs.contains_key(&operation_id));
4362        assert!(snapshot.completion_entries.is_empty());
4363
4364        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4365        assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4366    }
4367
4368    #[test]
4369    fn rollback_unreturned_background_operation_restores_absence_without_completion() {
4370        let registry = RuntimeOpsLifecycleRegistry::new();
4371        let spec = background_spec("unreturned");
4372        let operation_id = spec.id.clone();
4373
4374        registry
4375            .register_operation(spec)
4376            .expect("register unreturned operation");
4377        registry
4378            .provisioning_succeeded(&operation_id)
4379            .expect("start unreturned operation");
4380        registry
4381            .rollback_unreturned_operation(&operation_id)
4382            .expect("generated rollback should restore absence");
4383
4384        assert!(
4385            registry
4386                .snapshot(&operation_id)
4387                .expect("snapshot after rollback")
4388                .is_none(),
4389            "never-returned operation must not retain public lifecycle state"
4390        );
4391        let feed = registry
4392            .completion_feed()
4393            .expect("runtime registry exposes completion feed");
4394        assert!(
4395            feed.list_since(0).entries.is_empty(),
4396            "never-returned operation must not mint a phantom completion"
4397        );
4398    }
4399
4400    #[test]
4401    fn recovered_snapshot_uses_authority_operation_source() {
4402        let registry = RuntimeOpsLifecycleRegistry::new();
4403        let child_session_id = SessionId::new();
4404        let operation_source = OperationSource::session_child(child_session_id.clone());
4405        let spec = OperationSpec {
4406            id: OperationId::new(),
4407            kind: OperationKind::MobMemberChild,
4408            owner_session_id: SessionId::new(),
4409            display_name: "source-recovery".into(),
4410            source_label: "test".into(),
4411            operation_source: Some(operation_source.clone()),
4412            child_session_id: Some(child_session_id),
4413            expect_peer_channel: true,
4414        };
4415        let operation_id = spec.id.clone();
4416
4417        registry.register_operation(spec).unwrap();
4418        registry.provisioning_succeeded(&operation_id).unwrap();
4419        registry.mark_retired(&operation_id).unwrap();
4420
4421        let cursor_state = meerkat_core::EpochCursorState::new();
4422        let mut snapshot = registry
4423            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4424            .unwrap();
4425        assert_eq!(
4426            snapshot
4427                .authority_state
4428                .operations
4429                .get(&operation_id)
4430                .and_then(|state| state.operation_source.as_ref()),
4431            Some(&operation_source)
4432        );
4433
4434        snapshot
4435            .operation_specs
4436            .get_mut(&operation_id)
4437            .expect("persisted spec")
4438            .operation_source = None;
4439        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4440        assert_eq!(
4441            recovered
4442                .snapshot(&operation_id)
4443                .unwrap()
4444                .unwrap()
4445                .operation_source,
4446            Some(operation_source)
4447        );
4448    }
4449
4450    #[test]
4451    fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4452        let registry = RuntimeOpsLifecycleRegistry::new();
4453        let child_session_id = SessionId::new();
4454        let operation_source = OperationSource::session_child(child_session_id.clone());
4455        let spec = OperationSpec {
4456            id: OperationId::new(),
4457            kind: OperationKind::MobMemberChild,
4458            owner_session_id: SessionId::new(),
4459            display_name: "source-drift".into(),
4460            source_label: "test".into(),
4461            operation_source: Some(operation_source),
4462            child_session_id: Some(child_session_id),
4463            expect_peer_channel: true,
4464        };
4465        let operation_id = spec.id.clone();
4466
4467        registry.register_operation(spec).unwrap();
4468        registry.provisioning_succeeded(&operation_id).unwrap();
4469        registry.mark_retired(&operation_id).unwrap();
4470
4471        let cursor_state = meerkat_core::EpochCursorState::new();
4472        let mut snapshot = registry
4473            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4474            .unwrap();
4475        snapshot
4476            .operation_specs
4477            .get_mut(&operation_id)
4478            .expect("persisted spec")
4479            .operation_source = Some(OperationSource::session_child(SessionId::new()));
4480
4481        let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4482            .expect_err("source mirror drift must fail recovery");
4483        assert!(
4484            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4485            "unexpected recovery error: {err:?}"
4486        );
4487    }
4488
4489    #[test]
4490    fn persisted_authority_state_serializes_explicit_no_operation_source() {
4491        let registry = RuntimeOpsLifecycleRegistry::new();
4492
4493        let spec = background_spec("explicit-no-source");
4494        let operation_id = spec.id.clone();
4495        registry.register_operation(spec).unwrap();
4496        registry.provisioning_succeeded(&operation_id).unwrap();
4497
4498        let cursor_state = meerkat_core::EpochCursorState::new();
4499        let snapshot = registry
4500            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4501            .unwrap();
4502        let value = serde_json::to_value(&snapshot).unwrap();
4503        let operations = value
4504            .get("authority_state")
4505            .and_then(|state| state.get("operations"))
4506            .and_then(serde_json::Value::as_object)
4507            .expect("serialized authority operations");
4508        let persisted_state = operations
4509            .values()
4510            .next()
4511            .and_then(serde_json::Value::as_object)
4512            .expect("serialized operation state");
4513
4514        assert!(
4515            persisted_state
4516                .get("operation_source")
4517                .is_some_and(serde_json::Value::is_null),
4518            "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4519        );
4520
4521        let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4522        assert_eq!(
4523            recovered_snapshot
4524                .authority_state
4525                .operations
4526                .get(&operation_id)
4527                .expect("round-tripped operation")
4528                .operation_source,
4529            None
4530        );
4531    }
4532
4533    #[test]
4534    fn persisted_authority_state_rejects_missing_operation_source_fact() {
4535        let registry = RuntimeOpsLifecycleRegistry::new();
4536
4537        let spec = background_spec("missing-source-fact");
4538        registry.register_operation(spec).unwrap();
4539
4540        let cursor_state = meerkat_core::EpochCursorState::new();
4541        let snapshot = registry
4542            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4543            .unwrap();
4544        let mut value = serde_json::to_value(&snapshot).unwrap();
4545        let operations = value
4546            .get_mut("authority_state")
4547            .and_then(|state| state.get_mut("operations"))
4548            .and_then(serde_json::Value::as_object_mut)
4549            .expect("serialized authority operations");
4550        let operation_state = operations
4551            .values_mut()
4552            .next()
4553            .and_then(serde_json::Value::as_object_mut)
4554            .expect("serialized operation state");
4555        assert!(operation_state.remove("operation_source").is_some());
4556
4557        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4558            .expect_err("missing generated source fact must fail recovery snapshot decoding");
4559        assert!(
4560            err.to_string().contains("operation_source"),
4561            "unexpected decode error: {err}"
4562        );
4563    }
4564
4565    #[test]
4566    fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4567        let registry = RuntimeOpsLifecycleRegistry::new();
4568
4569        let spec = background_spec("missing-feed-authority");
4570        let operation_id = spec.id.clone();
4571        registry.register_operation(spec).unwrap();
4572        registry.provisioning_succeeded(&operation_id).unwrap();
4573        registry
4574            .complete_operation(
4575                &operation_id,
4576                OperationResult {
4577                    id: operation_id.clone(),
4578                    content: "done".into(),
4579                    is_error: false,
4580                    duration_ms: 1,
4581                    tokens_used: 0,
4582                },
4583            )
4584            .unwrap();
4585
4586        let cursor_state = meerkat_core::EpochCursorState::new();
4587        let snapshot = registry
4588            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4589            .unwrap();
4590        let mut value = serde_json::to_value(&snapshot).unwrap();
4591        let authority_state = value
4592            .get_mut("authority_state")
4593            .and_then(serde_json::Value::as_object_mut)
4594            .expect("serialized authority state");
4595        assert!(authority_state.remove("completion_feed_entries").is_some());
4596
4597        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4598            .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4599        assert!(
4600            err.to_string().contains("completion_feed_entries"),
4601            "unexpected decode error: {err}"
4602        );
4603    }
4604
4605    #[test]
4606    fn public_child_session_projection_uses_authority_operation_source() {
4607        let registry = RuntimeOpsLifecycleRegistry::new();
4608        let authority_child_session_id = SessionId::new();
4609        let stale_shell_child_session_id = SessionId::new();
4610        let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4611        let spec = OperationSpec {
4612            id: OperationId::new(),
4613            kind: OperationKind::MobMemberChild,
4614            owner_session_id: SessionId::new(),
4615            display_name: "child-projection".into(),
4616            source_label: "test".into(),
4617            operation_source: Some(operation_source),
4618            child_session_id: Some(stale_shell_child_session_id),
4619            expect_peer_channel: true,
4620        };
4621        let operation_id = spec.id.clone();
4622
4623        registry.register_operation(spec).unwrap();
4624
4625        assert_eq!(
4626            registry
4627                .snapshot(&operation_id)
4628                .unwrap()
4629                .unwrap()
4630                .child_session_id,
4631            Some(authority_child_session_id.clone())
4632        );
4633
4634        let cursor_state = meerkat_core::EpochCursorState::new();
4635        let snapshot = registry
4636            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4637            .unwrap();
4638        assert_eq!(
4639            snapshot
4640                .operation_specs
4641                .get(&operation_id)
4642                .expect("persisted spec")
4643                .child_session_id,
4644            Some(authority_child_session_id)
4645        );
4646    }
4647
4648    #[test]
4649    fn generated_terminal_payload_projection_fails_closed() {
4650        let registry = RuntimeOpsLifecycleRegistry::new();
4651
4652        let spec = background_spec("terminal-payload-drift");
4653        let operation_id = spec.id.clone();
4654        registry.register_operation(spec).unwrap();
4655        registry.provisioning_succeeded(&operation_id).unwrap();
4656        registry
4657            .complete_operation(
4658                &operation_id,
4659                OperationResult {
4660                    id: operation_id.clone(),
4661                    content: "done".into(),
4662                    is_error: false,
4663                    duration_ms: 1,
4664                    tokens_used: 0,
4665                },
4666            )
4667            .unwrap();
4668
4669        {
4670            let mut state = registry.write_state().unwrap();
4671            let mut machine_state = state.dsl.0.state().clone();
4672            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4673            machine_state
4674                .op_terminal_payload
4675                .insert(operation_id_key, OperationTerminalOutcome::Retired);
4676            state.dsl = DslAuthority(Box::new(
4677                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4678            ));
4679        }
4680
4681        let err = match registry.register_watcher(&operation_id) {
4682            Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4683            Err(err) => err,
4684        };
4685        assert!(
4686            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4687            "unexpected watcher error: {err:?}"
4688        );
4689        let err = registry
4690            .snapshot(&operation_id)
4691            .expect_err("invalid generated terminal payload must reject public snapshot");
4692        assert!(
4693            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
4694            "unexpected public snapshot error: {err:?}"
4695        );
4696
4697        let cursor_state = meerkat_core::EpochCursorState::new();
4698        let err = match registry
4699            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4700        {
4701            Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
4702            Err(err) => err,
4703        };
4704        assert!(
4705            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4706            "unexpected snapshot error: {err:?}"
4707        );
4708
4709        let err = match registry.collect_completed() {
4710            Ok(_) => panic!("invalid generated terminal payload must reject collection"),
4711            Err(err) => err,
4712        };
4713        assert!(
4714            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4715            "unexpected collection error: {err:?}"
4716        );
4717    }
4718
4719    #[test]
4720    fn generated_terminal_payload_missing_projection_fails_closed() {
4721        let registry = RuntimeOpsLifecycleRegistry::new();
4722
4723        let spec = background_spec("terminal-payload-missing");
4724        let operation_id = spec.id.clone();
4725        registry.register_operation(spec).unwrap();
4726        registry.provisioning_succeeded(&operation_id).unwrap();
4727        registry
4728            .fail_operation(&operation_id, "boom".into())
4729            .unwrap();
4730
4731        {
4732            let mut state = registry.write_state().unwrap();
4733            let mut machine_state = state.dsl.0.state().clone();
4734            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4735            machine_state.op_terminal_payload.remove(&operation_id_key);
4736            state.dsl = DslAuthority(Box::new(
4737                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4738            ));
4739        }
4740
4741        let err = match registry.register_watcher(&operation_id) {
4742            Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
4743            Err(err) => err,
4744        };
4745        assert!(
4746            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4747            "unexpected watcher error: {err:?}"
4748        );
4749        let err = registry
4750            .snapshot(&operation_id)
4751            .expect_err("missing generated terminal payload must reject public snapshot");
4752        assert!(
4753            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4754            "unexpected public snapshot error: {err:?}"
4755        );
4756    }
4757
4758    #[test]
4759    fn generated_terminal_status_without_outcome_fails_closed() {
4760        let registry = RuntimeOpsLifecycleRegistry::new();
4761
4762        let spec = background_spec("terminal-outcome-missing");
4763        let operation_id = spec.id.clone();
4764        registry.register_operation(spec).unwrap();
4765        registry.provisioning_succeeded(&operation_id).unwrap();
4766        registry
4767            .fail_operation(&operation_id, "boom".into())
4768            .unwrap();
4769
4770        {
4771            let mut state = registry.write_state().unwrap();
4772            let mut machine_state = state.dsl.0.state().clone();
4773            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4774            machine_state.op_terminal_outcomes.remove(&operation_id_key);
4775            state.dsl = DslAuthority(Box::new(
4776                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4777            ));
4778        }
4779
4780        let err = registry
4781            .snapshot(&operation_id)
4782            .expect_err("terminal status without outcome must reject public snapshot");
4783        assert!(
4784            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4785            "unexpected public snapshot error: {err:?}"
4786        );
4787
4788        let err = match registry.collect_completed() {
4789            Ok(_) => panic!("terminal status without outcome must reject collection"),
4790            Err(err) => err,
4791        };
4792        assert!(
4793            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4794            "unexpected collection error: {err:?}"
4795        );
4796    }
4797
4798    #[test]
4799    fn generated_operation_source_projection_fails_closed() {
4800        let registry = RuntimeOpsLifecycleRegistry::new();
4801        let child_session_id = SessionId::new();
4802        let operation_source = OperationSource::session_child(child_session_id.clone());
4803        let spec = OperationSpec {
4804            id: OperationId::new(),
4805            kind: OperationKind::MobMemberChild,
4806            owner_session_id: SessionId::new(),
4807            display_name: "source-authority-drift".into(),
4808            source_label: "test".into(),
4809            operation_source: Some(operation_source),
4810            child_session_id: Some(child_session_id),
4811            expect_peer_channel: true,
4812        };
4813        let operation_id = spec.id.clone();
4814
4815        registry.register_operation(spec).unwrap();
4816
4817        {
4818            let mut state = registry.write_state().unwrap();
4819            let mut machine_state = state.dsl.0.state().clone();
4820            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4821            machine_state
4822                .op_sources
4823                .get_mut(&operation_id_key)
4824                .expect("generated operation source")
4825                .session_id = None;
4826            state.dsl = DslAuthority(Box::new(
4827                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4828            ));
4829        }
4830
4831        let err = registry
4832            .snapshot(&operation_id)
4833            .expect_err("invalid generated operation source must reject public snapshot");
4834        assert!(
4835            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4836            "unexpected public snapshot error: {err:?}"
4837        );
4838        let err = registry
4839            .list_operations()
4840            .expect_err("invalid generated operation source must reject public operation list");
4841        assert!(
4842            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4843            "unexpected operation list error: {err:?}"
4844        );
4845
4846        let cursor_state = meerkat_core::EpochCursorState::new();
4847        let err = match registry
4848            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4849        {
4850            Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
4851            Err(err) => err,
4852        };
4853        assert!(
4854            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4855            "unexpected snapshot error: {err:?}"
4856        );
4857    }
4858
4859    #[test]
4860    fn generated_operation_id_projection_fails_closed() {
4861        let registry = RuntimeOpsLifecycleRegistry::new();
4862
4863        {
4864            let mut state = registry.write_state().unwrap();
4865            let mut machine_state = state.dsl.0.state().clone();
4866            machine_state.op_statuses.insert(
4867                "not-json-operation-id".into(),
4868                mm_dsl::OperationStatus::Running,
4869            );
4870            state.dsl = DslAuthority(Box::new(
4871                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4872            ));
4873        }
4874
4875        let err = registry
4876            .list_operations()
4877            .expect_err("invalid generated operation id must reject public operation list");
4878        assert!(
4879            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4880            "unexpected operation list error: {err:?}"
4881        );
4882
4883        let cursor_state = meerkat_core::EpochCursorState::new();
4884        let err = match registry
4885            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4886        {
4887            Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
4888            Err(err) => err,
4889        };
4890        assert!(
4891            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4892            "unexpected persistence snapshot error: {err:?}"
4893        );
4894    }
4895
4896    #[test]
4897    fn generated_missing_kind_projection_fails_closed() {
4898        let registry = RuntimeOpsLifecycleRegistry::new();
4899        let spec = background_spec("missing-kind");
4900        let operation_id = spec.id.clone();
4901        registry.register_operation(spec).unwrap();
4902
4903        {
4904            let mut state = registry.write_state().unwrap();
4905            let mut machine_state = state.dsl.0.state().clone();
4906            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4907            machine_state.op_kinds.remove(&operation_id_key);
4908            state.dsl = DslAuthority(Box::new(
4909                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4910            ));
4911        }
4912
4913        let err = registry
4914            .snapshot(&operation_id)
4915            .expect_err("missing generated kind must reject public snapshot");
4916        assert!(
4917            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4918            "unexpected public snapshot error: {err:?}"
4919        );
4920        let err = registry
4921            .list_operations()
4922            .expect_err("missing generated kind must reject public list");
4923        assert!(
4924            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4925            "unexpected public list error: {err:?}"
4926        );
4927
4928        let cursor_state = meerkat_core::EpochCursorState::new();
4929        let err = registry
4930            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4931            .expect_err("missing generated kind must reject persistence snapshot");
4932        assert!(
4933            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4934            "unexpected persistence snapshot error: {err:?}"
4935        );
4936    }
4937
4938    #[test]
4939    fn generated_missing_status_projection_fails_closed() {
4940        let registry = RuntimeOpsLifecycleRegistry::new();
4941        let spec = background_spec("missing-status");
4942        let operation_id = spec.id.clone();
4943        registry.register_operation(spec).unwrap();
4944
4945        {
4946            let mut state = registry.write_state().unwrap();
4947            let mut machine_state = state.dsl.0.state().clone();
4948            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4949            machine_state.op_statuses.remove(&operation_id_key);
4950            state.dsl = DslAuthority(Box::new(
4951                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4952            ));
4953        }
4954
4955        let err = registry
4956            .snapshot(&operation_id)
4957            .expect_err("missing generated status must reject public snapshot");
4958        assert!(
4959            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4960            "unexpected public snapshot error: {err:?}"
4961        );
4962        let err = registry
4963            .list_operations()
4964            .expect_err("missing generated status must reject public list");
4965        assert!(
4966            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4967            "unexpected public list error: {err:?}"
4968        );
4969        let err = registry
4970            .classify_operation_public_result(&operation_id)
4971            .expect_err("missing generated status must reject public-result classification");
4972        assert!(
4973            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4974            "unexpected public-result error: {err:?}"
4975        );
4976    }
4977
4978    #[test]
4979    fn generated_retiring_public_result_remains_running_until_terminal() {
4980        let registry = RuntimeOpsLifecycleRegistry::new();
4981        let spec = background_spec("retiring-public-result");
4982        let operation_id = spec.id.clone();
4983        registry.register_operation(spec).unwrap();
4984        registry.provisioning_succeeded(&operation_id).unwrap();
4985        registry.request_retire(&operation_id).unwrap();
4986
4987        let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
4988        assert_eq!(snapshot.status, OperationStatus::Retiring);
4989        assert!(snapshot.terminal_outcome.is_none());
4990        assert!(!snapshot.terminal);
4991        assert_eq!(
4992            snapshot.public_result_class,
4993            OperationPublicResultClass::Running
4994        );
4995        assert_eq!(
4996            registry
4997                .classify_operation_public_result(&operation_id)
4998                .unwrap(),
4999            OperationPublicResultClass::Running
5000        );
5001    }
5002
5003    #[test]
5004    fn generated_missing_peer_ready_projection_fails_closed() {
5005        let registry = RuntimeOpsLifecycleRegistry::new();
5006        let spec = background_spec("missing-peer-ready");
5007        let operation_id = spec.id.clone();
5008        registry.register_operation(spec).unwrap();
5009
5010        {
5011            let mut state = registry.write_state().unwrap();
5012            let mut machine_state = state.dsl.0.state().clone();
5013            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5014            machine_state.op_peer_ready.remove(&operation_id_key);
5015            state.dsl = DslAuthority(Box::new(
5016                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5017            ));
5018        }
5019
5020        let err = registry
5021            .snapshot(&operation_id)
5022            .expect_err("missing generated peer-ready fact must reject public snapshot");
5023        assert!(
5024            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
5025            "unexpected public snapshot error: {err:?}"
5026        );
5027
5028        let cursor_state = meerkat_core::EpochCursorState::new();
5029        let err = registry
5030            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5031            .expect_err("missing generated peer-ready fact must reject persistence snapshot");
5032        assert!(
5033            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
5034            "unexpected persistence snapshot error: {err:?}"
5035        );
5036    }
5037
5038    #[test]
5039    fn generated_missing_progress_count_projection_fails_closed() {
5040        let registry = RuntimeOpsLifecycleRegistry::new();
5041        let spec = background_spec("missing-progress-count");
5042        let operation_id = spec.id.clone();
5043        registry.register_operation(spec).unwrap();
5044
5045        {
5046            let mut state = registry.write_state().unwrap();
5047            let mut machine_state = state.dsl.0.state().clone();
5048            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5049            machine_state.op_progress_counts.remove(&operation_id_key);
5050            state.dsl = DslAuthority(Box::new(
5051                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5052            ));
5053        }
5054
5055        let err = registry
5056            .snapshot(&operation_id)
5057            .expect_err("missing generated progress count must reject public snapshot");
5058        assert!(
5059            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
5060            "unexpected public snapshot error: {err:?}"
5061        );
5062
5063        let cursor_state = meerkat_core::EpochCursorState::new();
5064        let err = registry
5065            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5066            .expect_err("missing generated progress count must reject persistence snapshot");
5067        assert!(
5068            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
5069            "unexpected persistence snapshot error: {err:?}"
5070        );
5071    }
5072
5073    #[test]
5074    fn generated_terminal_sequence_missing_persistence_fails_closed() {
5075        let registry = RuntimeOpsLifecycleRegistry::new();
5076        let spec = background_spec("terminal-sequence-missing");
5077        let operation_id = spec.id.clone();
5078        registry.register_operation(spec).unwrap();
5079        registry.provisioning_succeeded(&operation_id).unwrap();
5080        registry
5081            .complete_operation(
5082                &operation_id,
5083                OperationResult {
5084                    id: operation_id.clone(),
5085                    content: "done".into(),
5086                    is_error: false,
5087                    duration_ms: 1,
5088                    tokens_used: 0,
5089                },
5090            )
5091            .unwrap();
5092
5093        {
5094            let mut state = registry.write_state().unwrap();
5095            let mut machine_state = state.dsl.0.state().clone();
5096            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5097            machine_state.op_completion_seq.remove(&operation_id_key);
5098            state.dsl = DslAuthority(Box::new(
5099                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5100            ));
5101        }
5102
5103        let cursor_state = meerkat_core::EpochCursorState::new();
5104        let err = registry
5105            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5106            .expect_err("missing generated terminal sequence must reject persistence snapshot");
5107        assert!(
5108            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
5109            "unexpected persistence snapshot error: {err:?}"
5110        );
5111    }
5112
5113    #[test]
5114    fn generated_record_without_shell_projection_fails_closed() {
5115        let registry = RuntimeOpsLifecycleRegistry::new();
5116        let spec = background_spec("missing-shell-record");
5117        let operation_id = spec.id.clone();
5118        registry.register_operation(spec).unwrap();
5119
5120        {
5121            let mut state = registry.write_state().unwrap();
5122            state.records.remove(&operation_id);
5123        }
5124
5125        let err = registry
5126            .snapshot(&operation_id)
5127            .expect_err("generated operation without shell record must reject public snapshot");
5128        assert!(
5129            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5130            "unexpected public snapshot error: {err:?}"
5131        );
5132        let err = registry
5133            .list_operations()
5134            .expect_err("generated operation without shell record must reject public list");
5135        assert!(
5136            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5137            "unexpected public list error: {err:?}"
5138        );
5139    }
5140
5141    #[test]
5142    fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
5143        let registry = RuntimeOpsLifecycleRegistry::new();
5144        let child_session_id = SessionId::new();
5145        let operation_source = OperationSource::session_child(child_session_id.clone());
5146        let spec = OperationSpec {
5147            id: OperationId::new(),
5148            kind: OperationKind::MobMemberChild,
5149            owner_session_id: SessionId::new(),
5150            display_name: "child-drift".into(),
5151            source_label: "test".into(),
5152            operation_source: Some(operation_source),
5153            child_session_id: Some(child_session_id.clone()),
5154            expect_peer_channel: true,
5155        };
5156        let operation_id = spec.id.clone();
5157
5158        registry.register_operation(spec).unwrap();
5159        registry.provisioning_succeeded(&operation_id).unwrap();
5160        registry.mark_retired(&operation_id).unwrap();
5161
5162        let cursor_state = meerkat_core::EpochCursorState::new();
5163        let mut snapshot = registry
5164            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5165            .unwrap();
5166        snapshot
5167            .operation_specs
5168            .get_mut(&operation_id)
5169            .expect("persisted spec")
5170            .child_session_id = Some(SessionId::new());
5171
5172        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5173        assert_eq!(
5174            recovered
5175                .snapshot(&operation_id)
5176                .unwrap()
5177                .unwrap()
5178                .child_session_id,
5179            Some(child_session_id)
5180        );
5181    }
5182
5183    #[test]
5184    fn completion_wake_class_is_generated_by_operation_kind() {
5185        let registry = RuntimeOpsLifecycleRegistry::new();
5186        let operation_id = OperationId::new();
5187
5188        assert_eq!(
5189            registry
5190                .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
5191                .unwrap(),
5192            OperationCompletionWakeClass::Wake
5193        );
5194        assert_eq!(
5195            registry
5196                .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
5197                .unwrap(),
5198            OperationCompletionWakeClass::Ignore
5199        );
5200        assert_eq!(
5201            registry
5202                .classify_operation_completion_wake(
5203                    &operation_id,
5204                    OperationKind::BackgroundToolCapacitySlot,
5205                )
5206                .unwrap(),
5207            OperationCompletionWakeClass::Ignore
5208        );
5209    }
5210
5211    #[test]
5212    fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
5213        let registry = RuntimeOpsLifecycleRegistry::new();
5214
5215        let running_spec = background_spec("running");
5216        let running_id = running_spec.id.clone();
5217        registry.register_operation(running_spec.clone()).unwrap();
5218        registry.provisioning_succeeded(&running_id).unwrap();
5219
5220        let cursor_state = meerkat_core::EpochCursorState::new();
5221        let mut snapshot = registry
5222            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5223            .unwrap();
5224        snapshot.completion_entries.push(CompletionEntry {
5225            seq: 1,
5226            operation_id: running_id.clone(),
5227            kind: running_spec.kind,
5228            display_name: running_spec.display_name,
5229            terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5230                id: running_id,
5231                content: "phantom".into(),
5232                is_error: false,
5233                duration_ms: 1,
5234                tokens_used: 0,
5235            }),
5236            completed_at_ms: None,
5237        });
5238
5239        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5240            Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5241            Err(err) => err,
5242        };
5243        assert!(
5244            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5245            "unexpected recovery error: {err:?}"
5246        );
5247    }
5248
5249    #[test]
5250    fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5251        let registry = RuntimeOpsLifecycleRegistry::new();
5252        let phantom_id = OperationId::new();
5253        {
5254            let state = registry.read_state().unwrap();
5255            state.feed_buffer.push(CompletionEntry {
5256                seq: 1,
5257                operation_id: phantom_id.clone(),
5258                kind: OperationKind::BackgroundToolOp,
5259                display_name: "phantom".into(),
5260                terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5261                    id: phantom_id,
5262                    content: "phantom".into(),
5263                    is_error: false,
5264                    duration_ms: 1,
5265                    tokens_used: 0,
5266                }),
5267                completed_at_ms: None,
5268            });
5269        }
5270
5271        let cursor_state = meerkat_core::EpochCursorState::new();
5272        let err = match registry
5273            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5274        {
5275            Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5276            Err(err) => err,
5277        };
5278        assert!(
5279            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5280            "unexpected capture error: {err:?}"
5281        );
5282    }
5283
5284    #[test]
5285    fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5286        let registry = RuntimeOpsLifecycleRegistry::new();
5287
5288        let spec = background_spec("terminal");
5289        let operation_id = spec.id.clone();
5290        registry.register_operation(spec).unwrap();
5291        registry.provisioning_succeeded(&operation_id).unwrap();
5292        registry
5293            .complete_operation(
5294                &operation_id,
5295                OperationResult {
5296                    id: operation_id.clone(),
5297                    content: "done".into(),
5298                    is_error: false,
5299                    duration_ms: 1,
5300                    tokens_used: 0,
5301                },
5302            )
5303            .unwrap();
5304
5305        let cursor_state = meerkat_core::EpochCursorState::new();
5306        let mut snapshot = registry
5307            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5308            .unwrap();
5309        let phantom_id = OperationId::new();
5310        let phantom_result = OperationResult {
5311            id: phantom_id.clone(),
5312            content: "phantom".into(),
5313            is_error: false,
5314            duration_ms: 1,
5315            tokens_used: 0,
5316        };
5317        let phantom_entry = CompletionFeedCanonicalState {
5318            seq: snapshot.authority_state.next_completion_seq + 1,
5319            kind: OperationKind::BackgroundToolOp,
5320            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5321        };
5322        snapshot
5323            .authority_state
5324            .completion_feed_entries
5325            .insert(phantom_id.clone(), phantom_entry);
5326        snapshot.completion_entries.push(CompletionEntry {
5327            seq: snapshot.authority_state.next_completion_seq + 1,
5328            operation_id: phantom_id.clone(),
5329            kind: OperationKind::BackgroundToolOp,
5330            display_name: "phantom".into(),
5331            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5332            completed_at_ms: None,
5333        });
5334
5335        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5336            Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5337            Err(err) => err,
5338        };
5339        assert!(
5340            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5341            "unexpected recovery error: {err:?}"
5342        );
5343    }
5344
5345    #[test]
5346    fn recovered_completed_order_uses_generated_completion_sequences() {
5347        let registry = RuntimeOpsLifecycleRegistry::new();
5348
5349        let spec_a = background_spec("a");
5350        let id_a = spec_a.id.clone();
5351        registry.register_operation(spec_a).unwrap();
5352        registry.provisioning_succeeded(&id_a).unwrap();
5353        registry
5354            .complete_operation(
5355                &id_a,
5356                OperationResult {
5357                    id: id_a.clone(),
5358                    content: "a".into(),
5359                    is_error: false,
5360                    duration_ms: 1,
5361                    tokens_used: 0,
5362                },
5363            )
5364            .unwrap();
5365
5366        let spec_b = background_spec("b");
5367        let id_b = spec_b.id.clone();
5368        registry.register_operation(spec_b).unwrap();
5369        registry.provisioning_succeeded(&id_b).unwrap();
5370        registry
5371            .complete_operation(
5372                &id_b,
5373                OperationResult {
5374                    id: id_b.clone(),
5375                    content: "b".into(),
5376                    is_error: false,
5377                    duration_ms: 1,
5378                    tokens_used: 0,
5379                },
5380            )
5381            .unwrap();
5382
5383        let cursor_state = meerkat_core::EpochCursorState::new();
5384        let mut snapshot = registry
5385            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5386            .unwrap();
5387        snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5388
5389        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5390        let collected = recovered.collect_completed().unwrap();
5391
5392        assert_eq!(collected[0].0, id_a);
5393        assert_eq!(collected[1].0, id_b);
5394    }
5395
5396    #[test]
5397    fn max_concurrent_enforcement() {
5398        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5399            max_completed: DEFAULT_MAX_COMPLETED,
5400            max_concurrent: Some(2),
5401        });
5402
5403        let spec_a = background_spec("a");
5404        let id_a = spec_a.id.clone();
5405        registry.register_operation(spec_a).unwrap();
5406
5407        let spec_b = background_spec("b");
5408        registry.register_operation(spec_b).unwrap();
5409
5410        let spec_c = background_spec("c");
5411        let result = registry.register_operation(spec_c);
5412        assert!(matches!(
5413            result,
5414            Err(OpsLifecycleError::MaxConcurrentExceeded {
5415                limit: 2,
5416                active: 2,
5417            })
5418        ));
5419
5420        registry.provisioning_succeeded(&id_a).unwrap();
5421        registry
5422            .complete_operation(
5423                &id_a,
5424                OperationResult {
5425                    id: id_a.clone(),
5426                    content: "done".into(),
5427                    is_error: false,
5428                    duration_ms: 1,
5429                    tokens_used: 0,
5430                },
5431            )
5432            .unwrap();
5433
5434        let spec_d = background_spec("d");
5435        assert!(registry.register_operation(spec_d).is_ok());
5436    }
5437
5438    #[test]
5439    fn snapshot_includes_timestamps() {
5440        let registry = RuntimeOpsLifecycleRegistry::new();
5441        let spec = background_spec("timed");
5442        let op_id = spec.id.clone();
5443        registry.register_operation(spec).unwrap();
5444
5445        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5446        assert!(snap1.created_at_ms > 0);
5447        assert!(snap1.started_at_ms.is_none());
5448        assert!(snap1.completed_at_ms.is_none());
5449        assert!(snap1.elapsed_ms.is_none());
5450
5451        registry.provisioning_succeeded(&op_id).unwrap();
5452        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5453        assert!(snap2.started_at_ms.is_some());
5454        assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5455
5456        registry
5457            .complete_operation(
5458                &op_id,
5459                OperationResult {
5460                    id: op_id.clone(),
5461                    content: "done".into(),
5462                    is_error: false,
5463                    duration_ms: 1,
5464                    tokens_used: 0,
5465                },
5466            )
5467            .unwrap();
5468        let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5469        assert!(snap3.completed_at_ms.is_some());
5470        assert!(snap3.elapsed_ms.is_some());
5471        assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5472    }
5473
5474    #[test]
5475    fn snapshot_includes_peer_handle() {
5476        let registry = RuntimeOpsLifecycleRegistry::new();
5477        let child_session_id = SessionId::new();
5478        let spec = OperationSpec {
5479            id: OperationId::new(),
5480            kind: OperationKind::MobMemberChild,
5481            owner_session_id: SessionId::new(),
5482            display_name: "peer-test".into(),
5483            source_label: "test".into(),
5484            operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5485            child_session_id: Some(child_session_id),
5486            expect_peer_channel: true,
5487        };
5488        let op_id = spec.id.clone();
5489        registry.register_operation(spec).unwrap();
5490        registry.provisioning_succeeded(&op_id).unwrap();
5491
5492        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5493        assert!(snap1.peer_handle.is_none());
5494
5495        let handle = OperationPeerHandle {
5496            peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5497            trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5498                "member-x",
5499                PeerId::new(),
5500                "inproc://x",
5501            )
5502            .unwrap(),
5503        };
5504        registry.peer_ready(&op_id, handle).unwrap();
5505
5506        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5507        assert_eq!(
5508            snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5509            "member-x"
5510        );
5511    }
5512}