Skip to main content

meerkat_runtime/
ops_lifecycle.rs

1//! In-memory runtime implementation of the shared async-operation lifecycle seam.
2//!
3//! Per-operation canonical lifecycle state lives in the MeerkatMachine DSL
4//! authority (`op_statuses`, `op_terminal_outcomes`, `op_kinds`,
5//! `op_sources`, `op_peer_ready`, `op_progress_counts`, `active_op_count`,
6//! `wait_active`, `wait_operation_ids`). This shell layer owns pure mechanics: watcher
7//! channels, timestamps, peer handles, snapshot assembly, FIFO eviction
8//! bookkeeping, the completion feed buffer, and typed delivery of generated
9//! admission/rejection feedback.
10//!
11//! Per-transition legality ("is `CompleteOp` legal on a `Provisioning` op?")
12//! is NOT owned by the shell — it lives in the DSL's `from_status_valid`
13//! guards on each op-lifecycle transition. The shell's only job on a
14//! `GuardRejected` rejection is to ask the generated rejection resolver for
15//! the public result class.
16
17use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24    CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31    CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32    OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33    OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34    OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35    OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36    WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43// ---------------------------------------------------------------------------
44// Serde-only persisted canonical state shells
45// ---------------------------------------------------------------------------
46//
47// These structures preserve the persisted fact shape of `PersistedOpsSnapshot`.
48// They are pure serde shells — no methods beyond read-only field accessors,
49// no authority behavior. Optional generated facts that can be explicitly
50// absent still serialize as present `null` so recovery can distinguish
51// generated "none" from a missing persisted fact.
52
53fn deserialize_required_operation_source<'de, D>(
54    deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57    D: serde::Deserializer<'de>,
58{
59    <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62/// Canonical per-operation state as captured in persisted snapshots.
63#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65    status: OperationStatus,
66    kind: OperationKind,
67    #[serde(deserialize_with = "deserialize_required_operation_source")]
68    operation_source: Option<OperationSource>,
69    peer_ready: bool,
70    progress_count: u32,
71    watcher_count: u32,
72    terminal_outcome: Option<OperationTerminalOutcome>,
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    completion_sequence: Option<CompletionSeq>,
75    terminal_buffered: bool,
76}
77
78/// Generated-owned public completion feed fact captured from DSL authority.
79#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81    seq: CompletionSeq,
82    kind: OperationKind,
83    terminal_outcome: OperationTerminalOutcome,
84}
85
86/// Canonical registry-level state as captured in persisted snapshots.
87#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89    operations: HashMap<OperationId, OperationCanonicalState>,
90    completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91    completed_order: VecDeque<OperationId>,
92    max_completed: usize,
93    max_concurrent: Option<usize>,
94    active_count: usize,
95    wait_request_id: Option<WaitRequestId>,
96    wait_operation_ids: Vec<OperationId>,
97    next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101    /// Maximum completed operations retained at capture time.
102    pub fn max_completed(&self) -> usize {
103        self.max_completed
104    }
105
106    /// Maximum concurrent non-terminal operations at capture time.
107    pub fn max_concurrent(&self) -> Option<usize> {
108        self.max_concurrent
109    }
110
111    /// Number of operations captured in the snapshot.
112    pub fn operation_count(&self) -> usize {
113        self.operations.len()
114    }
115
116    /// Number of generated-owned public completion feed entries captured.
117    pub fn completion_feed_count(&self) -> usize {
118        self.completion_feed_entries.len()
119    }
120}
121
122// ---------------------------------------------------------------------------
123// Serializable snapshot for persistence
124// ---------------------------------------------------------------------------
125
126/// Serializable snapshot of the ops lifecycle registry state.
127///
128/// Captured on terminal transitions for durable persistence. Contains
129/// canonical state, operation specs, persisted completion feed entries, and
130/// consumer cursor values. Wire format preserved verbatim from legacy
131/// runtime versions for backward compatibility.
132#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134    /// Epoch identity at capture time.
135    pub epoch_id: meerkat_core::RuntimeEpochId,
136    /// Canonical machine-owned state at capture time.
137    pub authority_state: RegistryCanonicalState,
138    /// Per-operation specs for shell record reconstruction.
139    pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140    /// Persisted completion feed projection metadata. Canonical feed truth is
141    /// captured in `authority_state.completion_feed_entries`.
142    pub completion_entries: Vec<CompletionEntry>,
143    /// Consumer cursor snapshot at capture time.
144    pub cursors: meerkat_core::EpochCursorSnapshot,
145}
146
147#[derive(Debug)]
148pub struct OpsLifecyclePersistenceRequest {
149    snapshot: PersistedOpsSnapshot,
150    result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
151}
152
153impl OpsLifecyclePersistenceRequest {
154    pub fn snapshot(&self) -> &PersistedOpsSnapshot {
155        &self.snapshot
156    }
157
158    pub fn complete(self, result: Result<(), OpsLifecycleError>) {
159        let _ = self.result_tx.send(result);
160    }
161}
162
163// ---------------------------------------------------------------------------
164// Concrete completion feed buffer
165// ---------------------------------------------------------------------------
166
167/// Shared inner state of the completion feed buffer.
168///
169/// Protected by the registry's `RwLock<ShellState>` for writes, and by its
170/// own `RwLock` for reads by external consumers (agent boundary, idle wake).
171#[derive(Debug)]
172struct FeedBufferInner {
173    entries: VecDeque<CompletionEntry>,
174    watermark: CompletionSeq,
175    max_retained: usize,
176}
177
178/// Shared completion feed buffer owned by the runtime registry.
179///
180/// The registry writes entries under its own write lock. External consumers
181/// read through the [`RuntimeCompletionFeed`] handle.
182#[derive(Debug)]
183struct FeedBuffer {
184    inner: RwLock<FeedBufferInner>,
185    /// Atomic mirror of watermark for lock-free `watermark()` reads.
186    watermark_atomic: AtomicU64,
187    /// Notifies all waiters when new entries are appended.
188    notify: tokio::sync::Notify,
189}
190
191impl FeedBuffer {
192    fn new(max_retained: usize) -> Self {
193        Self {
194            inner: RwLock::new(FeedBufferInner {
195                entries: VecDeque::new(),
196                watermark: 0,
197                max_retained,
198            }),
199            watermark_atomic: AtomicU64::new(0),
200            notify: tokio::sync::Notify::new(),
201        }
202    }
203
204    fn push(&self, entry: CompletionEntry) {
205        let mut inner = self
206            .inner
207            .write()
208            .unwrap_or_else(std::sync::PoisonError::into_inner);
209        let seq = entry.seq;
210        inner.entries.push_back(entry);
211        inner.watermark = seq;
212
213        // Evict oldest if over capacity.
214        while inner.entries.len() > inner.max_retained {
215            inner.entries.pop_front();
216        }
217
218        drop(inner);
219
220        self.watermark_atomic.store(seq, Ordering::Release);
221        self.notify.notify_waiters();
222    }
223}
224
225/// Read-only handle to the runtime completion feed.
226///
227/// Implements [`CompletionFeed`] for external consumers. Obtained via
228/// [`RuntimeOpsLifecycleRegistry::completion_feed()`].
229#[derive(Debug, Clone)]
230pub struct RuntimeCompletionFeed {
231    buffer: Arc<FeedBuffer>,
232}
233
234impl CompletionFeed for RuntimeCompletionFeed {
235    fn watermark(&self) -> CompletionSeq {
236        self.buffer.watermark_atomic.load(Ordering::Acquire)
237    }
238
239    fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
240        let inner = self
241            .buffer
242            .inner
243            .read()
244            .unwrap_or_else(std::sync::PoisonError::into_inner);
245        let entries: Vec<CompletionEntry> = inner
246            .entries
247            .iter()
248            .filter(|e| e.seq > after_seq)
249            .cloned()
250            .collect();
251        let watermark = inner.watermark;
252        CompletionBatch { entries, watermark }
253    }
254
255    fn wait_for_advance(
256        &self,
257        after_seq: CompletionSeq,
258    ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
259        Box::pin(async move {
260            loop {
261                // Register the waiter BEFORE reading the watermark.
262                // notify_waiters() in push() only wakes already-registered
263                // listeners — if we read first and push() lands between the
264                // read and notified().await, the wake is lost.
265                let notified = self.buffer.notify.notified();
266                let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
267                if current > after_seq {
268                    return current;
269                }
270                notified.await;
271            }
272        })
273    }
274}
275
276// ---------------------------------------------------------------------------
277// Shell-only per-operation record (not part of canonical machine state)
278// ---------------------------------------------------------------------------
279
280#[derive(Debug)]
281struct OperationCompletionNotifier {
282    tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
283}
284
285impl OperationCompletionNotifier {
286    fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
287        Self { tx }
288    }
289
290    fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
291        let _ = self.tx.send(outcome.clone());
292    }
293}
294
295fn operation_completion_watch_from_receiver(
296    rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
297) -> OperationCompletionWatch {
298    Box::pin(async move {
299        rx.await
300            .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
301    })
302}
303
304fn resolved_operation_completion_watch(
305    outcome: OperationTerminalOutcome,
306) -> OperationCompletionWatch {
307    Box::pin(async move { Ok(outcome) })
308}
309
310/// Shell-owned data for a single operation. Canonical lifecycle state lives in
311/// the DSL authority; this struct holds I/O concerns that the DSL has no
312/// knowledge of.
313#[derive(Debug)]
314struct ShellRecord {
315    spec: OperationSpec,
316    peer_handle: Option<OperationPeerHandle>,
317    /// Private waiter plumbing. Notifiers are drained only from
318    /// `finalize_terminal()` after the generated authority has accepted and
319    /// stored a terminal outcome.
320    watchers: Vec<OperationCompletionNotifier>,
321    // Monotonic timestamps for elapsed computation
322    created_at: Instant,
323    started_at: Option<Instant>,
324    completed_at: Option<Instant>,
325    // Wall-clock anchor captured at creation for epoch millis
326    created_at_wall: SystemTime,
327}
328
329#[derive(Debug)]
330struct PendingWaitState {
331    wait_request_id: WaitRequestId,
332    sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
333}
334
335enum WaitAllAuthorityPlan {
336    AlreadySatisfied(WaitAllSatisfied),
337    ActivateBarrier,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq)]
341enum RecoveredOperationRecordDisposition {
342    Retain,
343    Discard,
344}
345
346impl ShellRecord {
347    fn new(spec: OperationSpec) -> Self {
348        Self {
349            spec,
350            peer_handle: None,
351            watchers: Vec::new(),
352            created_at: Instant::now(),
353            started_at: None,
354            completed_at: None,
355            created_at_wall: SystemTime::now(),
356        }
357    }
358
359    fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
360        wall_anchor
361            .duration_since(UNIX_EPOCH)
362            .map(|d| d.as_millis() as u64)
363            .unwrap_or(0)
364    }
365
366    fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
367        // Compute wall time for a given instant using the wall-clock anchor:
368        // wall_time = created_at_wall + (instant - created_at)
369        let offset = instant.saturating_duration_since(self.created_at);
370        let wall = self.created_at_wall + offset;
371        Self::epoch_millis(&wall)
372    }
373
374    /// Notify all watchers with the given terminal outcome and drain the list.
375    fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
376        for watcher in std::mem::take(&mut self.watchers) {
377            watcher.notify_after_generated_terminal(outcome);
378        }
379    }
380
381    /// Mark the completion timestamp.
382    fn mark_completed(&mut self) {
383        self.completed_at = Some(Instant::now());
384    }
385}
386
387// ---------------------------------------------------------------------------
388// Combined shell state: DSL authority + shell records
389// ---------------------------------------------------------------------------
390
391#[derive(Debug)]
392struct ShellState {
393    /// DSL authority — sole source of truth for per-op canonical state.
394    dsl: DslAuthority,
395    /// Shell-owned per-operation records (specs, watchers, timestamps, peer handles).
396    records: HashMap<OperationId, ShellRecord>,
397    /// Pending wait-all coordination (oneshot channel).
398    pending_wait: Option<PendingWaitState>,
399    /// FIFO ordering of completed operation IDs for bounded eviction.
400    completed_order: VecDeque<OperationId>,
401    /// Maximum completed operations to retain.
402    max_completed: usize,
403    /// Maximum concurrent non-terminal operations (None = unlimited).
404    max_concurrent: Option<usize>,
405    /// Oneshot correlation id for the currently-pending `wait_all` future.
406    ///
407    /// Barrier membership (`wait_operation_ids`) and activation (`wait_active`)
408    /// are DSL-owned. This field is pure transport mechanics — the identity
409    /// the oneshot sender is tagged with so `Drop` can correlate cancellation.
410    wait_request_id: Option<WaitRequestId>,
411    /// Shared feed buffer for completion events.
412    feed_buffer: Arc<FeedBuffer>,
413    /// Persistence channel for durable snapshot writes (set via `set_persistence_channel`).
414    persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
415    /// Terminal unregister permanently fences any later channel replacement.
416    persistence_sealed: bool,
417    /// Epoch ID for persistence snapshots.
418    persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
419    /// Shared cursor state for persistence snapshots.
420    persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
421    /// Mechanical epoch fence closed by canonical owner teardown.
422    ///
423    /// Generated authority terminalizes each known operation. This bit closes
424    /// the shell admission door so detached callbacks that retained an Arc to
425    /// the registry cannot create or mutate operations after unregister.
426    owner_retired: bool,
427}
428
429/// Wrapper around the DSL authority that provides `Debug` output.
430///
431/// The generated `MeerkatMachineAuthority` does not derive `Debug`, but
432/// `ShellState` requires it. This wrapper delegates to the inner state's
433/// `Debug` impl.
434struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
435
436impl std::fmt::Debug for DslAuthority {
437    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
438        f.debug_struct("DslAuthority")
439            .field("state", self.0.state())
440            .finish()
441    }
442}
443
444/// Create a DSL authority initialized through generated authority. Per-op
445/// transitions guard only on `op_statuses.contains_key(operation_id)`, so the
446/// phase stays in `Idle` permanently (they all `to Idle`).
447fn new_ops_dsl_authority() -> DslAuthority {
448    DslAuthority(Box::new(
449        crate::meerkat_machine::dsl_authority::new_initialized_authority(
450            "ops lifecycle DSL Initialize must be accepted",
451        ),
452    ))
453}
454
455impl ShellState {
456    fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
457        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
458        let dsl = new_ops_dsl_authority();
459        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
460        let feed_capacity = max_completed.saturating_mul(4).max(1024);
461        tracing::info!(
462            feed_capacity,
463            "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
464        );
465        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
466        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
467        Self {
468            dsl,
469            records: HashMap::new(),
470            pending_wait: None,
471            completed_order: VecDeque::new(),
472            max_completed,
473            max_concurrent,
474            wait_request_id: None,
475            // Feed buffer is larger than max_completed to absorb bursts.
476            // Entries are only evicted by buffer capacity, not by consumer cursor,
477            // so the buffer must be large enough that consumers drain before
478            // the oldest entry is evicted.
479            feed_buffer,
480            persist_tx: None,
481            persistence_sealed: false,
482            persist_epoch_id: None,
483            persist_cursor_state: None,
484            owner_retired: false,
485        }
486    }
487
488    fn ensure_owner_active(&self) -> Result<(), OpsLifecycleError> {
489        if self.owner_retired {
490            Err(OpsLifecycleError::OwnerRetired)
491        } else {
492            Ok(())
493        }
494    }
495
496    /// Apply a DSL input, mapping transition errors into
497    /// [`OpsLifecycleError::Internal`]. Callers that need to distinguish
498    /// guard rejections (legal-transition violations) from internal desync
499    /// should use [`Self::dsl_apply_raw`] and classify the error themselves;
500    /// this helper is for DSL inputs whose preconditions the caller has
501    /// already fully validated (e.g., `RequestWaitAll`, `SatisfyWaitAll`).
502    fn dsl_apply(
503        &mut self,
504        input: mm_dsl::MeerkatMachineInput,
505        context: &str,
506    ) -> Result<(), OpsLifecycleError> {
507        self.dsl_apply_raw(input).map_err(|err| {
508            OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
509        })
510    }
511
512    /// Apply a DSL input, returning the raw kernel-level rejection so callers
513    /// can distinguish `GuardRejected` (a legitimate legality violation, e.g.,
514    /// `complete_operation` on a `Provisioning` op) from
515    /// `NoMatchingTransition` (a shell/DSL desync). Op-lifecycle entry points
516    /// feed guard rejections into generated rejection feedback before surfacing
517    /// public result classes.
518    fn dsl_apply_raw(
519        &mut self,
520        input: mm_dsl::MeerkatMachineInput,
521    ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
522        mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
523    }
524
525    fn dsl_apply_with_effects(
526        &mut self,
527        input: mm_dsl::MeerkatMachineInput,
528        context: &str,
529    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
530        let transition =
531            mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
532                OpsLifecycleError::Internal(format!(
533                    "DSL rejected ops transition ({context}): {err:?}"
534                ))
535            })?;
536        Ok(transition.into_effects())
537    }
538
539    /// Fail-closed read of a typed terminal payload entry: the payload IS the
540    /// domain [`OperationTerminalOutcome`] (K8b fold — no JSON codec), but the
541    /// shell still refuses to surface a payload whose variant disagrees with
542    /// the recorded discriminant.
543    fn checked_terminal_payload(
544        kind: mm_dsl::OperationTerminalOutcomeKind,
545        payload: &OperationTerminalOutcome,
546        authority: &str,
547        operation_id: &str,
548    ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
549        if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
550            return Err(OpsLifecycleError::Internal(format!(
551                "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
552            )));
553        }
554        Ok(payload.clone())
555    }
556
557    /// Read the DSL operation status for `id`, or `None` if not registered.
558    fn status(&self, id: &OperationId) -> Option<OperationStatus> {
559        let id_key = mm_dsl::OperationId::from_domain(id).0;
560        self.dsl
561            .0
562            .state()
563            .op_statuses
564            .get(&id_key)
565            .copied()
566            .map(OperationStatus::from)
567    }
568
569    fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
570        self.status(id).ok_or_else(|| {
571            OpsLifecycleError::Internal(format!(
572                "generated op lifecycle authority missing status for {id}"
573            ))
574        })
575    }
576
577    /// Read the DSL operation kind for `id`, or `None` if not registered.
578    fn kind(&self, id: &OperationId) -> Option<OperationKind> {
579        let id_key = mm_dsl::OperationId::from_domain(id).0;
580        self.dsl
581            .0
582            .state()
583            .op_kinds
584            .get(&id_key)
585            .copied()
586            .map(OperationKind::from)
587    }
588
589    fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
590        self.kind(id).ok_or_else(|| {
591            OpsLifecycleError::Internal(format!(
592                "generated op lifecycle authority missing kind for {id}"
593            ))
594        })
595    }
596
597    fn operation_source(
598        &self,
599        id: &OperationId,
600    ) -> Result<Option<OperationSource>, OpsLifecycleError> {
601        let id_key = mm_dsl::OperationId::from_domain(id).0;
602        self.dsl
603            .0
604            .state()
605            .op_sources
606            .get(&id_key)
607            .map(|source| {
608                source.to_domain().map_err(|error| {
609                    OpsLifecycleError::Internal(format!(
610                        "generated operation source authority has invalid source for {id}: {error}"
611                    ))
612                })
613            })
614            .transpose()
615    }
616
617    fn child_session_id_from_operation_source(
618        operation_source: Option<&OperationSource>,
619    ) -> Option<SessionId> {
620        match operation_source {
621            Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
622            Some(OperationSource::BackendPeer { .. }) | None => None,
623        }
624    }
625
626    fn align_spec_child_session_id_to_source(
627        spec: &mut OperationSpec,
628        operation_source: Option<&OperationSource>,
629    ) {
630        spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
631    }
632
633    /// Read the peer-ready flag for `id`.
634    fn peer_ready(&self, id: &OperationId) -> Option<bool> {
635        let id_key = mm_dsl::OperationId::from_domain(id).0;
636        self.dsl.0.state().op_peer_ready.get(&id_key).copied()
637    }
638
639    fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
640        self.peer_ready(id).ok_or_else(|| {
641            OpsLifecycleError::Internal(format!(
642                "generated op peer wiring authority missing peer-ready fact for {id}"
643            ))
644        })
645    }
646
647    /// Read the progress counter for `id`.
648    fn progress_count(&self, id: &OperationId) -> Option<u32> {
649        let id_key = mm_dsl::OperationId::from_domain(id).0;
650        self.dsl
651            .0
652            .state()
653            .op_progress_counts
654            .get(&id_key)
655            .map(|v| (*v).min(u32::MAX as u64) as u32)
656    }
657
658    fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
659        self.progress_count(id).ok_or_else(|| {
660            OpsLifecycleError::Internal(format!(
661                "generated op progress authority missing progress count for {id}"
662            ))
663        })
664    }
665
666    /// Read the terminal outcome for `id` by pairing the DSL's typed
667    /// discriminant with the companion payload JSON. Returns `None` when the
668    /// op has no recorded terminal discriminant.
669    fn terminal_outcome(
670        &self,
671        id: &OperationId,
672    ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
673        let id_key = mm_dsl::OperationId::from_domain(id).0;
674        let state = self.dsl.0.state();
675        let status = self.status(id);
676        let terminal = match status {
677            Some(status) => Self::operation_status_is_terminal(id, status)?,
678            None => false,
679        };
680        let kind = state.op_terminal_outcomes.get(&id_key).copied();
681        let Some(kind) = kind else {
682            if terminal {
683                return Err(OpsLifecycleError::Internal(format!(
684                    "generated op terminal authority missing terminal outcome for {id}"
685                )));
686            }
687            return Ok(None);
688        };
689        if !terminal {
690            return Err(OpsLifecycleError::Internal(format!(
691                "generated op terminal authority has terminal outcome for non-terminal {id}"
692            )));
693        }
694        let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
695            OpsLifecycleError::Internal(format!(
696                "generated op terminal authority missing terminal payload for {id}"
697            ))
698        })?;
699        Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
700            .map(Some)
701    }
702
703    /// Whether the operation is currently tracked in DSL state.
704    fn contains(&self, id: &OperationId) -> bool {
705        let id_key = mm_dsl::OperationId::from_domain(id).0;
706        self.dsl.0.state().op_statuses.contains_key(&id_key)
707    }
708
709    /// Number of non-terminal operations (derived from DSL state).
710    fn active_count(&self) -> usize {
711        self.dsl.0.state().active_op_count as usize
712    }
713
714    /// Number of operations currently tracked (including terminal).
715    fn operation_count(&self) -> usize {
716        self.dsl.0.state().op_statuses.len()
717    }
718
719    /// Iterate over all tracked operation IDs (DSL keys converted to domain).
720    fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
721        let mut ids = BTreeSet::new();
722        let state = self.dsl.0.state();
723        Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
724        Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
725        Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
726        Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
727        Self::collect_operation_id_keys(
728            &mut ids,
729            "op_progress_counts",
730            state.op_progress_counts.keys(),
731        )?;
732        Self::collect_operation_id_keys(
733            &mut ids,
734            "op_terminal_outcomes",
735            state.op_terminal_outcomes.keys(),
736        )?;
737        Self::collect_operation_id_keys(
738            &mut ids,
739            "op_terminal_payload",
740            state.op_terminal_payload.keys(),
741        )?;
742        Self::collect_operation_id_keys(
743            &mut ids,
744            "op_completion_seq",
745            state.op_completion_seq.keys(),
746        )?;
747        ids.extend(self.records.keys().cloned());
748        Ok(ids.into_iter().collect())
749    }
750
751    fn collect_operation_id_keys<'a, I>(
752        ids: &mut BTreeSet<OperationId>,
753        field: &str,
754        keys: I,
755    ) -> Result<(), OpsLifecycleError>
756    where
757        I: IntoIterator<Item = &'a String>,
758    {
759        for key in keys {
760            let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
761                OpsLifecycleError::Internal(format!(
762                    "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
763                ))
764            })?;
765            ids.insert(id);
766        }
767        Ok(())
768    }
769
770    fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
771        let id_key = mm_dsl::OperationId::from_domain(id).0;
772        let state = self.dsl.0.state();
773        state.op_statuses.contains_key(&id_key)
774            || state.op_kinds.contains_key(&id_key)
775            || state.op_sources.contains_key(&id_key)
776            || state.op_peer_ready.contains_key(&id_key)
777            || state.op_progress_counts.contains_key(&id_key)
778            || state.op_terminal_outcomes.contains_key(&id_key)
779            || state.op_terminal_payload.contains_key(&id_key)
780            || state.op_completion_seq.contains_key(&id_key)
781    }
782
783    /// Read the DSL-minted completion sequence for a terminal operation.
784    fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
785        let id_key = mm_dsl::OperationId::from_domain(id).0;
786        self.dsl.0.state().op_completion_seq.get(&id_key).copied()
787    }
788
789    fn completion_feed_authority_entries(
790        &self,
791    ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
792        let state = self.dsl.0.state();
793        let sequence_keys: BTreeSet<String> =
794            state.completion_feed_sequences.keys().cloned().collect();
795        let companion_domains: [(&str, BTreeSet<String>); 3] = [
796            (
797                "completion_feed_kinds",
798                state.completion_feed_kinds.keys().cloned().collect(),
799            ),
800            (
801                "completion_feed_terminal_outcomes",
802                state
803                    .completion_feed_terminal_outcomes
804                    .keys()
805                    .cloned()
806                    .collect(),
807            ),
808            (
809                "completion_feed_terminal_payload",
810                state
811                    .completion_feed_terminal_payload
812                    .keys()
813                    .cloned()
814                    .collect(),
815            ),
816        ];
817        for (field, keys) in companion_domains {
818            if keys != sequence_keys {
819                return Err(OpsLifecycleError::Internal(format!(
820                    "generated completion feed authority has mismatched {field} domain"
821                )));
822            }
823        }
824
825        let mut entries = HashMap::new();
826        for (id_key, seq) in &state.completion_feed_sequences {
827            if !state.completion_sequence_claims.contains(seq) {
828                return Err(OpsLifecycleError::Internal(format!(
829                    "generated completion feed authority sequence {seq} for {id_key} is not claimed"
830                )));
831            }
832            let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
833                OpsLifecycleError::Internal(format!(
834                    "generated completion feed authority used invalid operation id key {id_key}: {error}"
835                ))
836            })?;
837            let kind = state
838                .completion_feed_kinds
839                .get(id_key)
840                .copied()
841                .map(OperationKind::from)
842                .ok_or_else(|| {
843                    OpsLifecycleError::Internal(format!(
844                        "generated completion feed authority missing kind for {id_key}"
845                    ))
846                })?;
847            let outcome_kind = state
848                .completion_feed_terminal_outcomes
849                .get(id_key)
850                .copied()
851                .ok_or_else(|| {
852                    OpsLifecycleError::Internal(format!(
853                        "generated completion feed authority missing terminal outcome for {id_key}"
854                    ))
855                })?;
856            let payload = state
857                .completion_feed_terminal_payload
858                .get(id_key)
859                .ok_or_else(|| {
860                    OpsLifecycleError::Internal(format!(
861                        "generated completion feed authority missing terminal payload for {id_key}"
862                    ))
863                })?;
864            let terminal_outcome = Self::checked_terminal_payload(
865                outcome_kind,
866                payload,
867                "generated completion feed authority",
868                id_key,
869            )?;
870            entries.insert(
871                operation_id,
872                CompletionFeedCanonicalState {
873                    seq: *seq,
874                    kind,
875                    terminal_outcome,
876                },
877            );
878        }
879        Ok(entries)
880    }
881
882    fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
883        let state = self.dsl.0.state();
884        match consumer {
885            CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
886            CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
887            CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
888        }
889    }
890
891    fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
892        meerkat_core::EpochCursorSnapshot {
893            agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
894            runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
895            runtime_last_injected_seq: self
896                .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
897        }
898    }
899
900    /// Build a snapshot from DSL state + shell record.
901    fn snapshot(
902        &self,
903        id: &OperationId,
904    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
905        let Some(shell) = self.records.get(id) else {
906            if self.has_generated_operation_record_fact(id) {
907                return Err(OpsLifecycleError::Internal(format!(
908                    "generated op lifecycle authority has operation facts without shell projection record for {id}"
909                )));
910            }
911            return Ok(None);
912        };
913        let kind = self.require_kind(id)?;
914        let status = self.require_status(id)?;
915        let terminal = Self::operation_status_is_terminal(id, status)?;
916        let public_result_class = Self::operation_public_result_class(id, status)?;
917        let peer_ready = self.require_peer_ready(id)?;
918        let progress_count = self.require_progress_count(id)?;
919        let operation_source = self.operation_source(id)?;
920        let terminal_outcome = self.terminal_outcome(id)?;
921
922        let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
923        let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
924        let completed_at_ms = shell
925            .completed_at
926            .map(|i| shell.epoch_millis_for_instant(i));
927        let elapsed_ms = shell.completed_at.map(|completed| {
928            completed
929                .saturating_duration_since(shell.created_at)
930                .as_millis() as u64
931        });
932
933        Ok(Some(OperationLifecycleSnapshot {
934            id: shell.spec.id.clone(),
935            kind,
936            owner_session_id: shell.spec.owner_session_id.clone(),
937            display_name: shell.spec.display_name.clone(),
938            source_label: shell.spec.source_label.clone(),
939            child_session_id: Self::child_session_id_from_operation_source(
940                operation_source.as_ref(),
941            ),
942            operation_source,
943            expect_peer_channel: shell.spec.expect_peer_channel,
944            status,
945            terminal,
946            public_result_class,
947            peer_ready,
948            progress_count,
949            watcher_count: shell.watchers.len() as u32,
950            terminal_outcome,
951            peer_handle: shell.peer_handle.clone(),
952            created_at_ms,
953            started_at_ms,
954            completed_at_ms,
955            elapsed_ms,
956        }))
957    }
958
959    /// Emit shell-side mechanics for a terminal transition: notify watchers,
960    /// push generated-authorized CompletionEntry rows, retain in FIFO, evict
961    /// as needed. Called AFTER the DSL transition has already persisted the
962    /// terminal status + outcome.
963    fn finalize_terminal(
964        &mut self,
965        id: &OperationId,
966    ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
967        let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
968            OpsLifecycleError::Internal(format!(
969                "generated op terminal transition did not mint terminal outcome for {id}"
970            ))
971        })?;
972        let kind = self.require_kind(id)?;
973
974        // Notify watchers and mark completion timestamp.
975        if let Some(shell) = self.records.get_mut(id) {
976            shell.notify_watchers(&outcome);
977            shell.mark_completed();
978        }
979
980        if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
981        {
982            self.dsl_apply(
983                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
984                    operation_id: mm_dsl::OperationId::from_domain(id).0,
985                },
986                "CollectCompletedOp",
987            )?;
988            self.records.remove(id);
989            self.completed_order.retain(|queued| queued != id);
990            return Ok(None);
991        }
992
993        let mut completion_entry = None;
994        if Self::operation_completion_feed_class(id, kind)?
995            == mm_dsl::OperationCompletionFeedClass::Emit
996        {
997            let feed_authority = self
998                .completion_feed_authority_entries()?
999                .remove(id)
1000                .ok_or_else(|| {
1001                    OpsLifecycleError::Internal(format!(
1002                        "generated op terminal transition did not mint completion feed authority for {id}"
1003                    ))
1004                })?;
1005            if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
1006                return Err(OpsLifecycleError::Internal(format!(
1007                    "generated completion feed authority drifted from terminal op authority for {id}"
1008                )));
1009            }
1010            let seq = self.completion_sequence(id).ok_or_else(|| {
1011                OpsLifecycleError::Internal(format!(
1012                    "generated op terminal transition did not mint completion sequence for {id}"
1013                ))
1014            })?;
1015            if feed_authority.seq != seq {
1016                return Err(OpsLifecycleError::Internal(format!(
1017                    "generated completion feed authority sequence drifted for {id}"
1018                )));
1019            }
1020            let display_name = self
1021                .records
1022                .get(id)
1023                .map(|r| r.spec.display_name.clone())
1024                .unwrap_or_default();
1025            let completed_at_ms = self
1026                .records
1027                .get(id)
1028                .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1029            completion_entry = Some(CompletionEntry {
1030                seq: feed_authority.seq,
1031                operation_id: id.clone(),
1032                kind: feed_authority.kind,
1033                display_name,
1034                terminal_outcome: feed_authority.terminal_outcome,
1035                completed_at_ms,
1036            });
1037        }
1038
1039        // FIFO retention + eviction.
1040        self.completed_order.push_back(id.clone());
1041        while self.completed_order.len() > self.max_completed {
1042            if let Some(evicted) = self.completed_order.pop_front() {
1043                self.dsl_apply(
1044                    mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1045                        operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1046                    },
1047                    "EvictCompletedOp",
1048                )?;
1049                self.records.remove(&evicted);
1050            }
1051        }
1052
1053        // Satisfy a pending wait request if all its ops are now terminal. On
1054        // authority-invariant corruption this propagates the typed fault (and
1055        // drops the barrier oneshot so the waiter resolves to Err) instead of
1056        // reporting the op terminal with a silently-hung barrier.
1057        self.maybe_satisfy_wait()?;
1058        Ok(completion_entry)
1059    }
1060
1061    fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1062        if let Some(entry) = entry {
1063            self.feed_buffer.push(entry);
1064        }
1065    }
1066
1067    /// Read barrier membership from DSL state (sole owner).
1068    fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1069        self.dsl
1070            .0
1071            .state()
1072            .wait_operation_ids
1073            .iter()
1074            .map(|key| {
1075                serde_json::from_str::<OperationId>(key).map_err(|error| {
1076                    OpsLifecycleError::Internal(format!(
1077                        "generated wait operation identity authority used invalid operation id key {key}: {error}"
1078                    ))
1079                })
1080            })
1081            .collect()
1082    }
1083
1084    /// Whether the DSL has a barrier wait active.
1085    #[cfg(test)]
1086    fn wait_active(&self) -> bool {
1087        self.dsl.0.state().wait_active
1088    }
1089
1090    fn wait_all_satisfied_from_effects(
1091        effects: &[mm_dsl::MeerkatMachineEffect],
1092    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1093        let mut satisfied = None;
1094        for effect in effects {
1095            let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1096                wait_request_id,
1097                run_id,
1098                operation_ids,
1099            } = effect
1100            else {
1101                continue;
1102            };
1103            if satisfied.is_some() {
1104                return Err(OpsLifecycleError::Internal(
1105                    "generated wait_all authority emitted multiple satisfaction effects".into(),
1106                ));
1107            }
1108            let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1109                OpsLifecycleError::Internal(format!(
1110                    "generated wait_all authority emitted invalid wait request id '{}': {err}",
1111                    wait_request_id.0
1112                ))
1113            })?;
1114            let mut ids = Vec::with_capacity(operation_ids.len());
1115            for operation_id in operation_ids {
1116                ids.push(
1117                    serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1118                        OpsLifecycleError::Internal(format!(
1119                            "generated wait_all authority emitted invalid operation id '{}': {err}",
1120                            operation_id.0
1121                        ))
1122                    })?,
1123                );
1124            }
1125            satisfied = Some(WaitAllSatisfied {
1126                wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1127                run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1128                    OpsLifecycleError::Internal(format!(
1129                        "generated wait_all authority emitted invalid run id '{}': {err}",
1130                        run_id.0
1131                    ))
1132                })?),
1133                operation_ids: ids,
1134            });
1135        }
1136        Ok(satisfied)
1137    }
1138
1139    fn parse_wait_all_operation_id(
1140        raw: &str,
1141        context: &str,
1142    ) -> Result<OperationId, OpsLifecycleError> {
1143        serde_json::from_str::<OperationId>(raw).map_err(|err| {
1144            OpsLifecycleError::Internal(format!(
1145                "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1146            ))
1147        })
1148    }
1149
1150    fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1151        let mut seen = HashSet::new();
1152        operation_ids
1153            .iter()
1154            .find(|operation_id| !seen.insert((*operation_id).clone()))
1155            .cloned()
1156    }
1157
1158    fn wait_all_admission_error_from_effects(
1159        wait_request_id: &WaitRequestId,
1160        effects: &[mm_dsl::MeerkatMachineEffect],
1161    ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1162        let mut admission = None;
1163        for effect in effects {
1164            let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1165                wait_request_id: resolved_wait_request_id,
1166                result,
1167                reject_reason,
1168                rejected_operation_id,
1169            } = effect
1170            else {
1171                continue;
1172            };
1173            if admission.is_some() {
1174                return Err(OpsLifecycleError::Internal(
1175                    "generated wait_all authority emitted multiple admission results".into(),
1176                ));
1177            }
1178            let resolved_uuid =
1179                uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1180                    OpsLifecycleError::Internal(format!(
1181                        "generated wait_all authority emitted invalid wait request id '{}': {err}",
1182                        resolved_wait_request_id.0
1183                    ))
1184                })?;
1185            let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1186            if &resolved_wait_request_id != wait_request_id {
1187                return Err(OpsLifecycleError::Internal(format!(
1188                    "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1189                )));
1190            }
1191            admission = Some(match result {
1192                mm_dsl::WaitAllAdmissionResultKind::Accept => {
1193                    if reject_reason.is_some() || rejected_operation_id.is_some() {
1194                        return Err(OpsLifecycleError::Internal(
1195                            "generated wait_all authority accepted with rejection payload".into(),
1196                        ));
1197                    }
1198                    None
1199                }
1200                mm_dsl::WaitAllAdmissionResultKind::Reject => {
1201                    let reason = reject_reason.ok_or_else(|| {
1202                        OpsLifecycleError::Internal(
1203                            "generated wait_all authority rejected without reason".into(),
1204                        )
1205                    })?;
1206                    let error = match reason {
1207                        mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1208                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1209                                OpsLifecycleError::Internal(
1210                                    "generated wait_all authority rejected duplicate without operation id"
1211                                        .into(),
1212                                )
1213                            })?;
1214                            OpsLifecycleError::DuplicateWaitOperation(
1215                                Self::parse_wait_all_operation_id(raw, "duplicate")?,
1216                            )
1217                        }
1218                        mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1219                            if rejected_operation_id.is_some() {
1220                                return Err(OpsLifecycleError::Internal(
1221                                    "generated wait_all authority rejected active wait with operation id"
1222                                        .into(),
1223                                ));
1224                            }
1225                            OpsLifecycleError::WaitAlreadyActive
1226                        }
1227                        mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1228                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1229                                OpsLifecycleError::Internal(
1230                                    "generated wait_all authority rejected missing operation without operation id"
1231                                        .into(),
1232                                )
1233                            })?;
1234                            OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1235                                raw, "missing",
1236                            )?)
1237                        }
1238                    };
1239                    Some(error)
1240                }
1241            });
1242        }
1243        admission.ok_or_else(|| {
1244            OpsLifecycleError::Internal(
1245                "generated wait_all authority emitted no admission result".into(),
1246            )
1247        })
1248    }
1249
1250    fn resolve_wait_all_admission(
1251        &mut self,
1252        wait_request_id: &WaitRequestId,
1253        operation_ids: &[OperationId],
1254        dsl_ids: &BTreeSet<String>,
1255        dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1256        operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1257        operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1258    ) -> Result<(), OpsLifecycleError> {
1259        let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1260            .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1261        let not_found = operation_ids
1262            .iter()
1263            .find(|operation_id| !self.contains(operation_id))
1264            .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1265        let dsl_id_sequence: Vec<String> = operation_ids
1266            .iter()
1267            .map(|id| mm_dsl::OperationId::from_domain(id).0)
1268            .collect();
1269        let effects = self.dsl_apply_with_effects(
1270            mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1271                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1272                operation_id_sequence: dsl_id_sequence,
1273                operation_ids: dsl_ids.clone(),
1274                operation_id_tokens: dsl_id_tokens.clone(),
1275                operation_token_by_id: operation_token_by_id.clone(),
1276                operation_id_by_token: operation_id_by_token.clone(),
1277                duplicate_operation_id: duplicate,
1278                not_found_operation_id: not_found,
1279            },
1280            "ResolveWaitAllAdmission",
1281        )?;
1282        if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1283        {
1284            return Err(error);
1285        }
1286        Ok(())
1287    }
1288
1289    fn try_satisfy_wait_all_authority(
1290        &mut self,
1291    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1292        let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1293            return Ok(None);
1294        };
1295        let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1296            return Err(OpsLifecycleError::Internal(
1297                "generated wait_all authority has active wait without run id".into(),
1298            ));
1299        };
1300        let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1301        let transition = match mm_dsl::MeerkatMachineMutator::apply(
1302            &mut *self.dsl.0,
1303            mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1304                wait_request_id: dsl_wait_request_id,
1305                run_id: dsl_run_id,
1306                operation_id_tokens: dsl_operation_id_tokens,
1307            },
1308        ) {
1309            Ok(transition) => transition,
1310            Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1311            Err(err) => {
1312                return Err(OpsLifecycleError::Internal(format!(
1313                    "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1314                )));
1315            }
1316        };
1317        Self::wait_all_satisfied_from_effects(transition.effects())?
1318            .ok_or_else(|| {
1319                OpsLifecycleError::Internal(
1320                    "generated wait_all authority accepted satisfaction without effect".into(),
1321                )
1322            })
1323            .map(Some)
1324    }
1325
1326    fn begin_wait_all_authority(
1327        &mut self,
1328        run_id: &RunId,
1329        wait_request_id: &WaitRequestId,
1330        operation_ids: &[OperationId],
1331    ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1332        let mut dsl_ids = BTreeSet::new();
1333        let mut dsl_id_tokens = BTreeSet::new();
1334        let mut operation_token_by_id = BTreeMap::new();
1335        let mut operation_id_by_token = BTreeMap::new();
1336        for id in operation_ids {
1337            let token = mm_dsl::OperationId::from_domain(id);
1338            let raw_id = token.0.clone();
1339            dsl_ids.insert(raw_id.clone());
1340            dsl_id_tokens.insert(token.clone());
1341            operation_token_by_id.insert(raw_id.clone(), token.clone());
1342            operation_id_by_token.insert(token, raw_id);
1343        }
1344        self.resolve_wait_all_admission(
1345            wait_request_id,
1346            operation_ids,
1347            &dsl_ids,
1348            &dsl_id_tokens,
1349            &operation_token_by_id,
1350            &operation_id_by_token,
1351        )?;
1352        self.dsl_apply(
1353            mm_dsl::MeerkatMachineInput::RequestWaitAll {
1354                run_id: mm_dsl::RunId::from_domain(run_id),
1355                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1356                operation_id_sequence: operation_ids
1357                    .iter()
1358                    .map(|id| mm_dsl::OperationId::from_domain(id).0)
1359                    .collect(),
1360                operation_ids: dsl_ids,
1361                operation_id_tokens: dsl_id_tokens,
1362                operation_token_by_id,
1363                operation_id_by_token,
1364            },
1365            "RequestWaitAll",
1366        )?;
1367        if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1368            return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1369        }
1370        Ok(WaitAllAuthorityPlan::ActivateBarrier)
1371    }
1372
1373    fn owner_termination_targets(
1374        &self,
1375    ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1376        let mut targets = Vec::new();
1377        for id in self.operation_ids()? {
1378            let status = self.require_status(&id)?;
1379            if !Self::operation_status_is_terminal(&id, status)? {
1380                targets.push((id, status));
1381            }
1382        }
1383        Ok(targets)
1384    }
1385
1386    fn operation_status_is_terminal(
1387        operation_id: &OperationId,
1388        status: OperationStatus,
1389    ) -> Result<bool, OpsLifecycleError> {
1390        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1391        let effects = Self::apply_stateless_classifier(
1392            mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1393                operation_id: operation_id_key.clone(),
1394                status: mm_dsl::OperationStatus::from(status),
1395            },
1396            "ClassifyOperationTerminality",
1397        )?;
1398        let mut terminal = None;
1399        for effect in effects {
1400            match effect {
1401                mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1402                    if operation_id == operation_id_key =>
1403                {
1404                    terminal = Some(true);
1405                }
1406                mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1407                    if operation_id == operation_id_key =>
1408                {
1409                    terminal = Some(false);
1410                }
1411                other => {
1412                    return Err(OpsLifecycleError::Internal(format!(
1413                        "unexpected generated operation terminality effect: {other:?}"
1414                    )));
1415                }
1416            }
1417        }
1418        terminal.ok_or_else(|| {
1419            OpsLifecycleError::Internal(format!(
1420                "generated operation terminality authority emitted no effect for {operation_id}"
1421            ))
1422        })
1423    }
1424
1425    fn operation_public_result_class(
1426        operation_id: &OperationId,
1427        status: OperationStatus,
1428    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1429        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1430        let effects = Self::apply_stateless_classifier(
1431            mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1432                operation_id: operation_id_key.clone(),
1433                status: mm_dsl::OperationStatus::from(status),
1434            },
1435            "ClassifyOperationPublicResult",
1436        )?;
1437        let mut result = None;
1438        for effect in effects {
1439            match effect {
1440                mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1441                    operation_id,
1442                    result: classified,
1443                } if operation_id == operation_id_key => {
1444                    result = Some(OperationPublicResultClass::from(classified));
1445                }
1446                other => {
1447                    return Err(OpsLifecycleError::Internal(format!(
1448                        "unexpected generated operation public-result effect: {other:?}"
1449                    )));
1450                }
1451            }
1452        }
1453        result.ok_or_else(|| {
1454            OpsLifecycleError::Internal(format!(
1455                "generated operation public-result authority emitted no effect for {operation_id}"
1456            ))
1457        })
1458    }
1459
1460    fn operation_transition_rejection_is_idempotent(
1461        operation_id: &OperationId,
1462        action: OperationLifecycleAction,
1463        status: OperationStatus,
1464    ) -> Result<bool, OpsLifecycleError> {
1465        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1466        let action = mm_dsl::OpLifecycleActionKind::from(action);
1467        let status = mm_dsl::OperationStatus::from(status);
1468        let effects = Self::apply_stateless_classifier(
1469            mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1470                operation_id: operation_id_key.clone(),
1471                action,
1472                status,
1473            },
1474            "ClassifyOperationTransitionIdempotence",
1475        )?;
1476        let mut idempotent = None;
1477        for effect in effects {
1478            match effect {
1479                mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1480                    operation_id,
1481                    action: effect_action,
1482                    status: effect_status,
1483                } if operation_id == operation_id_key
1484                    && effect_action == action
1485                    && effect_status == status =>
1486                {
1487                    idempotent = Some(true);
1488                }
1489                mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1490                    operation_id,
1491                    action: effect_action,
1492                    status: effect_status,
1493                } if operation_id == operation_id_key
1494                    && effect_action == action
1495                    && effect_status == status =>
1496                {
1497                    idempotent = Some(false);
1498                }
1499                other => {
1500                    return Err(OpsLifecycleError::Internal(format!(
1501                        "unexpected generated operation transition-idempotence effect: {other:?}"
1502                    )));
1503                }
1504            }
1505        }
1506        idempotent.ok_or_else(|| {
1507            OpsLifecycleError::Internal(format!(
1508                "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1509            ))
1510        })
1511    }
1512
1513    fn operation_completion_feed_class(
1514        operation_id: &OperationId,
1515        kind: OperationKind,
1516    ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1517        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1518        let kind = mm_dsl::OperationKind::from(kind);
1519        let effects = Self::apply_stateless_classifier(
1520            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1521                operation_id: operation_id_key.clone(),
1522                kind,
1523            },
1524            "ClassifyOperationCompletionFeed",
1525        )?;
1526        let mut class = None;
1527        for effect in effects {
1528            match effect {
1529                mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1530                    operation_id,
1531                    result,
1532                } if operation_id == operation_id_key => {
1533                    class = Some(result);
1534                }
1535                other => {
1536                    return Err(OpsLifecycleError::Internal(format!(
1537                        "unexpected generated operation completion-feed effect: {other:?}"
1538                    )));
1539                }
1540            }
1541        }
1542        class.ok_or_else(|| {
1543            OpsLifecycleError::Internal(format!(
1544                "generated operation completion-feed authority emitted no effect for {operation_id}"
1545            ))
1546        })
1547    }
1548
1549    fn operation_completion_wake_class(
1550        operation_id: &OperationId,
1551        kind: OperationKind,
1552    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1553        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1554        let kind = mm_dsl::OperationKind::from(kind);
1555        let effects = Self::apply_stateless_classifier(
1556            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1557                operation_id: operation_id_key.clone(),
1558                kind,
1559            },
1560            "ClassifyOperationCompletionWake",
1561        )?;
1562        let mut class = None;
1563        for effect in effects {
1564            match effect {
1565                mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1566                    operation_id,
1567                    result,
1568                } if operation_id == operation_id_key => {
1569                    class = Some(OperationCompletionWakeClass::from(result));
1570                }
1571                other => {
1572                    return Err(OpsLifecycleError::Internal(format!(
1573                        "unexpected generated operation completion-wake effect: {other:?}"
1574                    )));
1575                }
1576            }
1577        }
1578        class.ok_or_else(|| {
1579            OpsLifecycleError::Internal(format!(
1580                "generated operation completion-wake authority emitted no effect for {operation_id}"
1581            ))
1582        })
1583    }
1584
1585    fn operation_durability_class(
1586        operation_id: &OperationId,
1587        kind: OperationKind,
1588    ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1589        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1590        let kind = mm_dsl::OperationKind::from(kind);
1591        let effects = Self::apply_stateless_classifier(
1592            mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1593                operation_id: operation_id_key.clone(),
1594                kind,
1595            },
1596            "ClassifyOperationDurability",
1597        )?;
1598        let mut class = None;
1599        for effect in effects {
1600            match effect {
1601                mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1602                    operation_id,
1603                    result,
1604                } if operation_id == operation_id_key => {
1605                    class = Some(result);
1606                }
1607                other => {
1608                    return Err(OpsLifecycleError::Internal(format!(
1609                        "unexpected generated operation durability effect: {other:?}"
1610                    )));
1611                }
1612            }
1613        }
1614        class.ok_or_else(|| {
1615            OpsLifecycleError::Internal(format!(
1616                "generated operation durability authority emitted no effect for {operation_id}"
1617            ))
1618        })
1619    }
1620
1621    fn recovered_operation_record_disposition(
1622        operation_id: &OperationId,
1623        status: OperationStatus,
1624        kind: OperationKind,
1625        terminal_outcome_present: bool,
1626        terminal_payload_present: bool,
1627        completion_sequence_present: bool,
1628    ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1629        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1630        let effects = Self::apply_stateless_classifier(
1631            mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1632                operation_id: operation_id_key.clone(),
1633                status: mm_dsl::OperationStatus::from(status),
1634                kind: mm_dsl::OperationKind::from(kind),
1635                terminal_outcome_present,
1636                terminal_payload_present,
1637                completion_sequence_present,
1638            },
1639            "ClassifyRecoveredOperationRecord",
1640        )?;
1641        let mut disposition = None;
1642        for effect in effects {
1643            match effect {
1644                mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1645                    if operation_id == operation_id_key =>
1646                {
1647                    disposition = Some(RecoveredOperationRecordDisposition::Retain);
1648                }
1649                mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1650                    if operation_id == operation_id_key =>
1651                {
1652                    disposition = Some(RecoveredOperationRecordDisposition::Discard);
1653                }
1654                other => {
1655                    return Err(OpsLifecycleError::Internal(format!(
1656                        "unexpected generated recovered-operation classification effect: {other:?}"
1657                    )));
1658                }
1659            }
1660        }
1661        disposition.ok_or_else(|| {
1662            OpsLifecycleError::Internal(format!(
1663                "generated recovered-operation classifier emitted no effect for {operation_id}"
1664            ))
1665        })
1666    }
1667
1668    fn apply_stateless_classifier(
1669        input: mm_dsl::MeerkatMachineInput,
1670        label: &'static str,
1671    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1672        let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1673            "ops stateless classifier Initialize must be accepted",
1674        );
1675        let transition =
1676            mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1677                OpsLifecycleError::Internal(format!(
1678                    "DSL rejected ops transition ({label}): {err:?}"
1679                ))
1680            })?;
1681        Ok(transition.into_effects())
1682    }
1683
1684    /// Check whether a pending barrier wait is now satisfied and resolve it.
1685    ///
1686    /// Barrier membership and the "all members terminal" decision both live
1687    /// in the DSL: `wait_operation_ids` carries the set, `wait_active`
1688    /// signals a pending barrier, and `SatisfyWaitAll`'s
1689    /// `all_members_terminal` guard owns the fixed-point test. The shell
1690    /// echoes the DSL-owned request id and typed operation tokens into
1691    /// `SatisfyWaitAll` so the transition can clear the barrier before
1692    /// rendering the handoff effect. It still fires idempotently on every
1693    /// terminal transition and lets the DSL guard reject early firings as a
1694    /// no-op. On acceptance (transition returns `Ok`), the shell selects the
1695    /// correlated oneshot and delivers the `WaitAllSatisfied` obligation
1696    /// token.
1697    ///
1698    /// `wait_request_id` is the shell-owned oneshot correlation id that
1699    /// selects which sender to notify; when the DSL barrier satisfies
1700    /// without a live correlation (post-recovery, or duplicate resolution),
1701    /// the oneshot simply remains pending. That benign no-correlation case is
1702    /// the only path that leaves the oneshot pending: on authority-invariant
1703    /// corruption the pending sender is dropped so the waiter's
1704    /// `WaitAllFuture` resolves to `Err` and the typed
1705    /// [`OpsLifecycleError`] propagates to the terminal transition caller
1706    /// instead of reporting the op complete with a silently-hung barrier.
1707    fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1708        let satisfied = match self.try_satisfy_wait_all_authority() {
1709            Ok(Some(satisfied)) => satisfied,
1710            Ok(None) => return Ok(()),
1711            Err(err) => {
1712                // Authority-invariant corruption: do NOT report the op complete
1713                // with a silently-pending barrier oneshot. Drop the sender so
1714                // the waiter's `WaitAllFuture` resolves to `Err` via its
1715                // `Poll::Ready(Err(_))` arm (the same mechanism
1716                // `cancel_wait_all_internal` uses), then propagate the typed
1717                // fault. The invariant is already broken, so we do not attempt
1718                // a `CancelWaitAll` rollback on the corrupt machine.
1719                if let Some(pending) = self.pending_wait.take() {
1720                    drop(pending.sender);
1721                }
1722                self.wait_request_id = None;
1723                return Err(err);
1724            }
1725        };
1726        let shell_wait_id = self.wait_request_id.take();
1727        if shell_wait_id
1728            .as_ref()
1729            .is_some_and(|id| id != &satisfied.wait_request_id)
1730        {
1731            tracing::error!(
1732                shell_wait_request_id = ?shell_wait_id,
1733                authority_wait_request_id = %satisfied.wait_request_id,
1734                "generated wait_all authority satisfied a different wait request"
1735            );
1736        }
1737        if let Some(pending) = self.pending_wait.take() {
1738            if pending.wait_request_id == satisfied.wait_request_id {
1739                let _ = pending.sender.send(satisfied);
1740            } else if let Some(shell_wait_id) = shell_wait_id {
1741                tracing::error!(
1742                    shell_wait_request_id = %shell_wait_id,
1743                    pending_wait_request_id = %pending.wait_request_id,
1744                    authority_wait_request_id = %satisfied.wait_request_id,
1745                    "generated wait_all authority satisfied without a matching pending waiter"
1746                );
1747            }
1748        }
1749        Ok(())
1750    }
1751
1752    /// Persist a terminal snapshot if a persistence channel is wired.
1753    ///
1754    /// Called after terminal transitions. Captures authority + entries + cursors under the write
1755    /// lock (caller already holds it), submits a persistence request, and waits for the worker's
1756    /// durable-store result. Returning success only means the snapshot write itself succeeded.
1757    fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1758        let (tx, epoch_id, cursor_state) = match (
1759            &self.persist_tx,
1760            &self.persist_epoch_id,
1761            &self.persist_cursor_state,
1762        ) {
1763            (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1764            _ => return Ok(()),
1765        };
1766
1767        let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1768        let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1769        let request = OpsLifecyclePersistenceRequest {
1770            snapshot,
1771            result_tx,
1772        };
1773
1774        tx.send(request).map_err(|_| {
1775            OpsLifecycleError::Internal(
1776                "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1777                    .into(),
1778            )
1779        })?;
1780        result_rx.recv().map_err(|_| {
1781            OpsLifecycleError::Internal(
1782                "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1783                    .into(),
1784            )
1785        })?
1786    }
1787
1788    /// Capture the full persisted snapshot for the current state.
1789    fn capture_snapshot(
1790        &self,
1791        epoch_id: meerkat_core::RuntimeEpochId,
1792        _cursor_state: &meerkat_core::EpochCursorState,
1793    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1794        let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1795        for op_id in self.operation_ids()? {
1796            let status = self.require_status(&op_id)?;
1797            let kind = self.require_kind(&op_id)?;
1798            if Self::operation_durability_class(&op_id, kind)?
1799                != mm_dsl::OperationDurabilityClass::Retain
1800            {
1801                continue;
1802            }
1803            let peer_ready = self.require_peer_ready(&op_id)?;
1804            let progress_count = self.require_progress_count(&op_id)?;
1805            let operation_source = self.operation_source(&op_id)?;
1806            let terminal_outcome = self.terminal_outcome(&op_id)?;
1807            let completion_sequence = self.completion_sequence(&op_id);
1808            if terminal_outcome.is_some() && completion_sequence.is_none() {
1809                return Err(OpsLifecycleError::Internal(format!(
1810                    "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1811                )));
1812            }
1813            if terminal_outcome.is_none() && completion_sequence.is_some() {
1814                return Err(OpsLifecycleError::Internal(format!(
1815                    "generated op terminal authority has completion sequence for non-terminal {op_id}"
1816                )));
1817            }
1818            let terminal_buffered = terminal_outcome.is_some();
1819            let watcher_count = self
1820                .records
1821                .get(&op_id)
1822                .map(|r| r.watchers.len() as u32)
1823                .unwrap_or(0);
1824            operations.insert(
1825                op_id,
1826                OperationCanonicalState {
1827                    status,
1828                    kind,
1829                    operation_source,
1830                    peer_ready,
1831                    progress_count,
1832                    watcher_count,
1833                    terminal_outcome,
1834                    completion_sequence,
1835                    terminal_buffered,
1836                },
1837            );
1838        }
1839        let operation_specs: HashMap<OperationId, OperationSpec> = self
1840            .records
1841            .iter()
1842            .filter(|(id, _)| operations.contains_key(*id))
1843            .map(|(id, record)| {
1844                let mut spec = record.spec.clone();
1845                let operation_source = operations
1846                    .get(id)
1847                    .and_then(|state| state.operation_source.as_ref());
1848                Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1849                (id.clone(), spec)
1850            })
1851            .collect();
1852        let completed_order: VecDeque<OperationId> = self
1853            .completed_order
1854            .iter()
1855            .filter(|id| operations.contains_key(*id))
1856            .cloned()
1857            .collect();
1858        let active_count = operations
1859            .iter()
1860            .filter(|(id, state)| {
1861                matches!(
1862                    Self::operation_status_is_terminal(id, state.status),
1863                    Ok(false)
1864                )
1865            })
1866            .count();
1867        let authority_completion_entries = self.completion_feed_authority_entries()?;
1868        let published_completion_entries_by_id: HashMap<OperationId, CompletionEntry> = {
1869            let inner = self
1870                .feed_buffer
1871                .inner
1872                .read()
1873                .unwrap_or_else(std::sync::PoisonError::into_inner);
1874            let mut entries = HashMap::new();
1875            for entry in &inner.entries {
1876                if !authority_completion_entries.contains_key(&entry.operation_id) {
1877                    return Err(OpsLifecycleError::Internal(format!(
1878                        "public completion feed projection for {} has no generated authority",
1879                        entry.operation_id
1880                    )));
1881                }
1882                if entries
1883                    .insert(entry.operation_id.clone(), entry.clone())
1884                    .is_some()
1885                {
1886                    return Err(OpsLifecycleError::Internal(format!(
1887                        "public completion feed projection for {} appeared more than once",
1888                        entry.operation_id
1889                    )));
1890                }
1891            }
1892            entries
1893        };
1894        let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
1895            .iter()
1896            .map(|(operation_id, authority_entry)| {
1897                if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
1898                    if projection.seq != authority_entry.seq
1899                        || projection.kind != authority_entry.kind
1900                        || projection.terminal_outcome != authority_entry.terminal_outcome
1901                    {
1902                        return Err(OpsLifecycleError::Internal(format!(
1903                            "public completion feed projection for {operation_id} drifted from generated authority"
1904                        )));
1905                    }
1906                    return Ok(projection.clone());
1907                }
1908
1909                let display_name = self
1910                    .records
1911                    .get(operation_id)
1912                    .map(|record| record.spec.display_name.clone())
1913                    .unwrap_or_default();
1914                let completed_at_ms = self.records.get(operation_id).and_then(|record| {
1915                    record
1916                        .completed_at
1917                        .map(|completed_at| record.epoch_millis_for_instant(completed_at))
1918                });
1919
1920                Ok(CompletionEntry {
1921                    seq: authority_entry.seq,
1922                    operation_id: operation_id.clone(),
1923                    kind: authority_entry.kind,
1924                    display_name,
1925                    terminal_outcome: authority_entry.terminal_outcome.clone(),
1926                    completed_at_ms,
1927                })
1928            })
1929            .collect::<Result<_, _>>()?;
1930        completion_entries.sort_by_key(|entry| entry.seq);
1931
1932        let authority_state = RegistryCanonicalState {
1933            operations,
1934            completion_feed_entries: authority_completion_entries,
1935            completed_order,
1936            max_completed: self.max_completed,
1937            max_concurrent: self.max_concurrent,
1938            active_count,
1939            wait_request_id: self.wait_request_id.clone(),
1940            wait_operation_ids: self.wait_operation_ids()?,
1941            next_completion_seq: self.dsl.0.state().next_completion_seq,
1942        };
1943
1944        Ok(PersistedOpsSnapshot {
1945            epoch_id,
1946            authority_state,
1947            operation_specs,
1948            completion_entries,
1949            cursors: self.completion_cursor_snapshot(),
1950        })
1951    }
1952
1953    fn shell_record_mut(
1954        &mut self,
1955        id: &OperationId,
1956    ) -> Result<&mut ShellRecord, OpsLifecycleError> {
1957        self.records
1958            .get_mut(id)
1959            .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
1960    }
1961
1962    fn collect_wait_outcomes(
1963        &self,
1964        operation_ids: &[OperationId],
1965    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
1966        operation_ids
1967            .iter()
1968            .map(|operation_id| {
1969                let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
1970                    OpsLifecycleError::Internal(format!(
1971                        "wait_all completed without terminal outcome for {operation_id}"
1972                    ))
1973                })?;
1974                Ok((operation_id.clone(), outcome))
1975            })
1976            .collect()
1977    }
1978}
1979
1980impl Default for ShellState {
1981    fn default() -> Self {
1982        Self::new(DEFAULT_MAX_COMPLETED, None)
1983    }
1984}
1985
1986// ---------------------------------------------------------------------------
1987// Public configuration & registry
1988// ---------------------------------------------------------------------------
1989
1990/// Configuration for [`RuntimeOpsLifecycleRegistry`].
1991#[derive(Debug, Clone)]
1992pub struct OpsLifecycleConfig {
1993    /// Maximum number of completed operations to retain (default: 256).
1994    pub max_completed: usize,
1995    /// Maximum concurrent non-terminal operations (None = unlimited).
1996    pub max_concurrent: Option<usize>,
1997}
1998
1999impl Default for OpsLifecycleConfig {
2000    fn default() -> Self {
2001        Self {
2002            max_completed: DEFAULT_MAX_COMPLETED,
2003            max_concurrent: None,
2004        }
2005    }
2006}
2007
2008/// Per-runtime shared registry for async operation lifecycle truth.
2009///
2010/// Per-operation canonical lifecycle state is owned by the DSL authority
2011/// embedded in the shell. This struct manages I/O concerns: watcher
2012/// channels, timestamps, peer handles, snapshot assembly, FIFO eviction,
2013/// and the completion feed buffer.
2014#[derive(Debug)]
2015pub struct RuntimeOpsLifecycleRegistry {
2016    state: RwLock<ShellState>,
2017}
2018
2019#[derive(Debug, Clone)]
2020pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2021    pub operation_count: usize,
2022    pub active_count: usize,
2023    pub wait_request_id: Option<WaitRequestId>,
2024    pub pending_wait_present: bool,
2025    pub pending_wait_request_id: Option<WaitRequestId>,
2026    pub wait_operation_ids: Vec<OperationId>,
2027    pub operations: Vec<OperationLifecycleSnapshot>,
2028}
2029
2030impl Default for RuntimeOpsLifecycleRegistry {
2031    fn default() -> Self {
2032        Self {
2033            state: RwLock::new(ShellState::default()),
2034        }
2035    }
2036}
2037
2038impl RuntimeOpsLifecycleRegistry {
2039    pub fn new() -> Self {
2040        let dsl = new_ops_dsl_authority();
2041        let feed_capacity = DEFAULT_MAX_COMPLETED.saturating_mul(4).max(1024);
2042        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2043        Self {
2044            state: RwLock::new(ShellState {
2045                dsl,
2046                records: HashMap::new(),
2047                pending_wait: None,
2048                completed_order: VecDeque::new(),
2049                max_completed: DEFAULT_MAX_COMPLETED,
2050                max_concurrent: None,
2051                wait_request_id: None,
2052                feed_buffer,
2053                persist_tx: None,
2054                persistence_sealed: false,
2055                persist_epoch_id: None,
2056                persist_cursor_state: None,
2057                owner_retired: false,
2058            }),
2059        }
2060    }
2061
2062    pub fn with_config(config: OpsLifecycleConfig) -> Self {
2063        Self {
2064            state: RwLock::new(ShellState::new(config.max_completed, config.max_concurrent)),
2065        }
2066    }
2067
2068    fn recover_completion_feed_entry(
2069        shell: &mut ShellState,
2070        operation_id: &OperationId,
2071        entry: &CompletionFeedCanonicalState,
2072    ) -> Result<(), OpsLifecycleError> {
2073        let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2074        let terminal_outcome_kind =
2075            mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2076        let effects = shell.dsl_apply_with_effects(
2077            mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2078                operation_id: expected_operation_id.clone(),
2079                kind: mm_dsl::OperationKind::from(entry.kind),
2080                terminal_outcome: terminal_outcome_kind,
2081                terminal_payload: entry.terminal_outcome.clone(),
2082                completion_sequence: entry.seq,
2083            },
2084            "RecoverCompletionFeedEntry",
2085        )?;
2086        let recovered = effects.iter().find_map(|effect| match effect {
2087            mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2088                operation_id,
2089                seq,
2090                kind,
2091                terminal_outcome,
2092                terminal_payload,
2093            } => Some((
2094                operation_id,
2095                *seq,
2096                OperationKind::from(*kind),
2097                *terminal_outcome,
2098                terminal_payload,
2099            )),
2100            _ => None,
2101        });
2102        let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2103            return Err(OpsLifecycleError::Internal(
2104                "generated completion-feed recovery emitted no recovered entry".into(),
2105            ));
2106        };
2107        if operation_id != &expected_operation_id
2108            || seq != entry.seq
2109            || kind != entry.kind
2110            || terminal_outcome != terminal_outcome_kind
2111            || terminal_payload != &entry.terminal_outcome
2112        {
2113            return Err(OpsLifecycleError::Internal(format!(
2114                "generated completion-feed recovery drifted for {operation_id}"
2115            )));
2116        }
2117        Ok(())
2118    }
2119
2120    /// Wire a persistence channel for durable snapshot writes.
2121    ///
2122    /// After this call, terminal transitions (complete/fail/cancel/abort)
2123    /// capture a snapshot and queue it to the channel. A dedicated
2124    /// persistence task should drain the channel and write to the store.
2125    pub fn set_persistence_channel(
2126        &self,
2127        tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2128        epoch_id: meerkat_core::RuntimeEpochId,
2129        cursor_state: Arc<meerkat_core::EpochCursorState>,
2130    ) {
2131        if let Ok(mut state) = self.state.write() {
2132            if state.owner_retired || state.persistence_sealed {
2133                tracing::warn!(
2134                    "ignored attempt to rewire a terminally sealed ops persistence channel"
2135                );
2136                return;
2137            }
2138            state.persist_tx = Some(tx);
2139            state.persist_epoch_id = Some(epoch_id);
2140            state.persist_cursor_state = Some(cursor_state);
2141        }
2142    }
2143
2144    /// Terminalize every live operation, persist those generated transitions,
2145    /// then close both lifecycle admission and the persistence producer.
2146    ///
2147    /// The write lock excludes every callback that retained this registry.
2148    /// Each terminal transition waits for its durable worker acknowledgement;
2149    /// taking the sender afterward therefore leaves no queued write behind.
2150    /// The unregister coordinator must join the owned worker before publishing
2151    /// final lifecycle deletion in the RuntimeStore.
2152    pub(crate) fn retire_owner_for_unregister(
2153        &self,
2154        reason: String,
2155    ) -> Result<(), OpsLifecycleError> {
2156        let closed_sender = {
2157            let mut state = self.write_state()?;
2158            if state.owner_retired {
2159                return Ok(());
2160            }
2161            terminate_owner_locked(&mut state, &reason)?;
2162            state.owner_retired = true;
2163            state.persistence_sealed = true;
2164            state.persist_epoch_id = None;
2165            state.persist_cursor_state = None;
2166            state.persist_tx.take()
2167        };
2168        drop(closed_sender);
2169        Ok(())
2170    }
2171
2172    /// Recover from a persisted snapshot.
2173    ///
2174    /// Rebuilds DSL state (stripping non-terminal ops — only terminals
2175    /// survive recovery), creates fresh shell records from specs, and seeds
2176    /// the feed buffer only with completion entries accepted by generated
2177    /// recovery authority.
2178    pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2179        let PersistedOpsSnapshot {
2180            authority_state,
2181            operation_specs,
2182            completion_entries,
2183            cursors,
2184            ..
2185        } = snapshot;
2186        let max_completed = authority_state.max_completed;
2187        let max_concurrent = authority_state.max_concurrent;
2188        let next_completion_seq = authority_state.next_completion_seq;
2189        let authority_completion_entries = authority_state.completion_feed_entries;
2190        let authority_operations = authority_state.operations;
2191        let mut shell = ShellState::new(max_completed, max_concurrent);
2192
2193        // Replay every persisted op through generated recovery authority.
2194        // The transition accepts only terminal records with outcome and
2195        // completion-sequence witnesses. Volatile non-terminal rows are not
2196        // recovered; terminal/corrupt rows must fail closed instead of being
2197        // projected into shell/public feed state.
2198        let mut retained_ids: HashSet<OperationId> = HashSet::new();
2199        for (op_id, op_state) in authority_operations {
2200            let terminal_outcome = op_state
2201                .terminal_outcome
2202                .as_ref()
2203                .map(mm_dsl::OperationTerminalOutcomeKind::from);
2204            let terminal_payload = op_state.terminal_outcome.clone();
2205            let disposition = ShellState::recovered_operation_record_disposition(
2206                &op_id,
2207                op_state.status,
2208                op_state.kind,
2209                terminal_outcome.is_some(),
2210                terminal_payload.is_some(),
2211                op_state.completion_sequence.is_some(),
2212            )?;
2213            if disposition == RecoveredOperationRecordDisposition::Discard {
2214                continue;
2215            }
2216            if let Some(spec_source) = operation_specs
2217                .get(&op_id)
2218                .and_then(|spec| spec.operation_source.as_ref())
2219                && op_state.operation_source.as_ref() != Some(spec_source)
2220            {
2221                return Err(OpsLifecycleError::Internal(format!(
2222                    "persisted operation source mirror for {op_id} drifted from generated authority"
2223                )));
2224            }
2225            let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2226                operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2227                status: mm_dsl::OperationStatus::from(op_state.status),
2228                kind: mm_dsl::OperationKind::from(op_state.kind),
2229                source: op_state
2230                    .operation_source
2231                    .as_ref()
2232                    .map(mm_dsl::OperationSource::from_domain),
2233                peer_ready: op_state.peer_ready,
2234                progress_count: u64::from(op_state.progress_count),
2235                terminal_outcome,
2236                terminal_payload,
2237                completion_sequence: op_state.completion_sequence,
2238            };
2239            shell.dsl_apply(recovery, "RecoverOpRecord")?;
2240            let recovered_seq = shell.completion_sequence(&op_id).ok_or_else(|| {
2241                OpsLifecycleError::Internal(format!(
2242                    "generated op recovery accepted {op_id} without completion sequence"
2243                ))
2244            })?;
2245            if op_state.completion_sequence != Some(recovered_seq) {
2246                return Err(OpsLifecycleError::Internal(format!(
2247                    "generated op recovery completion sequence mismatch for {op_id}"
2248                )));
2249            }
2250            retained_ids.insert(op_id);
2251        }
2252        shell.dsl_apply(
2253            mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2254                next_completion_seq,
2255            },
2256            "RecoverOpsCompletionCursor",
2257        )?;
2258        shell.dsl_apply(
2259            mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2260                agent_applied_cursor: cursors.agent_applied_cursor,
2261                runtime_observed_cursor: cursors.runtime_observed_seq,
2262                runtime_injected_cursor: cursors.runtime_last_injected_seq,
2263            },
2264            "RecoverCompletionConsumerCursors",
2265        )?;
2266
2267        // Rebuild completed_order from generated completion-sequence truth,
2268        // never from the persisted shell ordering mirror.
2269        let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2270            .iter()
2271            .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2272            .collect();
2273        recovered_completed.sort_by_key(|(seq, _)| *seq);
2274        shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2275
2276        // Recover generated-owned feed authority for entries whose operation
2277        // record is no longer retained. Retained records already wrote their
2278        // feed authority through RecoverOpRecord above.
2279        for (operation_id, entry) in &authority_completion_entries {
2280            if !retained_ids.contains(operation_id) {
2281                Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2282            }
2283        }
2284
2285        let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2286        for (operation_id, entry) in &authority_completion_entries {
2287            let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2288                return Err(OpsLifecycleError::Internal(format!(
2289                    "persisted completion feed authority for {operation_id} was not recovered"
2290                )));
2291            };
2292            if recovered_entry != entry {
2293                return Err(OpsLifecycleError::Internal(format!(
2294                    "persisted completion feed authority drifted from generated recovery for {operation_id}"
2295                )));
2296            }
2297        }
2298
2299        // Projection rows may carry display metadata only after generated
2300        // feed authority has decided the operation id, sequence, kind, and
2301        // terminal outcome. Any semantic drift fails closed.
2302        let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2303        for entry in completion_entries {
2304            let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2305                return Err(OpsLifecycleError::Internal(format!(
2306                    "persisted completion feed projection for {} has no generated feed authority",
2307                    entry.operation_id
2308                )));
2309            };
2310            if authority_entry.seq != entry.seq
2311                || authority_entry.kind != entry.kind
2312                || authority_entry.terminal_outcome != entry.terminal_outcome
2313            {
2314                return Err(OpsLifecycleError::Internal(format!(
2315                    "persisted completion feed projection for {} drifted from generated feed authority",
2316                    entry.operation_id
2317                )));
2318            }
2319            projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2320        }
2321
2322        let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2323            canonical_feed_entries.into_iter().collect();
2324        recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2325        for (operation_id, entry) in recovered_entries {
2326            let projection = projection_entries_by_id.get(&operation_id);
2327            let display_name = operation_specs
2328                .get(&operation_id)
2329                .map(|spec| spec.display_name.clone())
2330                .or_else(|| projection.map(|entry| entry.display_name.clone()))
2331                .unwrap_or_default();
2332            let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2333            shell.feed_buffer.push(CompletionEntry {
2334                seq: entry.seq,
2335                operation_id,
2336                kind: entry.kind,
2337                display_name,
2338                terminal_outcome: entry.terminal_outcome,
2339                completed_at_ms,
2340            });
2341        }
2342
2343        // Rebuild shell records from specs (fresh timestamps, no watchers)
2344        // — only for operations still retained in the DSL state.
2345        for (op_id, spec) in operation_specs {
2346            if retained_ids.contains(&op_id) {
2347                let mut spec = spec;
2348                let operation_source = shell.operation_source(&op_id)?;
2349                ShellState::align_spec_child_session_id_to_source(
2350                    &mut spec,
2351                    operation_source.as_ref(),
2352                );
2353                shell.records.insert(
2354                    op_id,
2355                    ShellRecord {
2356                        spec,
2357                        peer_handle: None,
2358                        watchers: Vec::new(),
2359                        created_at: Instant::now(),
2360                        started_at: None,
2361                        completed_at: None,
2362                        created_at_wall: SystemTime::now(),
2363                    },
2364                );
2365            }
2366        }
2367
2368        Ok(Self {
2369            state: RwLock::new(shell),
2370        })
2371    }
2372
2373    /// Capture a serializable snapshot of the current state for persistence.
2374    ///
2375    /// Includes authority state, operation specs, completion entries, and
2376    /// generated completion-consumer cursor values.
2377    pub fn capture_persistence_snapshot(
2378        &self,
2379        epoch_id: meerkat_core::RuntimeEpochId,
2380        cursor_state: &meerkat_core::EpochCursorState,
2381    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2382        let state = self
2383            .state
2384            .read()
2385            .unwrap_or_else(std::sync::PoisonError::into_inner);
2386        state.capture_snapshot(epoch_id, cursor_state)
2387    }
2388
2389    /// Snapshot generated completion-consumer cursor state.
2390    pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2391        let state = self
2392            .state
2393            .read()
2394            .unwrap_or_else(std::sync::PoisonError::into_inner);
2395        state.completion_cursor_snapshot()
2396    }
2397
2398    /// Return a read handle to the completion feed.
2399    pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2400        let state = self
2401            .state
2402            .read()
2403            .unwrap_or_else(std::sync::PoisonError::into_inner);
2404        Arc::new(RuntimeCompletionFeed {
2405            buffer: Arc::clone(&state.feed_buffer),
2406        })
2407    }
2408
2409    /// Capture a stable diagnostic snapshot of the canonical ops lifecycle state.
2410    pub(crate) fn diagnostic_snapshot(
2411        &self,
2412    ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2413        let state = self
2414            .state
2415            .read()
2416            .unwrap_or_else(std::sync::PoisonError::into_inner);
2417        let mut operations = state
2418            .operation_ids()?
2419            .into_iter()
2420            .map(|id| state.snapshot(&id))
2421            .collect::<Result<Vec<_>, _>>()?
2422            .into_iter()
2423            .flatten()
2424            .collect::<Vec<_>>();
2425        operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2426        Ok(RuntimeOpsDiagnosticSnapshot {
2427            operation_count: state.operation_count(),
2428            active_count: state.active_count(),
2429            wait_request_id: state.wait_request_id.clone(),
2430            pending_wait_present: state.pending_wait.is_some(),
2431            pending_wait_request_id: state
2432                .pending_wait
2433                .as_ref()
2434                .map(|pending_wait| pending_wait.wait_request_id.clone()),
2435            wait_operation_ids: state.wait_operation_ids()?,
2436            operations,
2437        })
2438    }
2439
2440    fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2441        self.state
2442            .read()
2443            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2444    }
2445
2446    fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2447        self.state
2448            .write()
2449            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2450    }
2451
2452    fn cancel_wait_all_internal(
2453        &self,
2454        wait_request_id: &WaitRequestId,
2455    ) -> Result<(), OpsLifecycleError> {
2456        let mut state = self.write_state()?;
2457        match state.wait_request_id.as_ref() {
2458            Some(active) if active == wait_request_id => {
2459                // Clear the DSL barrier via the dedicated `CancelWaitAll`
2460                // transition. Unlike `SatisfyWaitAll`, it does not require
2461                // every member to be terminal (the request was dropped, not
2462                // resolved) and does not emit the `WaitAllSatisfied`.
2463                state.dsl_apply(
2464                    mm_dsl::MeerkatMachineInput::CancelWaitAll,
2465                    "CancelWaitAll(cancel)",
2466                )?;
2467                state.wait_request_id = None;
2468                if state
2469                    .pending_wait
2470                    .as_ref()
2471                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2472                {
2473                    state.pending_wait = None;
2474                }
2475                Ok(())
2476            }
2477            _ => {
2478                if state
2479                    .pending_wait
2480                    .as_ref()
2481                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2482                {
2483                    state.pending_wait = None;
2484                }
2485                Ok(())
2486            }
2487        }
2488    }
2489}
2490
2491enum WaitAllFutureState {
2492    Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2493    Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2494    Done,
2495}
2496
2497struct WaitAllFuture<'a> {
2498    registry: &'a RuntimeOpsLifecycleRegistry,
2499    wait_request_id: WaitRequestId,
2500    state: WaitAllFutureState,
2501}
2502
2503impl Future for WaitAllFuture<'_> {
2504    type Output = Result<WaitAllResult, OpsLifecycleError>;
2505
2506    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2507        match &mut self.state {
2508            WaitAllFutureState::Ready(result) => {
2509                let ready = result.take().unwrap_or_else(|| {
2510                    Err(OpsLifecycleError::Internal(
2511                        "wait_all future polled after completion".into(),
2512                    ))
2513                });
2514                self.state = WaitAllFutureState::Done;
2515                Poll::Ready(ready)
2516            }
2517            WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2518                Poll::Pending => Poll::Pending,
2519                Poll::Ready(Ok(satisfied)) => {
2520                    let outcomes = match self.registry.read_state() {
2521                        Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2522                        Err(err) => Err(err),
2523                    };
2524                    self.state = WaitAllFutureState::Done;
2525                    Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2526                        outcomes,
2527                        satisfied,
2528                    }))
2529                }
2530                Poll::Ready(Err(_)) => {
2531                    self.state = WaitAllFutureState::Done;
2532                    Poll::Ready(Err(OpsLifecycleError::Internal(
2533                        "wait_all completion channel dropped".into(),
2534                    )))
2535                }
2536            },
2537            WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2538                "wait_all future polled after completion".into(),
2539            ))),
2540        }
2541    }
2542}
2543
2544impl Drop for WaitAllFuture<'_> {
2545    fn drop(&mut self) {
2546        if matches!(self.state, WaitAllFutureState::Waiting(_))
2547            && let Err(err) = self
2548                .registry
2549                .cancel_wait_all_internal(&self.wait_request_id)
2550        {
2551            tracing::error!(
2552                wait_request_id = %self.wait_request_id,
2553                error = %err,
2554                "generated wait_all authority rejected cancellation during drop"
2555            );
2556        }
2557    }
2558}
2559
2560// ---------------------------------------------------------------------------
2561// Generated op lifecycle result-class feedback
2562// ---------------------------------------------------------------------------
2563
2564fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2565    match action {
2566        mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2567        mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2568        mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2569        mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2570        mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2571        mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2572        mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2573        mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2574        mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2575        mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2576    }
2577}
2578
2579fn op_lifecycle_rejection_error_from_effects(
2580    id: &OperationId,
2581    requested_action: mm_dsl::OpLifecycleActionKind,
2582    effects: &[mm_dsl::MeerkatMachineEffect],
2583) -> Result<OpsLifecycleError, OpsLifecycleError> {
2584    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2585    let mut rejection = None;
2586    for effect in effects {
2587        let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2588            operation_id,
2589            action,
2590            reason,
2591            status,
2592        } = effect
2593        else {
2594            continue;
2595        };
2596        if rejection.is_some() {
2597            return Err(OpsLifecycleError::Internal(
2598                "generated op lifecycle authority emitted multiple rejection results".into(),
2599            ));
2600        }
2601        if operation_id != &expected_id || *action != requested_action {
2602            return Err(OpsLifecycleError::Internal(format!(
2603                "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2604            )));
2605        }
2606        let error = match reason {
2607            mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2608                if status.is_some() {
2609                    return Err(OpsLifecycleError::Internal(
2610                        "generated op lifecycle authority emitted not-found with status".into(),
2611                    ));
2612                }
2613                OpsLifecycleError::NotFound(id.clone())
2614            }
2615            mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2616                let status = status.ok_or_else(|| {
2617                    OpsLifecycleError::Internal(
2618                        "generated op lifecycle authority emitted invalid-transition without status"
2619                            .into(),
2620                    )
2621                })?;
2622                OpsLifecycleError::InvalidTransition {
2623                    id: id.clone(),
2624                    status: OperationStatus::from(status),
2625                    action: op_lifecycle_action_label(requested_action),
2626                }
2627            }
2628            mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2629                if status.is_none() {
2630                    return Err(OpsLifecycleError::Internal(
2631                        "generated op lifecycle authority emitted peer-not-expected without status"
2632                            .into(),
2633                    ));
2634                }
2635                OpsLifecycleError::PeerNotExpected(id.clone())
2636            }
2637            mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2638                if status.is_none() {
2639                    return Err(OpsLifecycleError::Internal(
2640                        "generated op lifecycle authority emitted already-peer-ready without status"
2641                            .into(),
2642                    ));
2643                }
2644                OpsLifecycleError::AlreadyPeerReady(id.clone())
2645            }
2646        };
2647        rejection = Some(error);
2648    }
2649    rejection.ok_or_else(|| {
2650        OpsLifecycleError::Internal(
2651            "generated op lifecycle authority emitted no rejection result".into(),
2652        )
2653    })
2654}
2655
2656fn classify_generated_op_rejection(
2657    state: &mut ShellState,
2658    err: mm_dsl::MeerkatMachineTransitionError,
2659    id: &OperationId,
2660    action: mm_dsl::OpLifecycleActionKind,
2661) -> OpsLifecycleError {
2662    match err {
2663        mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2664            match state.dsl_apply_with_effects(
2665                mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2666                    operation_id: mm_dsl::OperationId::from_domain(id).0,
2667                    action,
2668                },
2669                "ResolveOpLifecycleTransitionRejection",
2670            ) {
2671                Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2672                    .unwrap_or_else(|err| err),
2673                Err(err) => err,
2674            }
2675        }
2676        other => OpsLifecycleError::Internal(format!(
2677            "DSL rejected ops transition ({}): {other:?}",
2678            op_lifecycle_action_label(action)
2679        )),
2680    }
2681}
2682
2683fn apply_op_transition(
2684    state: &mut ShellState,
2685    id: &OperationId,
2686    input: mm_dsl::MeerkatMachineInput,
2687    action: mm_dsl::OpLifecycleActionKind,
2688) -> Result<(), OpsLifecycleError> {
2689    state.ensure_owner_active()?;
2690    state
2691        .dsl_apply_raw(input)
2692        .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2693}
2694
2695fn apply_terminal_op_transition_and_persist(
2696    state: &mut ShellState,
2697    id: &OperationId,
2698    input: mm_dsl::MeerkatMachineInput,
2699    action: mm_dsl::OpLifecycleActionKind,
2700) -> Result<(), OpsLifecycleError> {
2701    let previous_snapshot = state.dsl.0.snapshot();
2702    apply_op_transition(state, id, input, action)?;
2703    if let Err(err) = state.maybe_persist() {
2704        state.dsl.0.restore_snapshot(previous_snapshot);
2705        return Err(err);
2706    }
2707    Ok(())
2708}
2709
2710fn terminate_owner_locked(state: &mut ShellState, reason: &str) -> Result<(), OpsLifecycleError> {
2711    state.ensure_owner_active()?;
2712    let to_terminate = state.owner_termination_targets()?;
2713
2714    for (op_id, _status) in &to_terminate {
2715        let terminal_outcome = OperationTerminalOutcome::Terminated {
2716            reason: reason.to_owned(),
2717        };
2718        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2719
2720        apply_terminal_op_transition_and_persist(
2721            state,
2722            op_id,
2723            mm_dsl::MeerkatMachineInput::TerminateOp {
2724                operation_id: mm_dsl::OperationId::from_domain(op_id).0,
2725                outcome: outcome_kind,
2726                payload: terminal_outcome,
2727            },
2728            mm_dsl::OpLifecycleActionKind::Terminate,
2729        )?;
2730
2731        if let Some(entry) = state.finalize_terminal(op_id)? {
2732            state.publish_completion_entry(Some(entry));
2733        }
2734    }
2735    Ok(())
2736}
2737
2738fn op_registration_error_from_effects(
2739    id: &OperationId,
2740    effects: &[mm_dsl::MeerkatMachineEffect],
2741) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2742    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2743    let mut admission = None;
2744    for effect in effects {
2745        let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2746            operation_id,
2747            result,
2748            reject_reason,
2749            max_concurrent_limit,
2750            active_op_count,
2751        } = effect
2752        else {
2753            continue;
2754        };
2755        if admission.is_some() {
2756            return Err(OpsLifecycleError::Internal(
2757                "generated op registration authority emitted multiple admission results".into(),
2758            ));
2759        }
2760        if operation_id != &expected_id {
2761            return Err(OpsLifecycleError::Internal(format!(
2762                "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2763            )));
2764        }
2765        admission = Some(match result {
2766            mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2767                if reject_reason.is_some() {
2768                    return Err(OpsLifecycleError::Internal(
2769                        "generated op registration authority accepted with rejection reason".into(),
2770                    ));
2771                }
2772                None
2773            }
2774            mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2775                let reason = reject_reason.ok_or_else(|| {
2776                    OpsLifecycleError::Internal(
2777                        "generated op registration authority rejected without reason".into(),
2778                    )
2779                })?;
2780                let error = match reason {
2781                    mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2782                        OpsLifecycleError::AlreadyRegistered(id.clone())
2783                    }
2784                    mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2785                        let limit = max_concurrent_limit.ok_or_else(|| {
2786                            OpsLifecycleError::Internal(
2787                                "generated op registration authority rejected capacity without limit"
2788                                    .into(),
2789                            )
2790                        })?;
2791                        OpsLifecycleError::MaxConcurrentExceeded {
2792                            limit: limit as usize,
2793                            active: *active_op_count as usize,
2794                        }
2795                    }
2796                };
2797                Some(error)
2798            }
2799        });
2800    }
2801    admission.ok_or_else(|| {
2802        OpsLifecycleError::Internal(
2803            "generated op registration authority emitted no admission result".into(),
2804        )
2805    })
2806}
2807
2808impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2809    fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2810        self.register_operation_with_admission_limit(spec, None)
2811    }
2812
2813    fn register_operation_with_admission_limit(
2814        &self,
2815        mut spec: OperationSpec,
2816        max_concurrent: Option<usize>,
2817    ) -> Result<(), OpsLifecycleError> {
2818        let mut state = self.write_state()?;
2819        state.ensure_owner_active()?;
2820        let operation_id = spec.id.clone();
2821        let kind = spec.kind;
2822        let max_concurrent = max_concurrent
2823            .or(state.max_concurrent)
2824            .map(|limit| limit as u64);
2825
2826        let effects = state.dsl_apply_with_effects(
2827            mm_dsl::MeerkatMachineInput::RegisterOp {
2828                operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2829                kind: mm_dsl::OperationKind::from_domain(&kind),
2830                source: spec
2831                    .operation_source
2832                    .as_ref()
2833                    .map(mm_dsl::OperationSource::from_domain),
2834                max_concurrent,
2835            },
2836            "RegisterOp",
2837        )?;
2838        if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
2839            return Err(error);
2840        }
2841
2842        let authority_operation_source = state.operation_source(&operation_id)?;
2843        ShellState::align_spec_child_session_id_to_source(
2844            &mut spec,
2845            authority_operation_source.as_ref(),
2846        );
2847
2848        // Insert shell record.
2849        state.records.insert(operation_id, ShellRecord::new(spec));
2850        Ok(())
2851    }
2852
2853    fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2854        let mut state = self.write_state()?;
2855
2856        apply_op_transition(
2857            &mut state,
2858            id,
2859            mm_dsl::MeerkatMachineInput::StartOp {
2860                operation_id: mm_dsl::OperationId::from_domain(id).0,
2861            },
2862            mm_dsl::OpLifecycleActionKind::Start,
2863        )?;
2864
2865        // Shell concern: record the started timestamp.
2866        if let Some(shell) = state.records.get_mut(id) {
2867            shell.started_at = Some(Instant::now());
2868        }
2869        Ok(())
2870    }
2871
2872    fn provisioning_failed(
2873        &self,
2874        id: &OperationId,
2875        error: String,
2876    ) -> Result<(), OpsLifecycleError> {
2877        let mut state = self.write_state()?;
2878
2879        let terminal_outcome = OperationTerminalOutcome::Failed { error };
2880        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2881
2882        apply_terminal_op_transition_and_persist(
2883            &mut state,
2884            id,
2885            mm_dsl::MeerkatMachineInput::FailOp {
2886                operation_id: mm_dsl::OperationId::from_domain(id).0,
2887                outcome: outcome_kind,
2888                payload: terminal_outcome,
2889            },
2890            mm_dsl::OpLifecycleActionKind::Fail,
2891        )?;
2892
2893        let completion_entry = state.finalize_terminal(id)?;
2894        state.publish_completion_entry(completion_entry);
2895        Ok(())
2896    }
2897
2898    fn peer_ready(
2899        &self,
2900        id: &OperationId,
2901        peer: OperationPeerHandle,
2902    ) -> Result<(), OpsLifecycleError> {
2903        let mut state = self.write_state()?;
2904
2905        apply_op_transition(
2906            &mut state,
2907            id,
2908            mm_dsl::MeerkatMachineInput::PeerReadyOp {
2909                operation_id: mm_dsl::OperationId::from_domain(id).0,
2910            },
2911            mm_dsl::OpLifecycleActionKind::PeerReady,
2912        )?;
2913
2914        // Shell concern: store the peer handle.
2915        if let Some(shell) = state.records.get_mut(id) {
2916            shell.peer_handle = Some(peer);
2917        }
2918        Ok(())
2919    }
2920
2921    fn register_watcher(
2922        &self,
2923        id: &OperationId,
2924    ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
2925        let mut state = self.write_state()?;
2926
2927        if !state.contains(id) {
2928            return Err(OpsLifecycleError::NotFound(id.clone()));
2929        }
2930
2931        // If already terminal, return an already-resolved watch.
2932        if let Some(outcome) = state.terminal_outcome(id)? {
2933            return Ok(resolved_operation_completion_watch(outcome));
2934        }
2935
2936        // Shell concern: create the channel and store the sender.
2937        let shell = state.shell_record_mut(id)?;
2938        let (tx, rx) = tokio::sync::oneshot::channel();
2939        let watch = operation_completion_watch_from_receiver(rx);
2940        shell.watchers.push(OperationCompletionNotifier::new(tx));
2941        Ok(watch)
2942    }
2943
2944    fn report_progress(
2945        &self,
2946        id: &OperationId,
2947        _update: OperationProgressUpdate,
2948    ) -> Result<(), OpsLifecycleError> {
2949        let mut state = self.write_state()?;
2950
2951        apply_op_transition(
2952            &mut state,
2953            id,
2954            mm_dsl::MeerkatMachineInput::ProgressReportedOp {
2955                operation_id: mm_dsl::OperationId::from_domain(id).0,
2956            },
2957            mm_dsl::OpLifecycleActionKind::ProgressReported,
2958        )?;
2959        Ok(())
2960    }
2961
2962    fn complete_operation(
2963        &self,
2964        id: &OperationId,
2965        result: OperationResult,
2966    ) -> Result<(), OpsLifecycleError> {
2967        let mut state = self.write_state()?;
2968
2969        let terminal_outcome = OperationTerminalOutcome::Completed(result);
2970        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2971
2972        apply_terminal_op_transition_and_persist(
2973            &mut state,
2974            id,
2975            mm_dsl::MeerkatMachineInput::CompleteOp {
2976                operation_id: mm_dsl::OperationId::from_domain(id).0,
2977                outcome: outcome_kind,
2978                payload: terminal_outcome,
2979            },
2980            mm_dsl::OpLifecycleActionKind::Complete,
2981        )?;
2982
2983        let completion_entry = state.finalize_terminal(id)?;
2984        state.publish_completion_entry(completion_entry);
2985        Ok(())
2986    }
2987
2988    fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
2989        let mut state = self.write_state()?;
2990
2991        let terminal_outcome = OperationTerminalOutcome::Failed { error };
2992        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2993
2994        apply_terminal_op_transition_and_persist(
2995            &mut state,
2996            id,
2997            mm_dsl::MeerkatMachineInput::FailOp {
2998                operation_id: mm_dsl::OperationId::from_domain(id).0,
2999                outcome: outcome_kind,
3000                payload: terminal_outcome,
3001            },
3002            mm_dsl::OpLifecycleActionKind::Fail,
3003        )?;
3004
3005        let completion_entry = state.finalize_terminal(id)?;
3006        state.publish_completion_entry(completion_entry);
3007        Ok(())
3008    }
3009
3010    fn abort_provisioning(
3011        &self,
3012        id: &OperationId,
3013        reason: Option<String>,
3014    ) -> Result<(), OpsLifecycleError> {
3015        let mut state = self.write_state()?;
3016
3017        let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
3018        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3019
3020        apply_terminal_op_transition_and_persist(
3021            &mut state,
3022            id,
3023            mm_dsl::MeerkatMachineInput::AbortOp {
3024                operation_id: mm_dsl::OperationId::from_domain(id).0,
3025                outcome: outcome_kind,
3026                payload: terminal_outcome,
3027            },
3028            mm_dsl::OpLifecycleActionKind::Abort,
3029        )?;
3030
3031        let completion_entry = state.finalize_terminal(id)?;
3032        state.publish_completion_entry(completion_entry);
3033        Ok(())
3034    }
3035
3036    fn cancel_operation(
3037        &self,
3038        id: &OperationId,
3039        reason: Option<String>,
3040    ) -> Result<(), OpsLifecycleError> {
3041        let mut state = self.write_state()?;
3042
3043        let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
3044        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3045
3046        apply_terminal_op_transition_and_persist(
3047            &mut state,
3048            id,
3049            mm_dsl::MeerkatMachineInput::CancelOp {
3050                operation_id: mm_dsl::OperationId::from_domain(id).0,
3051                outcome: outcome_kind,
3052                payload: terminal_outcome,
3053            },
3054            mm_dsl::OpLifecycleActionKind::Cancel,
3055        )?;
3056
3057        let completion_entry = state.finalize_terminal(id)?;
3058        state.publish_completion_entry(completion_entry);
3059        Ok(())
3060    }
3061
3062    fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3063        let mut state = self.write_state()?;
3064
3065        apply_op_transition(
3066            &mut state,
3067            id,
3068            mm_dsl::MeerkatMachineInput::RetireRequestedOp {
3069                operation_id: mm_dsl::OperationId::from_domain(id).0,
3070            },
3071            mm_dsl::OpLifecycleActionKind::RetireRequested,
3072        )?;
3073        Ok(())
3074    }
3075
3076    fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3077        let mut state = self.write_state()?;
3078
3079        let terminal_outcome = OperationTerminalOutcome::Retired;
3080        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3081
3082        apply_terminal_op_transition_and_persist(
3083            &mut state,
3084            id,
3085            mm_dsl::MeerkatMachineInput::RetireCompletedOp {
3086                operation_id: mm_dsl::OperationId::from_domain(id).0,
3087                outcome: outcome_kind,
3088                payload: terminal_outcome,
3089            },
3090            mm_dsl::OpLifecycleActionKind::RetireCompleted,
3091        )?;
3092
3093        let completion_entry = state.finalize_terminal(id)?;
3094        state.publish_completion_entry(completion_entry);
3095        Ok(())
3096    }
3097
3098    fn snapshot(
3099        &self,
3100        id: &OperationId,
3101    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3102        let state = self.read_state()?;
3103        state.snapshot(id)
3104    }
3105
3106    fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3107        let state = self.read_state()?;
3108        let mut snapshots = Vec::new();
3109        for id in state.operation_ids()? {
3110            let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3111                OpsLifecycleError::Internal(format!(
3112                    "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3113                ))
3114            })?;
3115            snapshots.push(snapshot);
3116        }
3117        snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3118        Ok(snapshots)
3119    }
3120
3121    fn classify_operation_terminality(
3122        &self,
3123        id: &OperationId,
3124        status: OperationStatus,
3125    ) -> Result<bool, OpsLifecycleError> {
3126        ShellState::operation_status_is_terminal(id, status)
3127    }
3128
3129    fn classify_operation_public_result(
3130        &self,
3131        id: &OperationId,
3132    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3133        let state = self.read_state()?;
3134        let status = match state.status(id) {
3135            Some(status) => status,
3136            None if state.records.contains_key(id)
3137                || state.has_generated_operation_record_fact(id) =>
3138            {
3139                return Err(OpsLifecycleError::Internal(format!(
3140                    "generated op lifecycle authority missing status for {id}"
3141                )));
3142            }
3143            None => OperationStatus::Absent,
3144        };
3145        ShellState::operation_public_result_class(id, status)
3146    }
3147
3148    fn classify_operation_completion_wake(
3149        &self,
3150        id: &OperationId,
3151        kind: OperationKind,
3152    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3153        ShellState::operation_completion_wake_class(id, kind)
3154    }
3155
3156    fn classify_operation_transition_idempotence(
3157        &self,
3158        id: &OperationId,
3159        action: OperationLifecycleAction,
3160    ) -> Result<bool, OpsLifecycleError> {
3161        let state = self.read_state()?;
3162        let status = match state.status(id) {
3163            Some(status) => status,
3164            None if state.records.contains_key(id)
3165                || state.has_generated_operation_record_fact(id) =>
3166            {
3167                return Err(OpsLifecycleError::Internal(format!(
3168                    "generated op lifecycle authority missing status for {id}"
3169                )));
3170            }
3171            None => OperationStatus::Absent,
3172        };
3173        ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3174    }
3175
3176    fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3177        let mut state = self.write_state()?;
3178        if state.owner_retired {
3179            return Ok(());
3180        }
3181        terminate_owner_locked(&mut state, &reason)
3182    }
3183
3184    fn collect_completed(
3185        &self,
3186    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3187        let mut state = self.write_state()?;
3188        state.ensure_owner_active()?;
3189
3190        let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3191        let mut collected = Vec::with_capacity(ids.len());
3192        for id in ids {
3193            let outcome = state.terminal_outcome(&id)?;
3194            state.dsl_apply(
3195                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3196                    operation_id: mm_dsl::OperationId::from_domain(&id).0,
3197                },
3198                "CollectCompletedOp",
3199            )?;
3200            state.completed_order.retain(|queued| queued != &id);
3201            state.records.remove(&id);
3202            if let Some(outcome) = outcome {
3203                collected.push((id, outcome));
3204            }
3205        }
3206        Ok(collected)
3207    }
3208
3209    fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3210        Some(self.completion_feed_handle())
3211    }
3212
3213    fn completion_cursor(
3214        &self,
3215        consumer: CompletionCursorConsumer,
3216    ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3217        let state = self.read_state()?;
3218        Ok(Some(state.completion_cursor(consumer)))
3219    }
3220
3221    fn advance_completion_cursor(
3222        &self,
3223        consumer: CompletionCursorConsumer,
3224        cursor: CompletionSeq,
3225        projection: Option<&meerkat_core::EpochCursorState>,
3226    ) -> Result<CompletionSeq, OpsLifecycleError> {
3227        let mut state = self.write_state()?;
3228        state.ensure_owner_active()?;
3229        let input = match consumer {
3230            CompletionCursorConsumer::AgentApplied => {
3231                mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3232            }
3233            CompletionCursorConsumer::RuntimeObserved => {
3234                mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3235            }
3236            CompletionCursorConsumer::RuntimeInjected => {
3237                mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3238            }
3239        };
3240        let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3241        let advanced = effects
3242            .iter()
3243            .find_map(|effect| match (consumer, effect) {
3244                (
3245                    CompletionCursorConsumer::AgentApplied,
3246                    mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3247                ) => Some(*cursor),
3248                (
3249                    CompletionCursorConsumer::RuntimeObserved,
3250                    mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3251                        cursor,
3252                    },
3253                ) => Some(*cursor),
3254                (
3255                    CompletionCursorConsumer::RuntimeInjected,
3256                    mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3257                        cursor,
3258                    },
3259                ) => Some(*cursor),
3260                _ => None,
3261            })
3262            .ok_or_else(|| {
3263                OpsLifecycleError::Internal(format!(
3264                    "generated completion cursor transition emitted no feedback for {consumer:?}"
3265                ))
3266            })?;
3267        if let Some(projection) = projection {
3268            projection.project_authorized_completion_cursor(consumer, advanced);
3269        }
3270        Ok(advanced)
3271    }
3272
3273    fn wait_all(
3274        &self,
3275        run_id: &RunId,
3276        ids: &[OperationId],
3277    ) -> std::pin::Pin<
3278        Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3279    > {
3280        let wait_request_id = WaitRequestId::new();
3281        let owned_ids = ids.to_vec();
3282
3283        let state = match self.write_state() {
3284            Ok(mut state) => {
3285                if let Err(error) = state.ensure_owner_active() {
3286                    return Box::pin(WaitAllFuture {
3287                        registry: self,
3288                        wait_request_id,
3289                        state: WaitAllFutureState::Ready(Some(Err(error))),
3290                    });
3291                }
3292                match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3293                    Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3294                        let outcomes =
3295                            state
3296                                .collect_wait_outcomes(&satisfied.operation_ids)
3297                                .map(|outcomes| WaitAllResult {
3298                                    outcomes,
3299                                    satisfied,
3300                                });
3301                        WaitAllFutureState::Ready(Some(outcomes))
3302                    }
3303                    Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3304                        if state.pending_wait.is_some() {
3305                            // Roll back the DSL barrier we just activated so the
3306                            // registry is not stuck in a wait-active state with
3307                            // no correlation oneshot to resolve. `CancelWaitAll`
3308                            // is the no-obligation clearer (members need not be
3309                            // terminal).
3310                            let rollback = state.dsl_apply(
3311                                mm_dsl::MeerkatMachineInput::CancelWaitAll,
3312                                "CancelWaitAll(rollback)",
3313                            );
3314                            return Box::pin(WaitAllFuture {
3315                                registry: self,
3316                                wait_request_id,
3317                                state: WaitAllFutureState::Ready(Some(Err(match rollback {
3318                                    Ok(()) => OpsLifecycleError::Internal(
3319                                        "wait_all started while a pending wait sender already existed"
3320                                            .into(),
3321                                    ),
3322                                    Err(err) => err,
3323                                }))),
3324                            });
3325                        }
3326                        state.wait_request_id = Some(wait_request_id.clone());
3327                        let (sender, receiver) = tokio::sync::oneshot::channel();
3328                        state.pending_wait = Some(PendingWaitState {
3329                            wait_request_id: wait_request_id.clone(),
3330                            sender,
3331                        });
3332                        WaitAllFutureState::Waiting(receiver)
3333                    }
3334                    Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3335                }
3336            }
3337            Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3338        };
3339
3340        Box::pin(WaitAllFuture {
3341            registry: self,
3342            wait_request_id,
3343            state,
3344        })
3345    }
3346}
3347
3348#[cfg(test)]
3349#[allow(clippy::unwrap_used, clippy::panic)]
3350mod tests {
3351    use super::*;
3352    use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3353    use meerkat_core::lifecycle::RunId;
3354    use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3355    use meerkat_core::types::SessionId;
3356    use std::sync::atomic::Ordering;
3357    use uuid::Uuid;
3358
3359    fn test_run_id() -> RunId {
3360        RunId(Uuid::from_u128(1))
3361    }
3362
3363    fn background_spec(name: &str) -> OperationSpec {
3364        OperationSpec {
3365            id: OperationId::new(),
3366            kind: OperationKind::BackgroundToolOp,
3367            owner_session_id: SessionId::new(),
3368            display_name: name.into(),
3369            source_label: "test".into(),
3370            operation_source: None,
3371            child_session_id: None,
3372            expect_peer_channel: false,
3373        }
3374    }
3375
3376    #[tokio::test]
3377    async fn late_watchers_resolve_immediately() {
3378        let registry = RuntimeOpsLifecycleRegistry::new();
3379        let spec = background_spec("late");
3380        let op_id = spec.id.clone();
3381        registry.register_operation(spec).unwrap();
3382        registry.provisioning_succeeded(&op_id).unwrap();
3383        registry
3384            .complete_operation(
3385                &op_id,
3386                OperationResult {
3387                    id: op_id.clone(),
3388                    content: "done".into(),
3389                    is_error: false,
3390                    duration_ms: 1,
3391                    tokens_used: 0,
3392                },
3393            )
3394            .unwrap();
3395
3396        let watch = registry.register_watcher(&op_id).unwrap();
3397        match watch
3398            .await
3399            .expect("operation completion watch should resolve")
3400        {
3401            OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3402            other => panic!("expected completed outcome, got {other:?}"),
3403        }
3404    }
3405
3406    #[tokio::test]
3407    async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3408        let (tx, rx) = tokio::sync::oneshot::channel();
3409        let watch = operation_completion_watch_from_receiver(rx);
3410        drop(tx);
3411
3412        assert_eq!(
3413            watch.await,
3414            Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3415        );
3416    }
3417
3418    #[test]
3419    fn peer_ready_requires_peer_expectation() {
3420        let registry = RuntimeOpsLifecycleRegistry::new();
3421        let spec = background_spec("no-peer");
3422        let op_id = spec.id.clone();
3423        registry.register_operation(spec).unwrap();
3424        registry.provisioning_succeeded(&op_id).unwrap();
3425
3426        let result = registry.peer_ready(
3427            &op_id,
3428            OperationPeerHandle {
3429                peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3430                trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3431                    "peer",
3432                    PeerId::new(),
3433                    "inproc://peer",
3434                )
3435                .unwrap(),
3436            },
3437        );
3438        assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3439    }
3440
3441    /// K8b pinning test: the terminal payload carried through the generated
3442    /// machine IS the typed domain outcome — no JSON codec exists in either
3443    /// direction. Every variant classifies to its matching discriminant, the
3444    /// fail-closed read returns the exact payload when the discriminant
3445    /// matches, and rejects any discriminant/variant disagreement.
3446    #[test]
3447    fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3448        let op_id = OperationId::new();
3449        let outcomes = vec![
3450            (
3451                OperationTerminalOutcome::Completed(OperationResult {
3452                    id: op_id.clone(),
3453                    content: "done".into(),
3454                    is_error: false,
3455                    duration_ms: 7,
3456                    tokens_used: 42,
3457                }),
3458                mm_dsl::OperationTerminalOutcomeKind::Completed,
3459            ),
3460            (
3461                OperationTerminalOutcome::Failed {
3462                    error: "boom".into(),
3463                },
3464                mm_dsl::OperationTerminalOutcomeKind::Failed,
3465            ),
3466            (
3467                OperationTerminalOutcome::Aborted {
3468                    reason: Some("user aborted".into()),
3469                },
3470                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3471            ),
3472            (
3473                OperationTerminalOutcome::Aborted { reason: None },
3474                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3475            ),
3476            (
3477                OperationTerminalOutcome::Cancelled {
3478                    reason: Some("cancelled".into()),
3479                },
3480                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3481            ),
3482            (
3483                OperationTerminalOutcome::Cancelled { reason: None },
3484                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3485            ),
3486            (
3487                OperationTerminalOutcome::Retired,
3488                mm_dsl::OperationTerminalOutcomeKind::Retired,
3489            ),
3490            (
3491                OperationTerminalOutcome::Terminated {
3492                    reason: "owner stopped".into(),
3493                },
3494                mm_dsl::OperationTerminalOutcomeKind::Terminated,
3495            ),
3496        ];
3497
3498        for (outcome, expected_kind) in &outcomes {
3499            assert_eq!(
3500                mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3501                *expected_kind,
3502                "typed payload {outcome:?} must classify to {expected_kind:?}"
3503            );
3504            let read = ShellState::checked_terminal_payload(
3505                *expected_kind,
3506                outcome,
3507                "test authority",
3508                "test-op",
3509            )
3510            .expect("matching discriminant must read back the exact payload");
3511            assert_eq!(&read, outcome);
3512        }
3513
3514        // Discriminant/variant disagreement fails closed.
3515        let err = ShellState::checked_terminal_payload(
3516            mm_dsl::OperationTerminalOutcomeKind::Completed,
3517            &OperationTerminalOutcome::Retired,
3518            "test authority",
3519            "test-op",
3520        )
3521        .expect_err("variant mismatch must be rejected");
3522        assert!(matches!(err, OpsLifecycleError::Internal(_)));
3523    }
3524
3525    /// K8b machine-ownership pin: a terminal payload whose variant disagrees
3526    /// with the transition's terminal kind is rejected by the generated
3527    /// machine guard (`payload_variant_matches_kind`) — not by any shell
3528    /// decode step. Drives the DSL input directly to prove the guard owns
3529    /// the invariant.
3530    #[test]
3531    fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3532        let registry = RuntimeOpsLifecycleRegistry::new();
3533        let spec = background_spec("variant-mismatch");
3534        let op_id = spec.id.clone();
3535        registry.register_operation(spec).unwrap();
3536        registry.provisioning_succeeded(&op_id).unwrap();
3537
3538        let mut state = registry.write_state().unwrap();
3539        let err = state
3540            .dsl_apply(
3541                mm_dsl::MeerkatMachineInput::CompleteOp {
3542                    operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3543                    outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3544                    // Variant mismatch: Completed kind with a Retired payload.
3545                    payload: OperationTerminalOutcome::Retired,
3546                },
3547                "CompleteOp",
3548            )
3549            .expect_err("machine must reject payload variant mismatch");
3550        // The kernel reports guard rejections without naming the guard; the
3551        // discriminating pin is the pair: identical input EXCEPT the payload
3552        // variant is guard-rejected here, then accepted below. Status
3553        // (`Running`) and discriminant (`Completed`) are identical in both
3554        // calls, so `payload_variant_matches_kind` is the only differing
3555        // guard.
3556        assert!(
3557            matches!(
3558                &err,
3559                OpsLifecycleError::Internal(message)
3560                    if message.contains("GuardRejected") && message.contains("CompleteOp")
3561            ),
3562            "expected generated guard rejection for CompleteOp, got: {err:?}"
3563        );
3564        drop(state);
3565
3566        // RetireCompletedOp requires the unit Retired payload; a data-carrying
3567        // payload is rejected by the same guard shape.
3568        registry
3569            .complete_operation(
3570                &op_id,
3571                OperationResult {
3572                    id: op_id.clone(),
3573                    content: "done".into(),
3574                    is_error: false,
3575                    duration_ms: 1,
3576                    tokens_used: 0,
3577                },
3578            )
3579            .expect("matching variant must complete");
3580    }
3581
3582    #[test]
3583    fn duplicate_registration_rejection_is_generated() {
3584        let registry = RuntimeOpsLifecycleRegistry::new();
3585        let spec = background_spec("duplicate");
3586        let op_id = spec.id.clone();
3587
3588        registry.register_operation(spec.clone()).unwrap();
3589        let result = registry.register_operation(spec);
3590
3591        assert!(matches!(
3592            result,
3593            Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3594        ));
3595    }
3596
3597    #[test]
3598    fn invalid_transition_rejection_is_generated() {
3599        let registry = RuntimeOpsLifecycleRegistry::new();
3600        let spec = background_spec("invalid-transition");
3601        let op_id = spec.id.clone();
3602        registry.register_operation(spec).unwrap();
3603
3604        let result = registry.complete_operation(
3605            &op_id,
3606            OperationResult {
3607                id: op_id.clone(),
3608                content: "too-early".into(),
3609                is_error: false,
3610                duration_ms: 1,
3611                tokens_used: 0,
3612            },
3613        );
3614
3615        assert!(matches!(
3616            result,
3617            Err(OpsLifecycleError::InvalidTransition {
3618                id,
3619                status: OperationStatus::Provisioning,
3620                action: "complete_operation",
3621            }) if id == op_id
3622        ));
3623    }
3624
3625    #[tokio::test]
3626    async fn multi_listener_completion() {
3627        let registry = RuntimeOpsLifecycleRegistry::new();
3628        let spec = background_spec("multi");
3629        let op_id = spec.id.clone();
3630        registry.register_operation(spec).unwrap();
3631        registry.provisioning_succeeded(&op_id).unwrap();
3632
3633        let watch1 = registry.register_watcher(&op_id).unwrap();
3634        let watch2 = registry.register_watcher(&op_id).unwrap();
3635        let watch3 = registry.register_watcher(&op_id).unwrap();
3636
3637        registry
3638            .complete_operation(
3639                &op_id,
3640                OperationResult {
3641                    id: op_id.clone(),
3642                    content: "multi-done".into(),
3643                    is_error: false,
3644                    duration_ms: 1,
3645                    tokens_used: 0,
3646                },
3647            )
3648            .unwrap();
3649
3650        for watch in [watch1, watch2, watch3] {
3651            match watch
3652                .await
3653                .expect("operation completion watch should resolve")
3654            {
3655                OperationTerminalOutcome::Completed(result) => {
3656                    assert_eq!(result.content, "multi-done");
3657                }
3658                other => panic!("expected completed, got {other:?}"),
3659            }
3660        }
3661    }
3662
3663    #[tokio::test]
3664    async fn wait_all_returns_all_outcomes() {
3665        let registry = RuntimeOpsLifecycleRegistry::new();
3666
3667        let spec_a = background_spec("a");
3668        let id_a = spec_a.id.clone();
3669        registry.register_operation(spec_a).unwrap();
3670        registry.provisioning_succeeded(&id_a).unwrap();
3671
3672        let spec_b = background_spec("b");
3673        let id_b = spec_b.id.clone();
3674        registry.register_operation(spec_b).unwrap();
3675        registry.provisioning_succeeded(&id_b).unwrap();
3676
3677        registry
3678            .complete_operation(
3679                &id_a,
3680                OperationResult {
3681                    id: id_a.clone(),
3682                    content: "a-done".into(),
3683                    is_error: false,
3684                    duration_ms: 1,
3685                    tokens_used: 0,
3686                },
3687            )
3688            .unwrap();
3689        registry.fail_operation(&id_b, "b-error".into()).unwrap();
3690
3691        let wait_result = registry
3692            .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3693            .await
3694            .unwrap();
3695        assert_eq!(wait_result.outcomes.len(), 2);
3696        assert_eq!(wait_result.outcomes[0].0, id_a);
3697        assert!(matches!(
3698            wait_result.outcomes[0].1,
3699            OperationTerminalOutcome::Completed(_)
3700        ));
3701        assert_eq!(wait_result.outcomes[1].0, id_b);
3702        assert!(matches!(
3703            wait_result.outcomes[1].1,
3704            OperationTerminalOutcome::Failed { .. }
3705        ));
3706        // Obligation carries the awaited IDs
3707        assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3708        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3709    }
3710
3711    /// Exercises the trait `wait_all` path (via `dyn OpsLifecycleRegistry`)
3712    /// which must submit WaitAll through the DSL for cross-machine handoff.
3713    #[tokio::test]
3714    async fn wait_all_trait_path_submits_through_authority() {
3715        let registry = RuntimeOpsLifecycleRegistry::new();
3716        let spec = background_spec("trait-wait");
3717        let op_id = spec.id.clone();
3718        registry.register_operation(spec).unwrap();
3719        registry.provisioning_succeeded(&op_id).unwrap();
3720        registry
3721            .complete_operation(
3722                &op_id,
3723                OperationResult {
3724                    id: op_id.clone(),
3725                    content: "done".into(),
3726                    is_error: false,
3727                    duration_ms: 1,
3728                    tokens_used: 0,
3729                },
3730            )
3731            .unwrap();
3732
3733        // Call through trait object to exercise the trait impl, not the inherent method.
3734        let trait_ref: &dyn OpsLifecycleRegistry = &registry;
3735        let wait_result = trait_ref
3736            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3737            .await
3738            .unwrap();
3739        assert_eq!(wait_result.outcomes.len(), 1);
3740        assert!(matches!(
3741            wait_result.outcomes[0].1,
3742            OperationTerminalOutcome::Completed(_)
3743        ));
3744        // Obligation carries the validated ID
3745        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3746        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3747        let state = registry.read_state().unwrap();
3748        assert!(
3749            !state.wait_active(),
3750            "already-satisfied wait_all must be cleared by generated satisfaction authority"
3751        );
3752        assert!(state.wait_operation_ids().unwrap().is_empty());
3753    }
3754
3755    #[tokio::test]
3756    async fn wait_all_duplicate_rejection_is_generated() {
3757        let registry = RuntimeOpsLifecycleRegistry::new();
3758        let spec = background_spec("duplicate-wait");
3759        let op_id = spec.id.clone();
3760        registry.register_operation(spec).unwrap();
3761        registry.provisioning_succeeded(&op_id).unwrap();
3762
3763        let result = registry
3764            .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3765            .await;
3766
3767        assert!(matches!(
3768            result,
3769            Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3770        ));
3771        let state = registry.read_state().unwrap();
3772        assert!(
3773            !state.wait_active(),
3774            "duplicate wait rejection must not create a shell or machine barrier"
3775        );
3776        assert!(state.wait_operation_ids().unwrap().is_empty());
3777    }
3778
3779    #[tokio::test]
3780    async fn wait_all_active_rejection_is_generated() {
3781        let registry = RuntimeOpsLifecycleRegistry::new();
3782        let spec = background_spec("active-wait");
3783        let op_id = spec.id.clone();
3784        registry.register_operation(spec).unwrap();
3785        registry.provisioning_succeeded(&op_id).unwrap();
3786
3787        let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3788        let result = registry
3789            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3790            .await;
3791
3792        assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3793        drop(active_wait);
3794        let state = registry.read_state().unwrap();
3795        assert!(!state.wait_active());
3796        assert!(state.wait_operation_ids().unwrap().is_empty());
3797    }
3798
3799    #[tokio::test]
3800    async fn wait_all_unknown_operation_rejection_is_generated() {
3801        let registry = RuntimeOpsLifecycleRegistry::new();
3802        let op_id = OperationId::new();
3803
3804        let result = registry
3805            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3806            .await;
3807
3808        assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3809        let state = registry.read_state().unwrap();
3810        assert!(!state.wait_active());
3811        assert!(state.wait_operation_ids().unwrap().is_empty());
3812    }
3813
3814    #[tokio::test]
3815    async fn wait_all_resolves_from_authority_owned_wait_request() {
3816        let registry = RuntimeOpsLifecycleRegistry::new();
3817        let run_id = test_run_id();
3818
3819        let spec = background_spec("pending");
3820        let op_id = spec.id.clone();
3821        registry.register_operation(spec).unwrap();
3822        registry.provisioning_succeeded(&op_id).unwrap();
3823
3824        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3825        tokio::pin!(wait_fut);
3826        assert!(
3827            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3828                .await
3829                .is_err()
3830        );
3831
3832        let active_wait_request_id = {
3833            let state = registry.read_state().unwrap();
3834            let wait_request_id = match state.wait_request_id.clone() {
3835                Some(wait_request_id) => wait_request_id,
3836                None => panic!("wait request should be active"),
3837            };
3838            assert_eq!(
3839                state.wait_operation_ids().unwrap().as_slice(),
3840                std::slice::from_ref(&op_id)
3841            );
3842            wait_request_id
3843        };
3844
3845        registry
3846            .complete_operation(
3847                &op_id,
3848                OperationResult {
3849                    id: op_id.clone(),
3850                    content: "done".into(),
3851                    is_error: false,
3852                    duration_ms: 1,
3853                    tokens_used: 0,
3854                },
3855            )
3856            .unwrap();
3857
3858        let wait_result = wait_fut.await.unwrap();
3859        assert_eq!(
3860            wait_result.satisfied.wait_request_id,
3861            active_wait_request_id
3862        );
3863        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
3864        assert!(matches!(
3865            wait_result.outcomes.as_slice(),
3866            [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
3867        ));
3868        assert!(registry.read_state().unwrap().wait_request_id.is_none());
3869    }
3870
3871    #[tokio::test]
3872    async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
3873        let registry = RuntimeOpsLifecycleRegistry::new();
3874        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3875        registry.set_persistence_channel(
3876            tx,
3877            meerkat_core::RuntimeEpochId::new(),
3878            Arc::new(meerkat_core::EpochCursorState::new()),
3879        );
3880        let worker = std::thread::spawn(move || {
3881            let request = rx.blocking_recv().expect("persistence request");
3882            let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
3883                "injected persistence failure".to_string(),
3884            )));
3885        });
3886
3887        let spec = background_spec("feed-after-persist");
3888        let op_id = spec.id.clone();
3889        registry.register_operation(spec).unwrap();
3890        registry.provisioning_succeeded(&op_id).unwrap();
3891        let watch = registry.register_watcher(&op_id).unwrap();
3892
3893        let err = registry
3894            .complete_operation(
3895                &op_id,
3896                OperationResult {
3897                    id: op_id.clone(),
3898                    content: "done".into(),
3899                    is_error: false,
3900                    duration_ms: 1,
3901                    tokens_used: 0,
3902                },
3903            )
3904            .expect_err("persistence failure must fail the terminal transition");
3905        assert!(
3906            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
3907            "unexpected persistence error: {err:?}"
3908        );
3909        worker.join().expect("persistence worker");
3910
3911        let feed = registry.completion_feed_handle();
3912        let batch = feed.list_since(0);
3913        assert!(
3914            batch.entries.is_empty(),
3915            "completion feed must not publish entries before durable snapshot success"
3916        );
3917        assert_eq!(batch.watermark, 0);
3918        assert!(
3919            tokio::time::timeout(std::time::Duration::from_millis(10), watch)
3920                .await
3921                .is_err(),
3922            "watchers must not resolve before durable terminal snapshot success"
3923        );
3924        assert_eq!(
3925            registry.snapshot(&op_id).unwrap().unwrap().status,
3926            OperationStatus::Running,
3927            "failed persistence must roll the generated terminal transition back"
3928        );
3929
3930        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3931        registry.set_persistence_channel(
3932            tx,
3933            meerkat_core::RuntimeEpochId::new(),
3934            Arc::new(meerkat_core::EpochCursorState::new()),
3935        );
3936        let worker = std::thread::spawn(move || {
3937            let request = rx.blocking_recv().expect("second persistence request");
3938            let _ = request.result_tx.send(Ok(()));
3939        });
3940        registry
3941            .complete_operation(
3942                &op_id,
3943                OperationResult {
3944                    id: op_id.clone(),
3945                    content: "done".into(),
3946                    is_error: false,
3947                    duration_ms: 1,
3948                    tokens_used: 0,
3949                },
3950            )
3951            .expect("rolled-back terminal transition should remain retryable");
3952        worker.join().expect("second persistence worker");
3953
3954        let batch = feed.list_since(0);
3955        assert_eq!(batch.entries.len(), 1);
3956        assert_eq!(batch.entries[0].operation_id, op_id);
3957    }
3958
3959    #[tokio::test]
3960    async fn dropping_wait_all_future_cancels_active_wait_request() {
3961        let registry = RuntimeOpsLifecycleRegistry::new();
3962        let run_id = test_run_id();
3963
3964        let spec = background_spec("cancelled-wait");
3965        let op_id = spec.id.clone();
3966        registry.register_operation(spec).unwrap();
3967        registry.provisioning_succeeded(&op_id).unwrap();
3968
3969        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3970        drop(wait_fut);
3971
3972        let state = registry.read_state().unwrap();
3973        assert!(state.wait_request_id.is_none());
3974        assert!(state.wait_operation_ids().unwrap().is_empty());
3975        assert!(!state.wait_active());
3976    }
3977
3978    /// id 101: an authority-invariant corruption surfaced during a terminal
3979    /// transition must NOT report the op terminal with a silently-hung barrier.
3980    /// The terminal call must return the typed `Internal` fault, AND the awaited
3981    /// `wait_all` future must resolve to `Err` (via the dropped-sender arm)
3982    /// instead of hanging forever.
3983    #[tokio::test]
3984    async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
3985        let registry = RuntimeOpsLifecycleRegistry::new();
3986        let run_id = test_run_id();
3987
3988        let spec = background_spec("corrupt-barrier");
3989        let op_id = spec.id.clone();
3990        registry.register_operation(spec).unwrap();
3991        registry.provisioning_succeeded(&op_id).unwrap();
3992
3993        // Activate a barrier whose waiter is still pending.
3994        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3995        tokio::pin!(wait_fut);
3996        assert!(
3997            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3998                .await
3999                .is_err(),
4000            "barrier waiter must still be pending before corruption"
4001        );
4002        assert!(registry.read_state().unwrap().wait_request_id.is_some());
4003
4004        // Corrupt the generated wait authority: an active wait_request_id with
4005        // no wait_run_id forces `try_satisfy_wait_all_authority` down its
4006        // non-GuardRejected `Internal` arm during the terminal transition.
4007        {
4008            let mut state = registry.write_state().unwrap();
4009            let mut machine_state = state.dsl.0.state().clone();
4010            machine_state.wait_run_id = None;
4011            state.dsl = DslAuthority(Box::new(
4012                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4013            ));
4014        }
4015
4016        // The terminal transition must FAIL with the typed authority fault, not
4017        // report the op complete.
4018        let err = registry
4019            .complete_operation(
4020                &op_id,
4021                OperationResult {
4022                    id: op_id.clone(),
4023                    content: "done".into(),
4024                    is_error: false,
4025                    duration_ms: 1,
4026                    tokens_used: 0,
4027                },
4028            )
4029            .expect_err("corrupt wait authority must fail the terminal transition");
4030        assert!(
4031            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
4032            "unexpected terminal error: {err:?}"
4033        );
4034
4035        // The waiter must resolve to Err (dropped sender), not hang.
4036        let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
4037            .await
4038            .expect("waiter must resolve, not hang, after authority corruption");
4039        match waiter_result {
4040            Err(OpsLifecycleError::Internal(message)) => assert!(
4041                message.contains("wait_all completion channel dropped"),
4042                "unexpected waiter error message: {message}"
4043            ),
4044            other => panic!("expected dropped-channel Internal error, got {other:?}"),
4045        }
4046    }
4047
4048    /// id 97: a poisoned registry lock must surface as a typed `Internal` fault
4049    /// from `completion_cursor`, NOT laundered into `Ok(None)` (which means "no
4050    /// generated cursor authority") or a bare `None`.
4051    #[test]
4052    fn completion_cursor_propagates_poison_not_none() {
4053        let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
4054
4055        // Poison the registry RwLock by panicking while holding the write guard.
4056        let poison_registry = std::sync::Arc::clone(&registry);
4057        let join = std::thread::spawn(move || {
4058            let _guard = poison_registry.write_state().unwrap();
4059            panic!("intentional panic to poison ops lifecycle registry lock");
4060        });
4061        assert!(
4062            join.join().is_err(),
4063            "poisoning thread must have panicked while holding the write guard"
4064        );
4065
4066        let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
4067        let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
4068        match result {
4069            Err(OpsLifecycleError::Internal(message)) => assert!(
4070                message.contains("ops lifecycle registry poisoned"),
4071                "unexpected cursor error message: {message}"
4072            ),
4073            other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
4074        }
4075    }
4076
4077    #[test]
4078    fn terminate_owner_only_targets_non_terminal_operations() {
4079        let registry = RuntimeOpsLifecycleRegistry::new();
4080
4081        let running_spec = background_spec("running");
4082        let running_id = running_spec.id.clone();
4083        registry.register_operation(running_spec).unwrap();
4084        registry.provisioning_succeeded(&running_id).unwrap();
4085
4086        let completed_spec = background_spec("completed");
4087        let completed_id = completed_spec.id.clone();
4088        registry.register_operation(completed_spec).unwrap();
4089        registry.provisioning_succeeded(&completed_id).unwrap();
4090        registry
4091            .complete_operation(
4092                &completed_id,
4093                OperationResult {
4094                    id: completed_id.clone(),
4095                    content: "done".into(),
4096                    is_error: false,
4097                    duration_ms: 1,
4098                    tokens_used: 0,
4099                },
4100            )
4101            .unwrap();
4102
4103        registry.terminate_owner("shutdown".into()).unwrap();
4104
4105        assert!(matches!(
4106            registry.snapshot(&running_id).unwrap().unwrap().status,
4107            OperationStatus::Terminated
4108        ));
4109        assert!(matches!(
4110            registry.snapshot(&completed_id).unwrap().unwrap().status,
4111            OperationStatus::Completed
4112        ));
4113    }
4114
4115    #[test]
4116    fn unregister_retirement_fences_detached_late_callbacks() {
4117        let registry = RuntimeOpsLifecycleRegistry::new();
4118        let running_spec = background_spec("detached-running");
4119        let running_id = running_spec.id.clone();
4120        registry.register_operation(running_spec).unwrap();
4121        registry.provisioning_succeeded(&running_id).unwrap();
4122
4123        registry
4124            .retire_owner_for_unregister("owner unregistered".into())
4125            .unwrap();
4126        assert!(matches!(
4127            registry.snapshot(&running_id).unwrap().unwrap().status,
4128            OperationStatus::Terminated
4129        ));
4130
4131        let late_result = OperationResult {
4132            id: running_id.clone(),
4133            content: "late detached completion".into(),
4134            is_error: false,
4135            duration_ms: 1,
4136            tokens_used: 0,
4137        };
4138        assert_eq!(
4139            registry.complete_operation(&running_id, late_result),
4140            Err(OpsLifecycleError::OwnerRetired)
4141        );
4142        assert_eq!(
4143            registry.register_operation(background_spec("late-new-op")),
4144            Err(OpsLifecycleError::OwnerRetired)
4145        );
4146        assert_eq!(
4147            registry.report_progress(
4148                &running_id,
4149                OperationProgressUpdate {
4150                    message: "late progress".into(),
4151                    percent: None,
4152                },
4153            ),
4154            Err(OpsLifecycleError::OwnerRetired)
4155        );
4156    }
4157
4158    #[test]
4159    fn collect_completed_drains_terminal_operations() {
4160        let registry = RuntimeOpsLifecycleRegistry::new();
4161
4162        let spec_a = background_spec("a");
4163        let id_a = spec_a.id.clone();
4164        registry.register_operation(spec_a).unwrap();
4165        registry.provisioning_succeeded(&id_a).unwrap();
4166        registry
4167            .complete_operation(
4168                &id_a,
4169                OperationResult {
4170                    id: id_a.clone(),
4171                    content: "done".into(),
4172                    is_error: false,
4173                    duration_ms: 1,
4174                    tokens_used: 0,
4175                },
4176            )
4177            .unwrap();
4178
4179        let spec_b = background_spec("b");
4180        let id_b = spec_b.id.clone();
4181        registry.register_operation(spec_b).unwrap();
4182
4183        let collected = registry.collect_completed().unwrap();
4184        assert_eq!(collected.len(), 1);
4185        assert_eq!(collected[0].0, id_a);
4186
4187        assert!(registry.snapshot(&id_a).unwrap().is_none());
4188        assert!(registry.snapshot(&id_b).unwrap().is_some());
4189
4190        let collected2 = registry.collect_completed().unwrap();
4191        assert!(collected2.is_empty());
4192    }
4193
4194    #[test]
4195    fn bounded_completed_retention_evicts_oldest() {
4196        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4197            max_completed: 3,
4198            max_concurrent: None,
4199        });
4200
4201        let mut ids = Vec::new();
4202        for i in 0..5 {
4203            let spec = background_spec(&format!("op-{i}"));
4204            let id = spec.id.clone();
4205            registry.register_operation(spec).unwrap();
4206            registry.provisioning_succeeded(&id).unwrap();
4207            registry
4208                .complete_operation(
4209                    &id,
4210                    OperationResult {
4211                        id: id.clone(),
4212                        content: format!("done-{i}"),
4213                        is_error: false,
4214                        duration_ms: 1,
4215                        tokens_used: 0,
4216                    },
4217                )
4218                .unwrap();
4219            ids.push(id);
4220        }
4221
4222        assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4223        assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4224        assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4225        assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4226        assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4227    }
4228
4229    #[test]
4230    fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4231        let registry = RuntimeOpsLifecycleRegistry::new();
4232
4233        let completed_spec = background_spec("completed");
4234        let completed_id = completed_spec.id.clone();
4235        registry.register_operation(completed_spec).unwrap();
4236        registry.provisioning_succeeded(&completed_id).unwrap();
4237        registry
4238            .complete_operation(
4239                &completed_id,
4240                OperationResult {
4241                    id: completed_id.clone(),
4242                    content: "done".into(),
4243                    is_error: false,
4244                    duration_ms: 1,
4245                    tokens_used: 0,
4246                },
4247            )
4248            .unwrap();
4249
4250        let running_spec = background_spec("running");
4251        let running_id = running_spec.id.clone();
4252        registry.register_operation(running_spec).unwrap();
4253        registry.provisioning_succeeded(&running_id).unwrap();
4254
4255        let cursor_state = meerkat_core::EpochCursorState::new();
4256        let snapshot = registry
4257            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4258            .unwrap();
4259        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4260
4261        assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4262        assert!(recovered.snapshot(&running_id).unwrap().is_none());
4263
4264        let collected = recovered.collect_completed().unwrap();
4265        assert_eq!(collected.len(), 1);
4266        assert_eq!(collected[0].0, completed_id);
4267    }
4268
4269    #[test]
4270    fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4271        let registry = RuntimeOpsLifecycleRegistry::new();
4272
4273        let mut spec = background_spec("capacity");
4274        spec.kind = OperationKind::BackgroundToolCapacitySlot;
4275        let operation_id = spec.id.clone();
4276        registry.register_operation(spec).unwrap();
4277        registry.provisioning_succeeded(&operation_id).unwrap();
4278        registry.mark_retired(&operation_id).unwrap();
4279
4280        assert!(registry.snapshot(&operation_id).unwrap().is_none());
4281
4282        let cursor_state = meerkat_core::EpochCursorState::new();
4283        let snapshot = registry
4284            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4285            .unwrap();
4286        assert!(
4287            !snapshot
4288                .authority_state
4289                .operations
4290                .contains_key(&operation_id)
4291        );
4292        assert!(!snapshot.operation_specs.contains_key(&operation_id));
4293        assert!(snapshot.completion_entries.is_empty());
4294
4295        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4296        assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4297    }
4298
4299    #[test]
4300    fn recovered_snapshot_uses_authority_operation_source() {
4301        let registry = RuntimeOpsLifecycleRegistry::new();
4302        let child_session_id = SessionId::new();
4303        let operation_source = OperationSource::session_child(child_session_id.clone());
4304        let spec = OperationSpec {
4305            id: OperationId::new(),
4306            kind: OperationKind::MobMemberChild,
4307            owner_session_id: SessionId::new(),
4308            display_name: "source-recovery".into(),
4309            source_label: "test".into(),
4310            operation_source: Some(operation_source.clone()),
4311            child_session_id: Some(child_session_id),
4312            expect_peer_channel: true,
4313        };
4314        let operation_id = spec.id.clone();
4315
4316        registry.register_operation(spec).unwrap();
4317        registry.provisioning_succeeded(&operation_id).unwrap();
4318        registry.mark_retired(&operation_id).unwrap();
4319
4320        let cursor_state = meerkat_core::EpochCursorState::new();
4321        let mut snapshot = registry
4322            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4323            .unwrap();
4324        assert_eq!(
4325            snapshot
4326                .authority_state
4327                .operations
4328                .get(&operation_id)
4329                .and_then(|state| state.operation_source.as_ref()),
4330            Some(&operation_source)
4331        );
4332
4333        snapshot
4334            .operation_specs
4335            .get_mut(&operation_id)
4336            .expect("persisted spec")
4337            .operation_source = None;
4338        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4339        assert_eq!(
4340            recovered
4341                .snapshot(&operation_id)
4342                .unwrap()
4343                .unwrap()
4344                .operation_source,
4345            Some(operation_source)
4346        );
4347    }
4348
4349    #[test]
4350    fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4351        let registry = RuntimeOpsLifecycleRegistry::new();
4352        let child_session_id = SessionId::new();
4353        let operation_source = OperationSource::session_child(child_session_id.clone());
4354        let spec = OperationSpec {
4355            id: OperationId::new(),
4356            kind: OperationKind::MobMemberChild,
4357            owner_session_id: SessionId::new(),
4358            display_name: "source-drift".into(),
4359            source_label: "test".into(),
4360            operation_source: Some(operation_source),
4361            child_session_id: Some(child_session_id),
4362            expect_peer_channel: true,
4363        };
4364        let operation_id = spec.id.clone();
4365
4366        registry.register_operation(spec).unwrap();
4367        registry.provisioning_succeeded(&operation_id).unwrap();
4368        registry.mark_retired(&operation_id).unwrap();
4369
4370        let cursor_state = meerkat_core::EpochCursorState::new();
4371        let mut snapshot = registry
4372            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4373            .unwrap();
4374        snapshot
4375            .operation_specs
4376            .get_mut(&operation_id)
4377            .expect("persisted spec")
4378            .operation_source = Some(OperationSource::session_child(SessionId::new()));
4379
4380        let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4381            .expect_err("source mirror drift must fail recovery");
4382        assert!(
4383            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4384            "unexpected recovery error: {err:?}"
4385        );
4386    }
4387
4388    #[test]
4389    fn persisted_authority_state_serializes_explicit_no_operation_source() {
4390        let registry = RuntimeOpsLifecycleRegistry::new();
4391
4392        let spec = background_spec("explicit-no-source");
4393        let operation_id = spec.id.clone();
4394        registry.register_operation(spec).unwrap();
4395        registry.provisioning_succeeded(&operation_id).unwrap();
4396
4397        let cursor_state = meerkat_core::EpochCursorState::new();
4398        let snapshot = registry
4399            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4400            .unwrap();
4401        let value = serde_json::to_value(&snapshot).unwrap();
4402        let operations = value
4403            .get("authority_state")
4404            .and_then(|state| state.get("operations"))
4405            .and_then(serde_json::Value::as_object)
4406            .expect("serialized authority operations");
4407        let persisted_state = operations
4408            .values()
4409            .next()
4410            .and_then(serde_json::Value::as_object)
4411            .expect("serialized operation state");
4412
4413        assert!(
4414            persisted_state
4415                .get("operation_source")
4416                .is_some_and(serde_json::Value::is_null),
4417            "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4418        );
4419
4420        let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4421        assert_eq!(
4422            recovered_snapshot
4423                .authority_state
4424                .operations
4425                .get(&operation_id)
4426                .expect("round-tripped operation")
4427                .operation_source,
4428            None
4429        );
4430    }
4431
4432    #[test]
4433    fn persisted_authority_state_rejects_missing_operation_source_fact() {
4434        let registry = RuntimeOpsLifecycleRegistry::new();
4435
4436        let spec = background_spec("missing-source-fact");
4437        registry.register_operation(spec).unwrap();
4438
4439        let cursor_state = meerkat_core::EpochCursorState::new();
4440        let snapshot = registry
4441            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4442            .unwrap();
4443        let mut value = serde_json::to_value(&snapshot).unwrap();
4444        let operations = value
4445            .get_mut("authority_state")
4446            .and_then(|state| state.get_mut("operations"))
4447            .and_then(serde_json::Value::as_object_mut)
4448            .expect("serialized authority operations");
4449        let operation_state = operations
4450            .values_mut()
4451            .next()
4452            .and_then(serde_json::Value::as_object_mut)
4453            .expect("serialized operation state");
4454        assert!(operation_state.remove("operation_source").is_some());
4455
4456        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4457            .expect_err("missing generated source fact must fail recovery snapshot decoding");
4458        assert!(
4459            err.to_string().contains("operation_source"),
4460            "unexpected decode error: {err}"
4461        );
4462    }
4463
4464    #[test]
4465    fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4466        let registry = RuntimeOpsLifecycleRegistry::new();
4467
4468        let spec = background_spec("missing-feed-authority");
4469        let operation_id = spec.id.clone();
4470        registry.register_operation(spec).unwrap();
4471        registry.provisioning_succeeded(&operation_id).unwrap();
4472        registry
4473            .complete_operation(
4474                &operation_id,
4475                OperationResult {
4476                    id: operation_id.clone(),
4477                    content: "done".into(),
4478                    is_error: false,
4479                    duration_ms: 1,
4480                    tokens_used: 0,
4481                },
4482            )
4483            .unwrap();
4484
4485        let cursor_state = meerkat_core::EpochCursorState::new();
4486        let snapshot = registry
4487            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4488            .unwrap();
4489        let mut value = serde_json::to_value(&snapshot).unwrap();
4490        let authority_state = value
4491            .get_mut("authority_state")
4492            .and_then(serde_json::Value::as_object_mut)
4493            .expect("serialized authority state");
4494        assert!(authority_state.remove("completion_feed_entries").is_some());
4495
4496        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4497            .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4498        assert!(
4499            err.to_string().contains("completion_feed_entries"),
4500            "unexpected decode error: {err}"
4501        );
4502    }
4503
4504    #[test]
4505    fn public_child_session_projection_uses_authority_operation_source() {
4506        let registry = RuntimeOpsLifecycleRegistry::new();
4507        let authority_child_session_id = SessionId::new();
4508        let stale_shell_child_session_id = SessionId::new();
4509        let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4510        let spec = OperationSpec {
4511            id: OperationId::new(),
4512            kind: OperationKind::MobMemberChild,
4513            owner_session_id: SessionId::new(),
4514            display_name: "child-projection".into(),
4515            source_label: "test".into(),
4516            operation_source: Some(operation_source),
4517            child_session_id: Some(stale_shell_child_session_id),
4518            expect_peer_channel: true,
4519        };
4520        let operation_id = spec.id.clone();
4521
4522        registry.register_operation(spec).unwrap();
4523
4524        assert_eq!(
4525            registry
4526                .snapshot(&operation_id)
4527                .unwrap()
4528                .unwrap()
4529                .child_session_id,
4530            Some(authority_child_session_id.clone())
4531        );
4532
4533        let cursor_state = meerkat_core::EpochCursorState::new();
4534        let snapshot = registry
4535            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4536            .unwrap();
4537        assert_eq!(
4538            snapshot
4539                .operation_specs
4540                .get(&operation_id)
4541                .expect("persisted spec")
4542                .child_session_id,
4543            Some(authority_child_session_id)
4544        );
4545    }
4546
4547    #[test]
4548    fn generated_terminal_payload_projection_fails_closed() {
4549        let registry = RuntimeOpsLifecycleRegistry::new();
4550
4551        let spec = background_spec("terminal-payload-drift");
4552        let operation_id = spec.id.clone();
4553        registry.register_operation(spec).unwrap();
4554        registry.provisioning_succeeded(&operation_id).unwrap();
4555        registry
4556            .complete_operation(
4557                &operation_id,
4558                OperationResult {
4559                    id: operation_id.clone(),
4560                    content: "done".into(),
4561                    is_error: false,
4562                    duration_ms: 1,
4563                    tokens_used: 0,
4564                },
4565            )
4566            .unwrap();
4567
4568        {
4569            let mut state = registry.write_state().unwrap();
4570            let mut machine_state = state.dsl.0.state().clone();
4571            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4572            machine_state
4573                .op_terminal_payload
4574                .insert(operation_id_key, OperationTerminalOutcome::Retired);
4575            state.dsl = DslAuthority(Box::new(
4576                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4577            ));
4578        }
4579
4580        let err = match registry.register_watcher(&operation_id) {
4581            Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4582            Err(err) => err,
4583        };
4584        assert!(
4585            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4586            "unexpected watcher error: {err:?}"
4587        );
4588        let err = registry
4589            .snapshot(&operation_id)
4590            .expect_err("invalid generated terminal payload must reject public snapshot");
4591        assert!(
4592            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
4593            "unexpected public snapshot error: {err:?}"
4594        );
4595
4596        let cursor_state = meerkat_core::EpochCursorState::new();
4597        let err = match registry
4598            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4599        {
4600            Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
4601            Err(err) => err,
4602        };
4603        assert!(
4604            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4605            "unexpected snapshot error: {err:?}"
4606        );
4607
4608        let err = match registry.collect_completed() {
4609            Ok(_) => panic!("invalid generated terminal payload must reject collection"),
4610            Err(err) => err,
4611        };
4612        assert!(
4613            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4614            "unexpected collection error: {err:?}"
4615        );
4616    }
4617
4618    #[test]
4619    fn generated_terminal_payload_missing_projection_fails_closed() {
4620        let registry = RuntimeOpsLifecycleRegistry::new();
4621
4622        let spec = background_spec("terminal-payload-missing");
4623        let operation_id = spec.id.clone();
4624        registry.register_operation(spec).unwrap();
4625        registry.provisioning_succeeded(&operation_id).unwrap();
4626        registry
4627            .fail_operation(&operation_id, "boom".into())
4628            .unwrap();
4629
4630        {
4631            let mut state = registry.write_state().unwrap();
4632            let mut machine_state = state.dsl.0.state().clone();
4633            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4634            machine_state.op_terminal_payload.remove(&operation_id_key);
4635            state.dsl = DslAuthority(Box::new(
4636                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4637            ));
4638        }
4639
4640        let err = match registry.register_watcher(&operation_id) {
4641            Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
4642            Err(err) => err,
4643        };
4644        assert!(
4645            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4646            "unexpected watcher error: {err:?}"
4647        );
4648        let err = registry
4649            .snapshot(&operation_id)
4650            .expect_err("missing generated terminal payload must reject public snapshot");
4651        assert!(
4652            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4653            "unexpected public snapshot error: {err:?}"
4654        );
4655    }
4656
4657    #[test]
4658    fn generated_terminal_status_without_outcome_fails_closed() {
4659        let registry = RuntimeOpsLifecycleRegistry::new();
4660
4661        let spec = background_spec("terminal-outcome-missing");
4662        let operation_id = spec.id.clone();
4663        registry.register_operation(spec).unwrap();
4664        registry.provisioning_succeeded(&operation_id).unwrap();
4665        registry
4666            .fail_operation(&operation_id, "boom".into())
4667            .unwrap();
4668
4669        {
4670            let mut state = registry.write_state().unwrap();
4671            let mut machine_state = state.dsl.0.state().clone();
4672            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4673            machine_state.op_terminal_outcomes.remove(&operation_id_key);
4674            state.dsl = DslAuthority(Box::new(
4675                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4676            ));
4677        }
4678
4679        let err = registry
4680            .snapshot(&operation_id)
4681            .expect_err("terminal status without outcome must reject public snapshot");
4682        assert!(
4683            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4684            "unexpected public snapshot error: {err:?}"
4685        );
4686
4687        let err = match registry.collect_completed() {
4688            Ok(_) => panic!("terminal status without outcome must reject collection"),
4689            Err(err) => err,
4690        };
4691        assert!(
4692            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4693            "unexpected collection error: {err:?}"
4694        );
4695    }
4696
4697    #[test]
4698    fn generated_operation_source_projection_fails_closed() {
4699        let registry = RuntimeOpsLifecycleRegistry::new();
4700        let child_session_id = SessionId::new();
4701        let operation_source = OperationSource::session_child(child_session_id.clone());
4702        let spec = OperationSpec {
4703            id: OperationId::new(),
4704            kind: OperationKind::MobMemberChild,
4705            owner_session_id: SessionId::new(),
4706            display_name: "source-authority-drift".into(),
4707            source_label: "test".into(),
4708            operation_source: Some(operation_source),
4709            child_session_id: Some(child_session_id),
4710            expect_peer_channel: true,
4711        };
4712        let operation_id = spec.id.clone();
4713
4714        registry.register_operation(spec).unwrap();
4715
4716        {
4717            let mut state = registry.write_state().unwrap();
4718            let mut machine_state = state.dsl.0.state().clone();
4719            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4720            machine_state
4721                .op_sources
4722                .get_mut(&operation_id_key)
4723                .expect("generated operation source")
4724                .session_id = None;
4725            state.dsl = DslAuthority(Box::new(
4726                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4727            ));
4728        }
4729
4730        let err = registry
4731            .snapshot(&operation_id)
4732            .expect_err("invalid generated operation source must reject public snapshot");
4733        assert!(
4734            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4735            "unexpected public snapshot error: {err:?}"
4736        );
4737        let err = registry
4738            .list_operations()
4739            .expect_err("invalid generated operation source must reject public operation list");
4740        assert!(
4741            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4742            "unexpected operation list error: {err:?}"
4743        );
4744
4745        let cursor_state = meerkat_core::EpochCursorState::new();
4746        let err = match registry
4747            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4748        {
4749            Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
4750            Err(err) => err,
4751        };
4752        assert!(
4753            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4754            "unexpected snapshot error: {err:?}"
4755        );
4756    }
4757
4758    #[test]
4759    fn generated_operation_id_projection_fails_closed() {
4760        let registry = RuntimeOpsLifecycleRegistry::new();
4761
4762        {
4763            let mut state = registry.write_state().unwrap();
4764            let mut machine_state = state.dsl.0.state().clone();
4765            machine_state.op_statuses.insert(
4766                "not-json-operation-id".into(),
4767                mm_dsl::OperationStatus::Running,
4768            );
4769            state.dsl = DslAuthority(Box::new(
4770                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4771            ));
4772        }
4773
4774        let err = registry
4775            .list_operations()
4776            .expect_err("invalid generated operation id must reject public operation list");
4777        assert!(
4778            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4779            "unexpected operation list error: {err:?}"
4780        );
4781
4782        let cursor_state = meerkat_core::EpochCursorState::new();
4783        let err = match registry
4784            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4785        {
4786            Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
4787            Err(err) => err,
4788        };
4789        assert!(
4790            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4791            "unexpected persistence snapshot error: {err:?}"
4792        );
4793    }
4794
4795    #[test]
4796    fn generated_missing_kind_projection_fails_closed() {
4797        let registry = RuntimeOpsLifecycleRegistry::new();
4798        let spec = background_spec("missing-kind");
4799        let operation_id = spec.id.clone();
4800        registry.register_operation(spec).unwrap();
4801
4802        {
4803            let mut state = registry.write_state().unwrap();
4804            let mut machine_state = state.dsl.0.state().clone();
4805            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4806            machine_state.op_kinds.remove(&operation_id_key);
4807            state.dsl = DslAuthority(Box::new(
4808                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4809            ));
4810        }
4811
4812        let err = registry
4813            .snapshot(&operation_id)
4814            .expect_err("missing generated kind must reject public snapshot");
4815        assert!(
4816            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4817            "unexpected public snapshot error: {err:?}"
4818        );
4819        let err = registry
4820            .list_operations()
4821            .expect_err("missing generated kind must reject public list");
4822        assert!(
4823            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4824            "unexpected public list error: {err:?}"
4825        );
4826
4827        let cursor_state = meerkat_core::EpochCursorState::new();
4828        let err = registry
4829            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4830            .expect_err("missing generated kind must reject persistence snapshot");
4831        assert!(
4832            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4833            "unexpected persistence snapshot error: {err:?}"
4834        );
4835    }
4836
4837    #[test]
4838    fn generated_missing_status_projection_fails_closed() {
4839        let registry = RuntimeOpsLifecycleRegistry::new();
4840        let spec = background_spec("missing-status");
4841        let operation_id = spec.id.clone();
4842        registry.register_operation(spec).unwrap();
4843
4844        {
4845            let mut state = registry.write_state().unwrap();
4846            let mut machine_state = state.dsl.0.state().clone();
4847            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4848            machine_state.op_statuses.remove(&operation_id_key);
4849            state.dsl = DslAuthority(Box::new(
4850                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4851            ));
4852        }
4853
4854        let err = registry
4855            .snapshot(&operation_id)
4856            .expect_err("missing generated status must reject public snapshot");
4857        assert!(
4858            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4859            "unexpected public snapshot error: {err:?}"
4860        );
4861        let err = registry
4862            .list_operations()
4863            .expect_err("missing generated status must reject public list");
4864        assert!(
4865            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4866            "unexpected public list error: {err:?}"
4867        );
4868        let err = registry
4869            .classify_operation_public_result(&operation_id)
4870            .expect_err("missing generated status must reject public-result classification");
4871        assert!(
4872            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4873            "unexpected public-result error: {err:?}"
4874        );
4875    }
4876
4877    #[test]
4878    fn generated_retiring_public_result_remains_running_until_terminal() {
4879        let registry = RuntimeOpsLifecycleRegistry::new();
4880        let spec = background_spec("retiring-public-result");
4881        let operation_id = spec.id.clone();
4882        registry.register_operation(spec).unwrap();
4883        registry.provisioning_succeeded(&operation_id).unwrap();
4884        registry.request_retire(&operation_id).unwrap();
4885
4886        let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
4887        assert_eq!(snapshot.status, OperationStatus::Retiring);
4888        assert!(snapshot.terminal_outcome.is_none());
4889        assert!(!snapshot.terminal);
4890        assert_eq!(
4891            snapshot.public_result_class,
4892            OperationPublicResultClass::Running
4893        );
4894        assert_eq!(
4895            registry
4896                .classify_operation_public_result(&operation_id)
4897                .unwrap(),
4898            OperationPublicResultClass::Running
4899        );
4900    }
4901
4902    #[test]
4903    fn generated_missing_peer_ready_projection_fails_closed() {
4904        let registry = RuntimeOpsLifecycleRegistry::new();
4905        let spec = background_spec("missing-peer-ready");
4906        let operation_id = spec.id.clone();
4907        registry.register_operation(spec).unwrap();
4908
4909        {
4910            let mut state = registry.write_state().unwrap();
4911            let mut machine_state = state.dsl.0.state().clone();
4912            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4913            machine_state.op_peer_ready.remove(&operation_id_key);
4914            state.dsl = DslAuthority(Box::new(
4915                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4916            ));
4917        }
4918
4919        let err = registry
4920            .snapshot(&operation_id)
4921            .expect_err("missing generated peer-ready fact must reject public snapshot");
4922        assert!(
4923            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4924            "unexpected public snapshot error: {err:?}"
4925        );
4926
4927        let cursor_state = meerkat_core::EpochCursorState::new();
4928        let err = registry
4929            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4930            .expect_err("missing generated peer-ready fact must reject persistence snapshot");
4931        assert!(
4932            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4933            "unexpected persistence snapshot error: {err:?}"
4934        );
4935    }
4936
4937    #[test]
4938    fn generated_missing_progress_count_projection_fails_closed() {
4939        let registry = RuntimeOpsLifecycleRegistry::new();
4940        let spec = background_spec("missing-progress-count");
4941        let operation_id = spec.id.clone();
4942        registry.register_operation(spec).unwrap();
4943
4944        {
4945            let mut state = registry.write_state().unwrap();
4946            let mut machine_state = state.dsl.0.state().clone();
4947            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4948            machine_state.op_progress_counts.remove(&operation_id_key);
4949            state.dsl = DslAuthority(Box::new(
4950                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4951            ));
4952        }
4953
4954        let err = registry
4955            .snapshot(&operation_id)
4956            .expect_err("missing generated progress count must reject public snapshot");
4957        assert!(
4958            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4959            "unexpected public snapshot error: {err:?}"
4960        );
4961
4962        let cursor_state = meerkat_core::EpochCursorState::new();
4963        let err = registry
4964            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4965            .expect_err("missing generated progress count must reject persistence snapshot");
4966        assert!(
4967            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4968            "unexpected persistence snapshot error: {err:?}"
4969        );
4970    }
4971
4972    #[test]
4973    fn generated_terminal_sequence_missing_persistence_fails_closed() {
4974        let registry = RuntimeOpsLifecycleRegistry::new();
4975        let spec = background_spec("terminal-sequence-missing");
4976        let operation_id = spec.id.clone();
4977        registry.register_operation(spec).unwrap();
4978        registry.provisioning_succeeded(&operation_id).unwrap();
4979        registry
4980            .complete_operation(
4981                &operation_id,
4982                OperationResult {
4983                    id: operation_id.clone(),
4984                    content: "done".into(),
4985                    is_error: false,
4986                    duration_ms: 1,
4987                    tokens_used: 0,
4988                },
4989            )
4990            .unwrap();
4991
4992        {
4993            let mut state = registry.write_state().unwrap();
4994            let mut machine_state = state.dsl.0.state().clone();
4995            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4996            machine_state.op_completion_seq.remove(&operation_id_key);
4997            state.dsl = DslAuthority(Box::new(
4998                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4999            ));
5000        }
5001
5002        let cursor_state = meerkat_core::EpochCursorState::new();
5003        let err = registry
5004            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5005            .expect_err("missing generated terminal sequence must reject persistence snapshot");
5006        assert!(
5007            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
5008            "unexpected persistence snapshot error: {err:?}"
5009        );
5010    }
5011
5012    #[test]
5013    fn generated_record_without_shell_projection_fails_closed() {
5014        let registry = RuntimeOpsLifecycleRegistry::new();
5015        let spec = background_spec("missing-shell-record");
5016        let operation_id = spec.id.clone();
5017        registry.register_operation(spec).unwrap();
5018
5019        {
5020            let mut state = registry.write_state().unwrap();
5021            state.records.remove(&operation_id);
5022        }
5023
5024        let err = registry
5025            .snapshot(&operation_id)
5026            .expect_err("generated operation without shell record must reject public snapshot");
5027        assert!(
5028            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5029            "unexpected public snapshot error: {err:?}"
5030        );
5031        let err = registry
5032            .list_operations()
5033            .expect_err("generated operation without shell record must reject public list");
5034        assert!(
5035            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5036            "unexpected public list error: {err:?}"
5037        );
5038    }
5039
5040    #[test]
5041    fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
5042        let registry = RuntimeOpsLifecycleRegistry::new();
5043        let child_session_id = SessionId::new();
5044        let operation_source = OperationSource::session_child(child_session_id.clone());
5045        let spec = OperationSpec {
5046            id: OperationId::new(),
5047            kind: OperationKind::MobMemberChild,
5048            owner_session_id: SessionId::new(),
5049            display_name: "child-drift".into(),
5050            source_label: "test".into(),
5051            operation_source: Some(operation_source),
5052            child_session_id: Some(child_session_id.clone()),
5053            expect_peer_channel: true,
5054        };
5055        let operation_id = spec.id.clone();
5056
5057        registry.register_operation(spec).unwrap();
5058        registry.provisioning_succeeded(&operation_id).unwrap();
5059        registry.mark_retired(&operation_id).unwrap();
5060
5061        let cursor_state = meerkat_core::EpochCursorState::new();
5062        let mut snapshot = registry
5063            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5064            .unwrap();
5065        snapshot
5066            .operation_specs
5067            .get_mut(&operation_id)
5068            .expect("persisted spec")
5069            .child_session_id = Some(SessionId::new());
5070
5071        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5072        assert_eq!(
5073            recovered
5074                .snapshot(&operation_id)
5075                .unwrap()
5076                .unwrap()
5077                .child_session_id,
5078            Some(child_session_id)
5079        );
5080    }
5081
5082    #[test]
5083    fn completion_wake_class_is_generated_by_operation_kind() {
5084        let registry = RuntimeOpsLifecycleRegistry::new();
5085        let operation_id = OperationId::new();
5086
5087        assert_eq!(
5088            registry
5089                .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
5090                .unwrap(),
5091            OperationCompletionWakeClass::Wake
5092        );
5093        assert_eq!(
5094            registry
5095                .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
5096                .unwrap(),
5097            OperationCompletionWakeClass::Ignore
5098        );
5099        assert_eq!(
5100            registry
5101                .classify_operation_completion_wake(
5102                    &operation_id,
5103                    OperationKind::BackgroundToolCapacitySlot,
5104                )
5105                .unwrap(),
5106            OperationCompletionWakeClass::Ignore
5107        );
5108    }
5109
5110    #[test]
5111    fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
5112        let registry = RuntimeOpsLifecycleRegistry::new();
5113
5114        let running_spec = background_spec("running");
5115        let running_id = running_spec.id.clone();
5116        registry.register_operation(running_spec.clone()).unwrap();
5117        registry.provisioning_succeeded(&running_id).unwrap();
5118
5119        let cursor_state = meerkat_core::EpochCursorState::new();
5120        let mut snapshot = registry
5121            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5122            .unwrap();
5123        snapshot.completion_entries.push(CompletionEntry {
5124            seq: 1,
5125            operation_id: running_id.clone(),
5126            kind: running_spec.kind,
5127            display_name: running_spec.display_name,
5128            terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5129                id: running_id,
5130                content: "phantom".into(),
5131                is_error: false,
5132                duration_ms: 1,
5133                tokens_used: 0,
5134            }),
5135            completed_at_ms: None,
5136        });
5137
5138        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5139            Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5140            Err(err) => err,
5141        };
5142        assert!(
5143            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5144            "unexpected recovery error: {err:?}"
5145        );
5146    }
5147
5148    #[test]
5149    fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5150        let registry = RuntimeOpsLifecycleRegistry::new();
5151        let phantom_id = OperationId::new();
5152        {
5153            let state = registry.read_state().unwrap();
5154            state.feed_buffer.push(CompletionEntry {
5155                seq: 1,
5156                operation_id: phantom_id.clone(),
5157                kind: OperationKind::BackgroundToolOp,
5158                display_name: "phantom".into(),
5159                terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5160                    id: phantom_id,
5161                    content: "phantom".into(),
5162                    is_error: false,
5163                    duration_ms: 1,
5164                    tokens_used: 0,
5165                }),
5166                completed_at_ms: None,
5167            });
5168        }
5169
5170        let cursor_state = meerkat_core::EpochCursorState::new();
5171        let err = match registry
5172            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5173        {
5174            Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5175            Err(err) => err,
5176        };
5177        assert!(
5178            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5179            "unexpected capture error: {err:?}"
5180        );
5181    }
5182
5183    #[test]
5184    fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5185        let registry = RuntimeOpsLifecycleRegistry::new();
5186
5187        let spec = background_spec("terminal");
5188        let operation_id = spec.id.clone();
5189        registry.register_operation(spec).unwrap();
5190        registry.provisioning_succeeded(&operation_id).unwrap();
5191        registry
5192            .complete_operation(
5193                &operation_id,
5194                OperationResult {
5195                    id: operation_id.clone(),
5196                    content: "done".into(),
5197                    is_error: false,
5198                    duration_ms: 1,
5199                    tokens_used: 0,
5200                },
5201            )
5202            .unwrap();
5203
5204        let cursor_state = meerkat_core::EpochCursorState::new();
5205        let mut snapshot = registry
5206            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5207            .unwrap();
5208        let phantom_id = OperationId::new();
5209        let phantom_result = OperationResult {
5210            id: phantom_id.clone(),
5211            content: "phantom".into(),
5212            is_error: false,
5213            duration_ms: 1,
5214            tokens_used: 0,
5215        };
5216        let phantom_entry = CompletionFeedCanonicalState {
5217            seq: snapshot.authority_state.next_completion_seq + 1,
5218            kind: OperationKind::BackgroundToolOp,
5219            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5220        };
5221        snapshot
5222            .authority_state
5223            .completion_feed_entries
5224            .insert(phantom_id.clone(), phantom_entry);
5225        snapshot.completion_entries.push(CompletionEntry {
5226            seq: snapshot.authority_state.next_completion_seq + 1,
5227            operation_id: phantom_id.clone(),
5228            kind: OperationKind::BackgroundToolOp,
5229            display_name: "phantom".into(),
5230            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5231            completed_at_ms: None,
5232        });
5233
5234        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5235            Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5236            Err(err) => err,
5237        };
5238        assert!(
5239            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5240            "unexpected recovery error: {err:?}"
5241        );
5242    }
5243
5244    #[test]
5245    fn recovered_completed_order_uses_generated_completion_sequences() {
5246        let registry = RuntimeOpsLifecycleRegistry::new();
5247
5248        let spec_a = background_spec("a");
5249        let id_a = spec_a.id.clone();
5250        registry.register_operation(spec_a).unwrap();
5251        registry.provisioning_succeeded(&id_a).unwrap();
5252        registry
5253            .complete_operation(
5254                &id_a,
5255                OperationResult {
5256                    id: id_a.clone(),
5257                    content: "a".into(),
5258                    is_error: false,
5259                    duration_ms: 1,
5260                    tokens_used: 0,
5261                },
5262            )
5263            .unwrap();
5264
5265        let spec_b = background_spec("b");
5266        let id_b = spec_b.id.clone();
5267        registry.register_operation(spec_b).unwrap();
5268        registry.provisioning_succeeded(&id_b).unwrap();
5269        registry
5270            .complete_operation(
5271                &id_b,
5272                OperationResult {
5273                    id: id_b.clone(),
5274                    content: "b".into(),
5275                    is_error: false,
5276                    duration_ms: 1,
5277                    tokens_used: 0,
5278                },
5279            )
5280            .unwrap();
5281
5282        let cursor_state = meerkat_core::EpochCursorState::new();
5283        let mut snapshot = registry
5284            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5285            .unwrap();
5286        snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5287
5288        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5289        let collected = recovered.collect_completed().unwrap();
5290
5291        assert_eq!(collected[0].0, id_a);
5292        assert_eq!(collected[1].0, id_b);
5293    }
5294
5295    #[test]
5296    fn max_concurrent_enforcement() {
5297        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5298            max_completed: DEFAULT_MAX_COMPLETED,
5299            max_concurrent: Some(2),
5300        });
5301
5302        let spec_a = background_spec("a");
5303        let id_a = spec_a.id.clone();
5304        registry.register_operation(spec_a).unwrap();
5305
5306        let spec_b = background_spec("b");
5307        registry.register_operation(spec_b).unwrap();
5308
5309        let spec_c = background_spec("c");
5310        let result = registry.register_operation(spec_c);
5311        assert!(matches!(
5312            result,
5313            Err(OpsLifecycleError::MaxConcurrentExceeded {
5314                limit: 2,
5315                active: 2,
5316            })
5317        ));
5318
5319        registry.provisioning_succeeded(&id_a).unwrap();
5320        registry
5321            .complete_operation(
5322                &id_a,
5323                OperationResult {
5324                    id: id_a.clone(),
5325                    content: "done".into(),
5326                    is_error: false,
5327                    duration_ms: 1,
5328                    tokens_used: 0,
5329                },
5330            )
5331            .unwrap();
5332
5333        let spec_d = background_spec("d");
5334        assert!(registry.register_operation(spec_d).is_ok());
5335    }
5336
5337    #[test]
5338    fn snapshot_includes_timestamps() {
5339        let registry = RuntimeOpsLifecycleRegistry::new();
5340        let spec = background_spec("timed");
5341        let op_id = spec.id.clone();
5342        registry.register_operation(spec).unwrap();
5343
5344        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5345        assert!(snap1.created_at_ms > 0);
5346        assert!(snap1.started_at_ms.is_none());
5347        assert!(snap1.completed_at_ms.is_none());
5348        assert!(snap1.elapsed_ms.is_none());
5349
5350        registry.provisioning_succeeded(&op_id).unwrap();
5351        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5352        assert!(snap2.started_at_ms.is_some());
5353        assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5354
5355        registry
5356            .complete_operation(
5357                &op_id,
5358                OperationResult {
5359                    id: op_id.clone(),
5360                    content: "done".into(),
5361                    is_error: false,
5362                    duration_ms: 1,
5363                    tokens_used: 0,
5364                },
5365            )
5366            .unwrap();
5367        let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5368        assert!(snap3.completed_at_ms.is_some());
5369        assert!(snap3.elapsed_ms.is_some());
5370        assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5371    }
5372
5373    #[test]
5374    fn snapshot_includes_peer_handle() {
5375        let registry = RuntimeOpsLifecycleRegistry::new();
5376        let child_session_id = SessionId::new();
5377        let spec = OperationSpec {
5378            id: OperationId::new(),
5379            kind: OperationKind::MobMemberChild,
5380            owner_session_id: SessionId::new(),
5381            display_name: "peer-test".into(),
5382            source_label: "test".into(),
5383            operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5384            child_session_id: Some(child_session_id),
5385            expect_peer_channel: true,
5386        };
5387        let op_id = spec.id.clone();
5388        registry.register_operation(spec).unwrap();
5389        registry.provisioning_succeeded(&op_id).unwrap();
5390
5391        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5392        assert!(snap1.peer_handle.is_none());
5393
5394        let handle = OperationPeerHandle {
5395            peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5396            trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5397                "member-x",
5398                PeerId::new(),
5399                "inproc://x",
5400            )
5401            .unwrap(),
5402        };
5403        registry.peer_ready(&op_id, handle).unwrap();
5404
5405        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5406        assert_eq!(
5407            snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5408            "member-x"
5409        );
5410    }
5411}