Skip to main content

meerkat_runtime/
ops_lifecycle.rs

1//! In-memory runtime implementation of the shared async-operation lifecycle seam.
2//!
3//! Per-operation canonical lifecycle state lives in the MeerkatMachine DSL
4//! authority (`op_statuses`, `op_terminal_outcomes`, `op_kinds`,
5//! `op_sources`, `op_peer_ready`, `op_progress_counts`, `active_op_count`,
6//! `wait_active`, `wait_operation_ids`). This shell layer owns pure mechanics: watcher
7//! channels, timestamps, peer handles, snapshot assembly, FIFO eviction
8//! bookkeeping, the completion feed buffer, and typed delivery of generated
9//! admission/rejection feedback.
10//!
11//! Per-transition legality ("is `CompleteOp` legal on a `Provisioning` op?")
12//! is NOT owned by the shell — it lives in the DSL's `from_status_valid`
13//! guards on each op-lifecycle transition. The shell's only job on a
14//! `GuardRejected` rejection is to ask the generated rejection resolver for
15//! the public result class.
16
17use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24    CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31    CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32    OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33    OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34    OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35    OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36    WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43// ---------------------------------------------------------------------------
44// Serde-only persisted canonical state shells
45// ---------------------------------------------------------------------------
46//
47// These structures preserve the persisted fact shape of `PersistedOpsSnapshot`.
48// They are pure serde shells — no methods beyond read-only field accessors,
49// no authority behavior. Optional generated facts that can be explicitly
50// absent still serialize as present `null` so recovery can distinguish
51// generated "none" from a missing persisted fact.
52
53fn deserialize_required_operation_source<'de, D>(
54    deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57    D: serde::Deserializer<'de>,
58{
59    <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62/// Canonical per-operation state as captured in persisted snapshots.
63#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65    status: OperationStatus,
66    kind: OperationKind,
67    #[serde(deserialize_with = "deserialize_required_operation_source")]
68    operation_source: Option<OperationSource>,
69    peer_ready: bool,
70    progress_count: u32,
71    watcher_count: u32,
72    terminal_outcome: Option<OperationTerminalOutcome>,
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    completion_sequence: Option<CompletionSeq>,
75    terminal_buffered: bool,
76}
77
78/// Generated-owned public completion feed fact captured from DSL authority.
79#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81    seq: CompletionSeq,
82    kind: OperationKind,
83    terminal_outcome: OperationTerminalOutcome,
84}
85
86/// Canonical registry-level state as captured in persisted snapshots.
87#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89    operations: HashMap<OperationId, OperationCanonicalState>,
90    completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91    completed_order: VecDeque<OperationId>,
92    max_completed: usize,
93    max_concurrent: Option<usize>,
94    active_count: usize,
95    wait_request_id: Option<WaitRequestId>,
96    wait_operation_ids: Vec<OperationId>,
97    next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101    /// Maximum completed operations retained at capture time.
102    pub fn max_completed(&self) -> usize {
103        self.max_completed
104    }
105
106    /// Maximum concurrent non-terminal operations at capture time.
107    pub fn max_concurrent(&self) -> Option<usize> {
108        self.max_concurrent
109    }
110
111    /// Number of operations captured in the snapshot.
112    pub fn operation_count(&self) -> usize {
113        self.operations.len()
114    }
115
116    /// Number of generated-owned public completion feed entries captured.
117    pub fn completion_feed_count(&self) -> usize {
118        self.completion_feed_entries.len()
119    }
120}
121
122// ---------------------------------------------------------------------------
123// Serializable snapshot for persistence
124// ---------------------------------------------------------------------------
125
126/// Serializable snapshot of the ops lifecycle registry state.
127///
128/// Captured on terminal transitions for durable persistence. Contains
129/// canonical state, operation specs, persisted completion feed entries, and
130/// consumer cursor values. Wire format preserved verbatim from legacy
131/// runtime versions for backward compatibility.
132#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134    /// Epoch identity at capture time.
135    pub epoch_id: meerkat_core::RuntimeEpochId,
136    /// Canonical machine-owned state at capture time.
137    pub authority_state: RegistryCanonicalState,
138    /// Per-operation specs for shell record reconstruction.
139    pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140    /// Persisted completion feed projection metadata. Canonical feed truth is
141    /// captured in `authority_state.completion_feed_entries`.
142    pub completion_entries: Vec<CompletionEntry>,
143    /// Consumer cursor snapshot at capture time.
144    pub cursors: meerkat_core::EpochCursorSnapshot,
145}
146
147#[derive(Debug)]
148pub struct OpsLifecyclePersistenceRequest {
149    snapshot: PersistedOpsSnapshot,
150    result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
151}
152
153impl OpsLifecyclePersistenceRequest {
154    pub fn snapshot(&self) -> &PersistedOpsSnapshot {
155        &self.snapshot
156    }
157
158    pub fn complete(self, result: Result<(), OpsLifecycleError>) {
159        let _ = self.result_tx.send(result);
160    }
161}
162
163// ---------------------------------------------------------------------------
164// Concrete completion feed buffer
165// ---------------------------------------------------------------------------
166
167/// Shared inner state of the completion feed buffer.
168///
169/// Protected by the registry's `RwLock<ShellState>` for writes, and by its
170/// own `RwLock` for reads by external consumers (agent boundary, idle wake).
171#[derive(Debug)]
172struct FeedBufferInner {
173    entries: VecDeque<CompletionEntry>,
174    watermark: CompletionSeq,
175    max_retained: usize,
176}
177
178/// Shared completion feed buffer owned by the runtime registry.
179///
180/// The registry writes entries under its own write lock. External consumers
181/// read through the [`RuntimeCompletionFeed`] handle.
182#[derive(Debug)]
183struct FeedBuffer {
184    inner: RwLock<FeedBufferInner>,
185    /// Atomic mirror of watermark for lock-free `watermark()` reads.
186    watermark_atomic: AtomicU64,
187    /// Notifies all waiters when new entries are appended.
188    notify: tokio::sync::Notify,
189}
190
191impl FeedBuffer {
192    fn new(max_retained: usize) -> Self {
193        Self {
194            inner: RwLock::new(FeedBufferInner {
195                entries: VecDeque::new(),
196                watermark: 0,
197                max_retained,
198            }),
199            watermark_atomic: AtomicU64::new(0),
200            notify: tokio::sync::Notify::new(),
201        }
202    }
203
204    fn push(&self, entry: CompletionEntry) {
205        let mut inner = self
206            .inner
207            .write()
208            .unwrap_or_else(std::sync::PoisonError::into_inner);
209        let seq = entry.seq;
210        inner.entries.push_back(entry);
211        inner.watermark = seq;
212
213        // Evict oldest if over capacity.
214        while inner.entries.len() > inner.max_retained {
215            inner.entries.pop_front();
216        }
217
218        drop(inner);
219
220        self.watermark_atomic.store(seq, Ordering::Release);
221        self.notify.notify_waiters();
222    }
223}
224
225/// Read-only handle to the runtime completion feed.
226///
227/// Implements [`CompletionFeed`] for external consumers. Obtained via
228/// [`RuntimeOpsLifecycleRegistry::completion_feed()`].
229#[derive(Debug, Clone)]
230pub struct RuntimeCompletionFeed {
231    buffer: Arc<FeedBuffer>,
232}
233
234impl CompletionFeed for RuntimeCompletionFeed {
235    fn watermark(&self) -> CompletionSeq {
236        self.buffer.watermark_atomic.load(Ordering::Acquire)
237    }
238
239    fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
240        let inner = self
241            .buffer
242            .inner
243            .read()
244            .unwrap_or_else(std::sync::PoisonError::into_inner);
245        let entries: Vec<CompletionEntry> = inner
246            .entries
247            .iter()
248            .filter(|e| e.seq > after_seq)
249            .cloned()
250            .collect();
251        let watermark = inner.watermark;
252        CompletionBatch { entries, watermark }
253    }
254
255    fn wait_for_advance(
256        &self,
257        after_seq: CompletionSeq,
258    ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
259        Box::pin(async move {
260            loop {
261                // Register the waiter BEFORE reading the watermark.
262                // notify_waiters() in push() only wakes already-registered
263                // listeners — if we read first and push() lands between the
264                // read and notified().await, the wake is lost.
265                let notified = self.buffer.notify.notified();
266                let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
267                if current > after_seq {
268                    return current;
269                }
270                notified.await;
271            }
272        })
273    }
274}
275
276// ---------------------------------------------------------------------------
277// Shell-only per-operation record (not part of canonical machine state)
278// ---------------------------------------------------------------------------
279
280#[derive(Debug)]
281struct OperationCompletionNotifier {
282    tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
283}
284
285impl OperationCompletionNotifier {
286    fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
287        Self { tx }
288    }
289
290    fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
291        let _ = self.tx.send(outcome.clone());
292    }
293}
294
295fn operation_completion_watch_from_receiver(
296    rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
297) -> OperationCompletionWatch {
298    Box::pin(async move {
299        rx.await
300            .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
301    })
302}
303
304fn resolved_operation_completion_watch(
305    outcome: OperationTerminalOutcome,
306) -> OperationCompletionWatch {
307    Box::pin(async move { Ok(outcome) })
308}
309
310/// Shell-owned data for a single operation. Canonical lifecycle state lives in
311/// the DSL authority; this struct holds I/O concerns that the DSL has no
312/// knowledge of.
313#[derive(Debug)]
314struct ShellRecord {
315    spec: OperationSpec,
316    peer_handle: Option<OperationPeerHandle>,
317    /// Private waiter plumbing. Notifiers are drained only from
318    /// `finalize_terminal()` after the generated authority has accepted and
319    /// stored a terminal outcome.
320    watchers: Vec<OperationCompletionNotifier>,
321    // Monotonic timestamps for elapsed computation
322    created_at: Instant,
323    started_at: Option<Instant>,
324    completed_at: Option<Instant>,
325    // Wall-clock anchor captured at creation for epoch millis
326    created_at_wall: SystemTime,
327}
328
329#[derive(Debug)]
330struct PendingWaitState {
331    wait_request_id: WaitRequestId,
332    sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
333}
334
335enum WaitAllAuthorityPlan {
336    AlreadySatisfied(WaitAllSatisfied),
337    ActivateBarrier,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq)]
341enum RecoveredOperationRecordDisposition {
342    Retain,
343    Discard,
344}
345
346impl ShellRecord {
347    fn new(spec: OperationSpec) -> Self {
348        Self {
349            spec,
350            peer_handle: None,
351            watchers: Vec::new(),
352            created_at: Instant::now(),
353            started_at: None,
354            completed_at: None,
355            created_at_wall: SystemTime::now(),
356        }
357    }
358
359    fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
360        wall_anchor
361            .duration_since(UNIX_EPOCH)
362            .map(|d| d.as_millis() as u64)
363            .unwrap_or(0)
364    }
365
366    fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
367        // Compute wall time for a given instant using the wall-clock anchor:
368        // wall_time = created_at_wall + (instant - created_at)
369        let offset = instant.saturating_duration_since(self.created_at);
370        let wall = self.created_at_wall + offset;
371        Self::epoch_millis(&wall)
372    }
373
374    /// Notify all watchers with the given terminal outcome and drain the list.
375    fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
376        for watcher in std::mem::take(&mut self.watchers) {
377            watcher.notify_after_generated_terminal(outcome);
378        }
379    }
380
381    /// Mark the completion timestamp.
382    fn mark_completed(&mut self) {
383        self.completed_at = Some(Instant::now());
384    }
385}
386
387// ---------------------------------------------------------------------------
388// Combined shell state: DSL authority + shell records
389// ---------------------------------------------------------------------------
390
391#[derive(Debug)]
392struct ShellState {
393    /// DSL authority — sole source of truth for per-op canonical state.
394    dsl: DslAuthority,
395    /// Shell-owned per-operation records (specs, watchers, timestamps, peer handles).
396    records: HashMap<OperationId, ShellRecord>,
397    /// Pending wait-all coordination (oneshot channel).
398    pending_wait: Option<PendingWaitState>,
399    /// FIFO ordering of completed operation IDs for bounded eviction.
400    completed_order: VecDeque<OperationId>,
401    /// Maximum completed operations to retain.
402    max_completed: usize,
403    /// Maximum concurrent non-terminal operations (None = unlimited).
404    max_concurrent: Option<usize>,
405    /// Oneshot correlation id for the currently-pending `wait_all` future.
406    ///
407    /// Barrier membership (`wait_operation_ids`) and activation (`wait_active`)
408    /// are DSL-owned. This field is pure transport mechanics — the identity
409    /// the oneshot sender is tagged with so `Drop` can correlate cancellation.
410    wait_request_id: Option<WaitRequestId>,
411    /// Shared feed buffer for completion events.
412    feed_buffer: Arc<FeedBuffer>,
413    /// Persistence channel for durable snapshot writes (set via `set_persistence_channel`).
414    persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
415    /// Epoch ID for persistence snapshots.
416    persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
417    /// Shared cursor state for persistence snapshots.
418    persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
419}
420
421/// Wrapper around the DSL authority that provides `Debug` output.
422///
423/// The generated `MeerkatMachineAuthority` does not derive `Debug`, but
424/// `ShellState` requires it. This wrapper delegates to the inner state's
425/// `Debug` impl.
426struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
427
428impl std::fmt::Debug for DslAuthority {
429    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
430        f.debug_struct("DslAuthority")
431            .field("state", self.0.state())
432            .finish()
433    }
434}
435
436/// Create a DSL authority initialized through generated authority. Per-op
437/// transitions guard only on `op_statuses.contains_key(operation_id)`, so the
438/// phase stays in `Idle` permanently (they all `to Idle`).
439fn new_ops_dsl_authority() -> DslAuthority {
440    DslAuthority(Box::new(
441        crate::meerkat_machine::dsl_authority::new_initialized_authority(
442            "ops lifecycle DSL Initialize must be accepted",
443        ),
444    ))
445}
446
447impl ShellState {
448    fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
449        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
450        let dsl = new_ops_dsl_authority();
451        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
452        let feed_capacity = max_completed.saturating_mul(4).max(1024);
453        tracing::info!(
454            feed_capacity,
455            "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
456        );
457        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
458        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
459        Self {
460            dsl,
461            records: HashMap::new(),
462            pending_wait: None,
463            completed_order: VecDeque::new(),
464            max_completed,
465            max_concurrent,
466            wait_request_id: None,
467            // Feed buffer is larger than max_completed to absorb bursts.
468            // Entries are only evicted by buffer capacity, not by consumer cursor,
469            // so the buffer must be large enough that consumers drain before
470            // the oldest entry is evicted.
471            feed_buffer,
472            persist_tx: None,
473            persist_epoch_id: None,
474            persist_cursor_state: None,
475        }
476    }
477
478    /// Apply a DSL input, mapping transition errors into
479    /// [`OpsLifecycleError::Internal`]. Callers that need to distinguish
480    /// guard rejections (legal-transition violations) from internal desync
481    /// should use [`Self::dsl_apply_raw`] and classify the error themselves;
482    /// this helper is for DSL inputs whose preconditions the caller has
483    /// already fully validated (e.g., `RequestWaitAll`, `SatisfyWaitAll`).
484    fn dsl_apply(
485        &mut self,
486        input: mm_dsl::MeerkatMachineInput,
487        context: &str,
488    ) -> Result<(), OpsLifecycleError> {
489        self.dsl_apply_raw(input).map_err(|err| {
490            OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
491        })
492    }
493
494    /// Apply a DSL input, returning the raw kernel-level rejection so callers
495    /// can distinguish `GuardRejected` (a legitimate legality violation, e.g.,
496    /// `complete_operation` on a `Provisioning` op) from
497    /// `NoMatchingTransition` (a shell/DSL desync). Op-lifecycle entry points
498    /// feed guard rejections into generated rejection feedback before surfacing
499    /// public result classes.
500    fn dsl_apply_raw(
501        &mut self,
502        input: mm_dsl::MeerkatMachineInput,
503    ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
504        mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
505    }
506
507    fn dsl_apply_with_effects(
508        &mut self,
509        input: mm_dsl::MeerkatMachineInput,
510        context: &str,
511    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
512        let transition =
513            mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
514                OpsLifecycleError::Internal(format!(
515                    "DSL rejected ops transition ({context}): {err:?}"
516                ))
517            })?;
518        Ok(transition.into_effects())
519    }
520
521    /// Fail-closed read of a typed terminal payload entry: the payload IS the
522    /// domain [`OperationTerminalOutcome`] (K8b fold — no JSON codec), but the
523    /// shell still refuses to surface a payload whose variant disagrees with
524    /// the recorded discriminant.
525    fn checked_terminal_payload(
526        kind: mm_dsl::OperationTerminalOutcomeKind,
527        payload: &OperationTerminalOutcome,
528        authority: &str,
529        operation_id: &str,
530    ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
531        if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
532            return Err(OpsLifecycleError::Internal(format!(
533                "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
534            )));
535        }
536        Ok(payload.clone())
537    }
538
539    /// Read the DSL operation status for `id`, or `None` if not registered.
540    fn status(&self, id: &OperationId) -> Option<OperationStatus> {
541        let id_key = mm_dsl::OperationId::from_domain(id).0;
542        self.dsl
543            .0
544            .state()
545            .op_statuses
546            .get(&id_key)
547            .copied()
548            .map(OperationStatus::from)
549    }
550
551    fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
552        self.status(id).ok_or_else(|| {
553            OpsLifecycleError::Internal(format!(
554                "generated op lifecycle authority missing status for {id}"
555            ))
556        })
557    }
558
559    /// Read the DSL operation kind for `id`, or `None` if not registered.
560    fn kind(&self, id: &OperationId) -> Option<OperationKind> {
561        let id_key = mm_dsl::OperationId::from_domain(id).0;
562        self.dsl
563            .0
564            .state()
565            .op_kinds
566            .get(&id_key)
567            .copied()
568            .map(OperationKind::from)
569    }
570
571    fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
572        self.kind(id).ok_or_else(|| {
573            OpsLifecycleError::Internal(format!(
574                "generated op lifecycle authority missing kind for {id}"
575            ))
576        })
577    }
578
579    fn operation_source(
580        &self,
581        id: &OperationId,
582    ) -> Result<Option<OperationSource>, OpsLifecycleError> {
583        let id_key = mm_dsl::OperationId::from_domain(id).0;
584        self.dsl
585            .0
586            .state()
587            .op_sources
588            .get(&id_key)
589            .map(|source| {
590                source.to_domain().map_err(|error| {
591                    OpsLifecycleError::Internal(format!(
592                        "generated operation source authority has invalid source for {id}: {error}"
593                    ))
594                })
595            })
596            .transpose()
597    }
598
599    fn child_session_id_from_operation_source(
600        operation_source: Option<&OperationSource>,
601    ) -> Option<SessionId> {
602        match operation_source {
603            Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
604            Some(OperationSource::BackendPeer { .. }) | None => None,
605        }
606    }
607
608    fn align_spec_child_session_id_to_source(
609        spec: &mut OperationSpec,
610        operation_source: Option<&OperationSource>,
611    ) {
612        spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
613    }
614
615    /// Read the peer-ready flag for `id`.
616    fn peer_ready(&self, id: &OperationId) -> Option<bool> {
617        let id_key = mm_dsl::OperationId::from_domain(id).0;
618        self.dsl.0.state().op_peer_ready.get(&id_key).copied()
619    }
620
621    fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
622        self.peer_ready(id).ok_or_else(|| {
623            OpsLifecycleError::Internal(format!(
624                "generated op peer wiring authority missing peer-ready fact for {id}"
625            ))
626        })
627    }
628
629    /// Read the progress counter for `id`.
630    fn progress_count(&self, id: &OperationId) -> Option<u32> {
631        let id_key = mm_dsl::OperationId::from_domain(id).0;
632        self.dsl
633            .0
634            .state()
635            .op_progress_counts
636            .get(&id_key)
637            .map(|v| (*v).min(u32::MAX as u64) as u32)
638    }
639
640    fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
641        self.progress_count(id).ok_or_else(|| {
642            OpsLifecycleError::Internal(format!(
643                "generated op progress authority missing progress count for {id}"
644            ))
645        })
646    }
647
648    /// Read the terminal outcome for `id` by pairing the DSL's typed
649    /// discriminant with the companion payload JSON. Returns `None` when the
650    /// op has no recorded terminal discriminant.
651    fn terminal_outcome(
652        &self,
653        id: &OperationId,
654    ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
655        let id_key = mm_dsl::OperationId::from_domain(id).0;
656        let state = self.dsl.0.state();
657        let status = self.status(id);
658        let terminal = match status {
659            Some(status) => Self::operation_status_is_terminal(id, status)?,
660            None => false,
661        };
662        let kind = state.op_terminal_outcomes.get(&id_key).copied();
663        let Some(kind) = kind else {
664            if terminal {
665                return Err(OpsLifecycleError::Internal(format!(
666                    "generated op terminal authority missing terminal outcome for {id}"
667                )));
668            }
669            return Ok(None);
670        };
671        if !terminal {
672            return Err(OpsLifecycleError::Internal(format!(
673                "generated op terminal authority has terminal outcome for non-terminal {id}"
674            )));
675        }
676        let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
677            OpsLifecycleError::Internal(format!(
678                "generated op terminal authority missing terminal payload for {id}"
679            ))
680        })?;
681        Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
682            .map(Some)
683    }
684
685    /// Whether the operation is currently tracked in DSL state.
686    fn contains(&self, id: &OperationId) -> bool {
687        let id_key = mm_dsl::OperationId::from_domain(id).0;
688        self.dsl.0.state().op_statuses.contains_key(&id_key)
689    }
690
691    /// Number of non-terminal operations (derived from DSL state).
692    fn active_count(&self) -> usize {
693        self.dsl.0.state().active_op_count as usize
694    }
695
696    /// Number of operations currently tracked (including terminal).
697    fn operation_count(&self) -> usize {
698        self.dsl.0.state().op_statuses.len()
699    }
700
701    /// Iterate over all tracked operation IDs (DSL keys converted to domain).
702    fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
703        let mut ids = BTreeSet::new();
704        let state = self.dsl.0.state();
705        Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
706        Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
707        Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
708        Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
709        Self::collect_operation_id_keys(
710            &mut ids,
711            "op_progress_counts",
712            state.op_progress_counts.keys(),
713        )?;
714        Self::collect_operation_id_keys(
715            &mut ids,
716            "op_terminal_outcomes",
717            state.op_terminal_outcomes.keys(),
718        )?;
719        Self::collect_operation_id_keys(
720            &mut ids,
721            "op_terminal_payload",
722            state.op_terminal_payload.keys(),
723        )?;
724        Self::collect_operation_id_keys(
725            &mut ids,
726            "op_completion_seq",
727            state.op_completion_seq.keys(),
728        )?;
729        ids.extend(self.records.keys().cloned());
730        Ok(ids.into_iter().collect())
731    }
732
733    fn collect_operation_id_keys<'a, I>(
734        ids: &mut BTreeSet<OperationId>,
735        field: &str,
736        keys: I,
737    ) -> Result<(), OpsLifecycleError>
738    where
739        I: IntoIterator<Item = &'a String>,
740    {
741        for key in keys {
742            let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
743                OpsLifecycleError::Internal(format!(
744                    "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
745                ))
746            })?;
747            ids.insert(id);
748        }
749        Ok(())
750    }
751
752    fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
753        let id_key = mm_dsl::OperationId::from_domain(id).0;
754        let state = self.dsl.0.state();
755        state.op_statuses.contains_key(&id_key)
756            || state.op_kinds.contains_key(&id_key)
757            || state.op_sources.contains_key(&id_key)
758            || state.op_peer_ready.contains_key(&id_key)
759            || state.op_progress_counts.contains_key(&id_key)
760            || state.op_terminal_outcomes.contains_key(&id_key)
761            || state.op_terminal_payload.contains_key(&id_key)
762            || state.op_completion_seq.contains_key(&id_key)
763    }
764
765    /// Read the DSL-minted completion sequence for a terminal operation.
766    fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
767        let id_key = mm_dsl::OperationId::from_domain(id).0;
768        self.dsl.0.state().op_completion_seq.get(&id_key).copied()
769    }
770
771    fn completion_feed_authority_entries(
772        &self,
773    ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
774        let state = self.dsl.0.state();
775        let sequence_keys: BTreeSet<String> =
776            state.completion_feed_sequences.keys().cloned().collect();
777        let companion_domains: [(&str, BTreeSet<String>); 3] = [
778            (
779                "completion_feed_kinds",
780                state.completion_feed_kinds.keys().cloned().collect(),
781            ),
782            (
783                "completion_feed_terminal_outcomes",
784                state
785                    .completion_feed_terminal_outcomes
786                    .keys()
787                    .cloned()
788                    .collect(),
789            ),
790            (
791                "completion_feed_terminal_payload",
792                state
793                    .completion_feed_terminal_payload
794                    .keys()
795                    .cloned()
796                    .collect(),
797            ),
798        ];
799        for (field, keys) in companion_domains {
800            if keys != sequence_keys {
801                return Err(OpsLifecycleError::Internal(format!(
802                    "generated completion feed authority has mismatched {field} domain"
803                )));
804            }
805        }
806
807        let mut entries = HashMap::new();
808        for (id_key, seq) in &state.completion_feed_sequences {
809            if !state.completion_sequence_claims.contains(seq) {
810                return Err(OpsLifecycleError::Internal(format!(
811                    "generated completion feed authority sequence {seq} for {id_key} is not claimed"
812                )));
813            }
814            let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
815                OpsLifecycleError::Internal(format!(
816                    "generated completion feed authority used invalid operation id key {id_key}: {error}"
817                ))
818            })?;
819            let kind = state
820                .completion_feed_kinds
821                .get(id_key)
822                .copied()
823                .map(OperationKind::from)
824                .ok_or_else(|| {
825                    OpsLifecycleError::Internal(format!(
826                        "generated completion feed authority missing kind for {id_key}"
827                    ))
828                })?;
829            let outcome_kind = state
830                .completion_feed_terminal_outcomes
831                .get(id_key)
832                .copied()
833                .ok_or_else(|| {
834                    OpsLifecycleError::Internal(format!(
835                        "generated completion feed authority missing terminal outcome for {id_key}"
836                    ))
837                })?;
838            let payload = state
839                .completion_feed_terminal_payload
840                .get(id_key)
841                .ok_or_else(|| {
842                    OpsLifecycleError::Internal(format!(
843                        "generated completion feed authority missing terminal payload for {id_key}"
844                    ))
845                })?;
846            let terminal_outcome = Self::checked_terminal_payload(
847                outcome_kind,
848                payload,
849                "generated completion feed authority",
850                id_key,
851            )?;
852            entries.insert(
853                operation_id,
854                CompletionFeedCanonicalState {
855                    seq: *seq,
856                    kind,
857                    terminal_outcome,
858                },
859            );
860        }
861        Ok(entries)
862    }
863
864    fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
865        let state = self.dsl.0.state();
866        match consumer {
867            CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
868            CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
869            CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
870        }
871    }
872
873    fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
874        meerkat_core::EpochCursorSnapshot {
875            agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
876            runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
877            runtime_last_injected_seq: self
878                .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
879        }
880    }
881
882    /// Build a snapshot from DSL state + shell record.
883    fn snapshot(
884        &self,
885        id: &OperationId,
886    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
887        let Some(shell) = self.records.get(id) else {
888            if self.has_generated_operation_record_fact(id) {
889                return Err(OpsLifecycleError::Internal(format!(
890                    "generated op lifecycle authority has operation facts without shell projection record for {id}"
891                )));
892            }
893            return Ok(None);
894        };
895        let kind = self.require_kind(id)?;
896        let status = self.require_status(id)?;
897        let terminal = Self::operation_status_is_terminal(id, status)?;
898        let public_result_class = Self::operation_public_result_class(id, status)?;
899        let peer_ready = self.require_peer_ready(id)?;
900        let progress_count = self.require_progress_count(id)?;
901        let operation_source = self.operation_source(id)?;
902        let terminal_outcome = self.terminal_outcome(id)?;
903
904        let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
905        let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
906        let completed_at_ms = shell
907            .completed_at
908            .map(|i| shell.epoch_millis_for_instant(i));
909        let elapsed_ms = shell.completed_at.map(|completed| {
910            completed
911                .saturating_duration_since(shell.created_at)
912                .as_millis() as u64
913        });
914
915        Ok(Some(OperationLifecycleSnapshot {
916            id: shell.spec.id.clone(),
917            kind,
918            display_name: shell.spec.display_name.clone(),
919            child_session_id: Self::child_session_id_from_operation_source(
920                operation_source.as_ref(),
921            ),
922            operation_source,
923            status,
924            terminal,
925            public_result_class,
926            peer_ready,
927            progress_count,
928            watcher_count: shell.watchers.len() as u32,
929            terminal_outcome,
930            peer_handle: shell.peer_handle.clone(),
931            created_at_ms,
932            started_at_ms,
933            completed_at_ms,
934            elapsed_ms,
935        }))
936    }
937
938    /// Emit shell-side mechanics for a terminal transition: notify watchers,
939    /// push generated-authorized CompletionEntry rows, retain in FIFO, evict
940    /// as needed. Called AFTER the DSL transition has already persisted the
941    /// terminal status + outcome.
942    fn finalize_terminal(
943        &mut self,
944        id: &OperationId,
945    ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
946        let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
947            OpsLifecycleError::Internal(format!(
948                "generated op terminal transition did not mint terminal outcome for {id}"
949            ))
950        })?;
951        let kind = self.require_kind(id)?;
952
953        // Notify watchers and mark completion timestamp.
954        if let Some(shell) = self.records.get_mut(id) {
955            shell.notify_watchers(&outcome);
956            shell.mark_completed();
957        }
958
959        if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
960        {
961            self.dsl_apply(
962                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
963                    operation_id: mm_dsl::OperationId::from_domain(id).0,
964                },
965                "CollectCompletedOp",
966            )?;
967            self.records.remove(id);
968            self.completed_order.retain(|queued| queued != id);
969            return Ok(None);
970        }
971
972        let mut completion_entry = None;
973        if Self::operation_completion_feed_class(id, kind)?
974            == mm_dsl::OperationCompletionFeedClass::Emit
975        {
976            let feed_authority = self
977                .completion_feed_authority_entries()?
978                .remove(id)
979                .ok_or_else(|| {
980                    OpsLifecycleError::Internal(format!(
981                        "generated op terminal transition did not mint completion feed authority for {id}"
982                    ))
983                })?;
984            if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
985                return Err(OpsLifecycleError::Internal(format!(
986                    "generated completion feed authority drifted from terminal op authority for {id}"
987                )));
988            }
989            let seq = self.completion_sequence(id).ok_or_else(|| {
990                OpsLifecycleError::Internal(format!(
991                    "generated op terminal transition did not mint completion sequence for {id}"
992                ))
993            })?;
994            if feed_authority.seq != seq {
995                return Err(OpsLifecycleError::Internal(format!(
996                    "generated completion feed authority sequence drifted for {id}"
997                )));
998            }
999            let display_name = self
1000                .records
1001                .get(id)
1002                .map(|r| r.spec.display_name.clone())
1003                .unwrap_or_default();
1004            let completed_at_ms = self
1005                .records
1006                .get(id)
1007                .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1008            completion_entry = Some(CompletionEntry {
1009                seq: feed_authority.seq,
1010                operation_id: id.clone(),
1011                kind: feed_authority.kind,
1012                display_name,
1013                terminal_outcome: feed_authority.terminal_outcome,
1014                completed_at_ms,
1015            });
1016        }
1017
1018        // FIFO retention + eviction.
1019        self.completed_order.push_back(id.clone());
1020        while self.completed_order.len() > self.max_completed {
1021            if let Some(evicted) = self.completed_order.pop_front() {
1022                self.dsl_apply(
1023                    mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1024                        operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1025                    },
1026                    "EvictCompletedOp",
1027                )?;
1028                self.records.remove(&evicted);
1029            }
1030        }
1031
1032        // Satisfy a pending wait request if all its ops are now terminal. On
1033        // authority-invariant corruption this propagates the typed fault (and
1034        // drops the barrier oneshot so the waiter resolves to Err) instead of
1035        // reporting the op terminal with a silently-hung barrier.
1036        self.maybe_satisfy_wait()?;
1037        Ok(completion_entry)
1038    }
1039
1040    fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1041        if let Some(entry) = entry {
1042            self.feed_buffer.push(entry);
1043        }
1044    }
1045
1046    /// Read barrier membership from DSL state (sole owner).
1047    fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1048        self.dsl
1049            .0
1050            .state()
1051            .wait_operation_ids
1052            .iter()
1053            .map(|key| {
1054                serde_json::from_str::<OperationId>(key).map_err(|error| {
1055                    OpsLifecycleError::Internal(format!(
1056                        "generated wait operation identity authority used invalid operation id key {key}: {error}"
1057                    ))
1058                })
1059            })
1060            .collect()
1061    }
1062
1063    /// Whether the DSL has a barrier wait active.
1064    #[cfg(test)]
1065    fn wait_active(&self) -> bool {
1066        self.dsl.0.state().wait_active
1067    }
1068
1069    fn wait_all_satisfied_from_effects(
1070        effects: &[mm_dsl::MeerkatMachineEffect],
1071    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1072        let mut satisfied = None;
1073        for effect in effects {
1074            let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1075                wait_request_id,
1076                run_id,
1077                operation_ids,
1078            } = effect
1079            else {
1080                continue;
1081            };
1082            if satisfied.is_some() {
1083                return Err(OpsLifecycleError::Internal(
1084                    "generated wait_all authority emitted multiple satisfaction effects".into(),
1085                ));
1086            }
1087            let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1088                OpsLifecycleError::Internal(format!(
1089                    "generated wait_all authority emitted invalid wait request id '{}': {err}",
1090                    wait_request_id.0
1091                ))
1092            })?;
1093            let mut ids = Vec::with_capacity(operation_ids.len());
1094            for operation_id in operation_ids {
1095                ids.push(
1096                    serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1097                        OpsLifecycleError::Internal(format!(
1098                            "generated wait_all authority emitted invalid operation id '{}': {err}",
1099                            operation_id.0
1100                        ))
1101                    })?,
1102                );
1103            }
1104            satisfied = Some(WaitAllSatisfied {
1105                wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1106                run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1107                    OpsLifecycleError::Internal(format!(
1108                        "generated wait_all authority emitted invalid run id '{}': {err}",
1109                        run_id.0
1110                    ))
1111                })?),
1112                operation_ids: ids,
1113            });
1114        }
1115        Ok(satisfied)
1116    }
1117
1118    fn parse_wait_all_operation_id(
1119        raw: &str,
1120        context: &str,
1121    ) -> Result<OperationId, OpsLifecycleError> {
1122        serde_json::from_str::<OperationId>(raw).map_err(|err| {
1123            OpsLifecycleError::Internal(format!(
1124                "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1125            ))
1126        })
1127    }
1128
1129    fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1130        let mut seen = HashSet::new();
1131        operation_ids
1132            .iter()
1133            .find(|operation_id| !seen.insert((*operation_id).clone()))
1134            .cloned()
1135    }
1136
1137    fn wait_all_admission_error_from_effects(
1138        wait_request_id: &WaitRequestId,
1139        effects: &[mm_dsl::MeerkatMachineEffect],
1140    ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1141        let mut admission = None;
1142        for effect in effects {
1143            let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1144                wait_request_id: resolved_wait_request_id,
1145                result,
1146                reject_reason,
1147                rejected_operation_id,
1148            } = effect
1149            else {
1150                continue;
1151            };
1152            if admission.is_some() {
1153                return Err(OpsLifecycleError::Internal(
1154                    "generated wait_all authority emitted multiple admission results".into(),
1155                ));
1156            }
1157            let resolved_uuid =
1158                uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1159                    OpsLifecycleError::Internal(format!(
1160                        "generated wait_all authority emitted invalid wait request id '{}': {err}",
1161                        resolved_wait_request_id.0
1162                    ))
1163                })?;
1164            let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1165            if &resolved_wait_request_id != wait_request_id {
1166                return Err(OpsLifecycleError::Internal(format!(
1167                    "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1168                )));
1169            }
1170            admission = Some(match result {
1171                mm_dsl::WaitAllAdmissionResultKind::Accept => {
1172                    if reject_reason.is_some() || rejected_operation_id.is_some() {
1173                        return Err(OpsLifecycleError::Internal(
1174                            "generated wait_all authority accepted with rejection payload".into(),
1175                        ));
1176                    }
1177                    None
1178                }
1179                mm_dsl::WaitAllAdmissionResultKind::Reject => {
1180                    let reason = reject_reason.ok_or_else(|| {
1181                        OpsLifecycleError::Internal(
1182                            "generated wait_all authority rejected without reason".into(),
1183                        )
1184                    })?;
1185                    let error = match reason {
1186                        mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1187                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1188                                OpsLifecycleError::Internal(
1189                                    "generated wait_all authority rejected duplicate without operation id"
1190                                        .into(),
1191                                )
1192                            })?;
1193                            OpsLifecycleError::DuplicateWaitOperation(
1194                                Self::parse_wait_all_operation_id(raw, "duplicate")?,
1195                            )
1196                        }
1197                        mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1198                            if rejected_operation_id.is_some() {
1199                                return Err(OpsLifecycleError::Internal(
1200                                    "generated wait_all authority rejected active wait with operation id"
1201                                        .into(),
1202                                ));
1203                            }
1204                            OpsLifecycleError::WaitAlreadyActive
1205                        }
1206                        mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1207                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1208                                OpsLifecycleError::Internal(
1209                                    "generated wait_all authority rejected missing operation without operation id"
1210                                        .into(),
1211                                )
1212                            })?;
1213                            OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1214                                raw, "missing",
1215                            )?)
1216                        }
1217                    };
1218                    Some(error)
1219                }
1220            });
1221        }
1222        admission.ok_or_else(|| {
1223            OpsLifecycleError::Internal(
1224                "generated wait_all authority emitted no admission result".into(),
1225            )
1226        })
1227    }
1228
1229    fn resolve_wait_all_admission(
1230        &mut self,
1231        wait_request_id: &WaitRequestId,
1232        operation_ids: &[OperationId],
1233        dsl_ids: &BTreeSet<String>,
1234        dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1235        operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1236        operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1237    ) -> Result<(), OpsLifecycleError> {
1238        let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1239            .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1240        let not_found = operation_ids
1241            .iter()
1242            .find(|operation_id| !self.contains(operation_id))
1243            .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1244        let dsl_id_sequence: Vec<String> = operation_ids
1245            .iter()
1246            .map(|id| mm_dsl::OperationId::from_domain(id).0)
1247            .collect();
1248        let effects = self.dsl_apply_with_effects(
1249            mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1250                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1251                operation_id_sequence: dsl_id_sequence,
1252                operation_ids: dsl_ids.clone(),
1253                operation_id_tokens: dsl_id_tokens.clone(),
1254                operation_token_by_id: operation_token_by_id.clone(),
1255                operation_id_by_token: operation_id_by_token.clone(),
1256                duplicate_operation_id: duplicate,
1257                not_found_operation_id: not_found,
1258            },
1259            "ResolveWaitAllAdmission",
1260        )?;
1261        if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1262        {
1263            return Err(error);
1264        }
1265        Ok(())
1266    }
1267
1268    fn try_satisfy_wait_all_authority(
1269        &mut self,
1270    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1271        let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1272            return Ok(None);
1273        };
1274        let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1275            return Err(OpsLifecycleError::Internal(
1276                "generated wait_all authority has active wait without run id".into(),
1277            ));
1278        };
1279        let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1280        let transition = match mm_dsl::MeerkatMachineMutator::apply(
1281            &mut *self.dsl.0,
1282            mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1283                wait_request_id: dsl_wait_request_id,
1284                run_id: dsl_run_id,
1285                operation_id_tokens: dsl_operation_id_tokens,
1286            },
1287        ) {
1288            Ok(transition) => transition,
1289            Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1290            Err(err) => {
1291                return Err(OpsLifecycleError::Internal(format!(
1292                    "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1293                )));
1294            }
1295        };
1296        Self::wait_all_satisfied_from_effects(transition.effects())?
1297            .ok_or_else(|| {
1298                OpsLifecycleError::Internal(
1299                    "generated wait_all authority accepted satisfaction without effect".into(),
1300                )
1301            })
1302            .map(Some)
1303    }
1304
1305    fn begin_wait_all_authority(
1306        &mut self,
1307        run_id: &RunId,
1308        wait_request_id: &WaitRequestId,
1309        operation_ids: &[OperationId],
1310    ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1311        let mut dsl_ids = BTreeSet::new();
1312        let mut dsl_id_tokens = BTreeSet::new();
1313        let mut operation_token_by_id = BTreeMap::new();
1314        let mut operation_id_by_token = BTreeMap::new();
1315        for id in operation_ids {
1316            let token = mm_dsl::OperationId::from_domain(id);
1317            let raw_id = token.0.clone();
1318            dsl_ids.insert(raw_id.clone());
1319            dsl_id_tokens.insert(token.clone());
1320            operation_token_by_id.insert(raw_id.clone(), token.clone());
1321            operation_id_by_token.insert(token, raw_id);
1322        }
1323        self.resolve_wait_all_admission(
1324            wait_request_id,
1325            operation_ids,
1326            &dsl_ids,
1327            &dsl_id_tokens,
1328            &operation_token_by_id,
1329            &operation_id_by_token,
1330        )?;
1331        self.dsl_apply(
1332            mm_dsl::MeerkatMachineInput::RequestWaitAll {
1333                run_id: mm_dsl::RunId::from_domain(run_id),
1334                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1335                operation_id_sequence: operation_ids
1336                    .iter()
1337                    .map(|id| mm_dsl::OperationId::from_domain(id).0)
1338                    .collect(),
1339                operation_ids: dsl_ids,
1340                operation_id_tokens: dsl_id_tokens,
1341                operation_token_by_id,
1342                operation_id_by_token,
1343            },
1344            "RequestWaitAll",
1345        )?;
1346        if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1347            return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1348        }
1349        Ok(WaitAllAuthorityPlan::ActivateBarrier)
1350    }
1351
1352    fn owner_termination_targets(
1353        &self,
1354    ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1355        let mut targets = Vec::new();
1356        for id in self.operation_ids()? {
1357            let status = self.require_status(&id)?;
1358            if !Self::operation_status_is_terminal(&id, status)? {
1359                targets.push((id, status));
1360            }
1361        }
1362        Ok(targets)
1363    }
1364
1365    fn operation_status_is_terminal(
1366        operation_id: &OperationId,
1367        status: OperationStatus,
1368    ) -> Result<bool, OpsLifecycleError> {
1369        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1370        let effects = Self::apply_stateless_classifier(
1371            mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1372                operation_id: operation_id_key.clone(),
1373                status: mm_dsl::OperationStatus::from(status),
1374            },
1375            "ClassifyOperationTerminality",
1376        )?;
1377        let mut terminal = None;
1378        for effect in effects {
1379            match effect {
1380                mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1381                    if operation_id == operation_id_key =>
1382                {
1383                    terminal = Some(true);
1384                }
1385                mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1386                    if operation_id == operation_id_key =>
1387                {
1388                    terminal = Some(false);
1389                }
1390                other => {
1391                    return Err(OpsLifecycleError::Internal(format!(
1392                        "unexpected generated operation terminality effect: {other:?}"
1393                    )));
1394                }
1395            }
1396        }
1397        terminal.ok_or_else(|| {
1398            OpsLifecycleError::Internal(format!(
1399                "generated operation terminality authority emitted no effect for {operation_id}"
1400            ))
1401        })
1402    }
1403
1404    fn operation_public_result_class(
1405        operation_id: &OperationId,
1406        status: OperationStatus,
1407    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1408        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1409        let effects = Self::apply_stateless_classifier(
1410            mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1411                operation_id: operation_id_key.clone(),
1412                status: mm_dsl::OperationStatus::from(status),
1413            },
1414            "ClassifyOperationPublicResult",
1415        )?;
1416        let mut result = None;
1417        for effect in effects {
1418            match effect {
1419                mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1420                    operation_id,
1421                    result: classified,
1422                } if operation_id == operation_id_key => {
1423                    result = Some(OperationPublicResultClass::from(classified));
1424                }
1425                other => {
1426                    return Err(OpsLifecycleError::Internal(format!(
1427                        "unexpected generated operation public-result effect: {other:?}"
1428                    )));
1429                }
1430            }
1431        }
1432        result.ok_or_else(|| {
1433            OpsLifecycleError::Internal(format!(
1434                "generated operation public-result authority emitted no effect for {operation_id}"
1435            ))
1436        })
1437    }
1438
1439    fn operation_transition_rejection_is_idempotent(
1440        operation_id: &OperationId,
1441        action: OperationLifecycleAction,
1442        status: OperationStatus,
1443    ) -> Result<bool, OpsLifecycleError> {
1444        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1445        let action = mm_dsl::OpLifecycleActionKind::from(action);
1446        let status = mm_dsl::OperationStatus::from(status);
1447        let effects = Self::apply_stateless_classifier(
1448            mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1449                operation_id: operation_id_key.clone(),
1450                action,
1451                status,
1452            },
1453            "ClassifyOperationTransitionIdempotence",
1454        )?;
1455        let mut idempotent = None;
1456        for effect in effects {
1457            match effect {
1458                mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1459                    operation_id,
1460                    action: effect_action,
1461                    status: effect_status,
1462                } if operation_id == operation_id_key
1463                    && effect_action == action
1464                    && effect_status == status =>
1465                {
1466                    idempotent = Some(true);
1467                }
1468                mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1469                    operation_id,
1470                    action: effect_action,
1471                    status: effect_status,
1472                } if operation_id == operation_id_key
1473                    && effect_action == action
1474                    && effect_status == status =>
1475                {
1476                    idempotent = Some(false);
1477                }
1478                other => {
1479                    return Err(OpsLifecycleError::Internal(format!(
1480                        "unexpected generated operation transition-idempotence effect: {other:?}"
1481                    )));
1482                }
1483            }
1484        }
1485        idempotent.ok_or_else(|| {
1486            OpsLifecycleError::Internal(format!(
1487                "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1488            ))
1489        })
1490    }
1491
1492    fn operation_completion_feed_class(
1493        operation_id: &OperationId,
1494        kind: OperationKind,
1495    ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1496        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1497        let kind = mm_dsl::OperationKind::from(kind);
1498        let effects = Self::apply_stateless_classifier(
1499            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1500                operation_id: operation_id_key.clone(),
1501                kind,
1502            },
1503            "ClassifyOperationCompletionFeed",
1504        )?;
1505        let mut class = None;
1506        for effect in effects {
1507            match effect {
1508                mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1509                    operation_id,
1510                    result,
1511                } if operation_id == operation_id_key => {
1512                    class = Some(result);
1513                }
1514                other => {
1515                    return Err(OpsLifecycleError::Internal(format!(
1516                        "unexpected generated operation completion-feed effect: {other:?}"
1517                    )));
1518                }
1519            }
1520        }
1521        class.ok_or_else(|| {
1522            OpsLifecycleError::Internal(format!(
1523                "generated operation completion-feed authority emitted no effect for {operation_id}"
1524            ))
1525        })
1526    }
1527
1528    fn operation_completion_wake_class(
1529        operation_id: &OperationId,
1530        kind: OperationKind,
1531    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1532        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1533        let kind = mm_dsl::OperationKind::from(kind);
1534        let effects = Self::apply_stateless_classifier(
1535            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1536                operation_id: operation_id_key.clone(),
1537                kind,
1538            },
1539            "ClassifyOperationCompletionWake",
1540        )?;
1541        let mut class = None;
1542        for effect in effects {
1543            match effect {
1544                mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1545                    operation_id,
1546                    result,
1547                } if operation_id == operation_id_key => {
1548                    class = Some(OperationCompletionWakeClass::from(result));
1549                }
1550                other => {
1551                    return Err(OpsLifecycleError::Internal(format!(
1552                        "unexpected generated operation completion-wake effect: {other:?}"
1553                    )));
1554                }
1555            }
1556        }
1557        class.ok_or_else(|| {
1558            OpsLifecycleError::Internal(format!(
1559                "generated operation completion-wake authority emitted no effect for {operation_id}"
1560            ))
1561        })
1562    }
1563
1564    fn operation_durability_class(
1565        operation_id: &OperationId,
1566        kind: OperationKind,
1567    ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1568        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1569        let kind = mm_dsl::OperationKind::from(kind);
1570        let effects = Self::apply_stateless_classifier(
1571            mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1572                operation_id: operation_id_key.clone(),
1573                kind,
1574            },
1575            "ClassifyOperationDurability",
1576        )?;
1577        let mut class = None;
1578        for effect in effects {
1579            match effect {
1580                mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1581                    operation_id,
1582                    result,
1583                } if operation_id == operation_id_key => {
1584                    class = Some(result);
1585                }
1586                other => {
1587                    return Err(OpsLifecycleError::Internal(format!(
1588                        "unexpected generated operation durability effect: {other:?}"
1589                    )));
1590                }
1591            }
1592        }
1593        class.ok_or_else(|| {
1594            OpsLifecycleError::Internal(format!(
1595                "generated operation durability authority emitted no effect for {operation_id}"
1596            ))
1597        })
1598    }
1599
1600    fn recovered_operation_record_disposition(
1601        operation_id: &OperationId,
1602        status: OperationStatus,
1603        kind: OperationKind,
1604        terminal_outcome_present: bool,
1605        terminal_payload_present: bool,
1606        completion_sequence_present: bool,
1607    ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1608        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1609        let effects = Self::apply_stateless_classifier(
1610            mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1611                operation_id: operation_id_key.clone(),
1612                status: mm_dsl::OperationStatus::from(status),
1613                kind: mm_dsl::OperationKind::from(kind),
1614                terminal_outcome_present,
1615                terminal_payload_present,
1616                completion_sequence_present,
1617            },
1618            "ClassifyRecoveredOperationRecord",
1619        )?;
1620        let mut disposition = None;
1621        for effect in effects {
1622            match effect {
1623                mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1624                    if operation_id == operation_id_key =>
1625                {
1626                    disposition = Some(RecoveredOperationRecordDisposition::Retain);
1627                }
1628                mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1629                    if operation_id == operation_id_key =>
1630                {
1631                    disposition = Some(RecoveredOperationRecordDisposition::Discard);
1632                }
1633                other => {
1634                    return Err(OpsLifecycleError::Internal(format!(
1635                        "unexpected generated recovered-operation classification effect: {other:?}"
1636                    )));
1637                }
1638            }
1639        }
1640        disposition.ok_or_else(|| {
1641            OpsLifecycleError::Internal(format!(
1642                "generated recovered-operation classifier emitted no effect for {operation_id}"
1643            ))
1644        })
1645    }
1646
1647    fn apply_stateless_classifier(
1648        input: mm_dsl::MeerkatMachineInput,
1649        label: &'static str,
1650    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1651        let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1652            "ops stateless classifier Initialize must be accepted",
1653        );
1654        let transition =
1655            mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1656                OpsLifecycleError::Internal(format!(
1657                    "DSL rejected ops transition ({label}): {err:?}"
1658                ))
1659            })?;
1660        Ok(transition.into_effects())
1661    }
1662
1663    /// Check whether a pending barrier wait is now satisfied and resolve it.
1664    ///
1665    /// Barrier membership and the "all members terminal" decision both live
1666    /// in the DSL: `wait_operation_ids` carries the set, `wait_active`
1667    /// signals a pending barrier, and `SatisfyWaitAll`'s
1668    /// `all_members_terminal` guard owns the fixed-point test. The shell
1669    /// echoes the DSL-owned request id and typed operation tokens into
1670    /// `SatisfyWaitAll` so the transition can clear the barrier before
1671    /// rendering the handoff effect. It still fires idempotently on every
1672    /// terminal transition and lets the DSL guard reject early firings as a
1673    /// no-op. On acceptance (transition returns `Ok`), the shell selects the
1674    /// correlated oneshot and delivers the `WaitAllSatisfied` obligation
1675    /// token.
1676    ///
1677    /// `wait_request_id` is the shell-owned oneshot correlation id that
1678    /// selects which sender to notify; when the DSL barrier satisfies
1679    /// without a live correlation (post-recovery, or duplicate resolution),
1680    /// the oneshot simply remains pending. That benign no-correlation case is
1681    /// the only path that leaves the oneshot pending: on authority-invariant
1682    /// corruption the pending sender is dropped so the waiter's
1683    /// `WaitAllFuture` resolves to `Err` and the typed
1684    /// [`OpsLifecycleError`] propagates to the terminal transition caller
1685    /// instead of reporting the op complete with a silently-hung barrier.
1686    fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1687        let satisfied = match self.try_satisfy_wait_all_authority() {
1688            Ok(Some(satisfied)) => satisfied,
1689            Ok(None) => return Ok(()),
1690            Err(err) => {
1691                // Authority-invariant corruption: do NOT report the op complete
1692                // with a silently-pending barrier oneshot. Drop the sender so
1693                // the waiter's `WaitAllFuture` resolves to `Err` via its
1694                // `Poll::Ready(Err(_))` arm (the same mechanism
1695                // `cancel_wait_all_internal` uses), then propagate the typed
1696                // fault. The invariant is already broken, so we do not attempt
1697                // a `CancelWaitAll` rollback on the corrupt machine.
1698                if let Some(pending) = self.pending_wait.take() {
1699                    drop(pending.sender);
1700                }
1701                self.wait_request_id = None;
1702                return Err(err);
1703            }
1704        };
1705        let shell_wait_id = self.wait_request_id.take();
1706        if shell_wait_id
1707            .as_ref()
1708            .is_some_and(|id| id != &satisfied.wait_request_id)
1709        {
1710            tracing::error!(
1711                shell_wait_request_id = ?shell_wait_id,
1712                authority_wait_request_id = %satisfied.wait_request_id,
1713                "generated wait_all authority satisfied a different wait request"
1714            );
1715        }
1716        if let Some(pending) = self.pending_wait.take() {
1717            if pending.wait_request_id == satisfied.wait_request_id {
1718                let _ = pending.sender.send(satisfied);
1719            } else if let Some(shell_wait_id) = shell_wait_id {
1720                tracing::error!(
1721                    shell_wait_request_id = %shell_wait_id,
1722                    pending_wait_request_id = %pending.wait_request_id,
1723                    authority_wait_request_id = %satisfied.wait_request_id,
1724                    "generated wait_all authority satisfied without a matching pending waiter"
1725                );
1726            }
1727        }
1728        Ok(())
1729    }
1730
1731    /// Persist a terminal snapshot if a persistence channel is wired.
1732    ///
1733    /// Called after terminal transitions. Captures authority + entries + cursors under the write
1734    /// lock (caller already holds it), submits a persistence request, and waits for the worker's
1735    /// durable-store result. Returning success only means the snapshot write itself succeeded.
1736    fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1737        let (tx, epoch_id, cursor_state) = match (
1738            &self.persist_tx,
1739            &self.persist_epoch_id,
1740            &self.persist_cursor_state,
1741        ) {
1742            (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1743            _ => return Ok(()),
1744        };
1745
1746        let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1747        let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1748        let request = OpsLifecyclePersistenceRequest {
1749            snapshot,
1750            result_tx,
1751        };
1752
1753        tx.send(request).map_err(|_| {
1754            OpsLifecycleError::Internal(
1755                "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1756                    .into(),
1757            )
1758        })?;
1759        result_rx.recv().map_err(|_| {
1760            OpsLifecycleError::Internal(
1761                "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1762                    .into(),
1763            )
1764        })?
1765    }
1766
1767    /// Capture the full persisted snapshot for the current state.
1768    fn capture_snapshot(
1769        &self,
1770        epoch_id: meerkat_core::RuntimeEpochId,
1771        _cursor_state: &meerkat_core::EpochCursorState,
1772    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1773        let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1774        for op_id in self.operation_ids()? {
1775            let status = self.require_status(&op_id)?;
1776            let kind = self.require_kind(&op_id)?;
1777            if Self::operation_durability_class(&op_id, kind)?
1778                != mm_dsl::OperationDurabilityClass::Retain
1779            {
1780                continue;
1781            }
1782            let peer_ready = self.require_peer_ready(&op_id)?;
1783            let progress_count = self.require_progress_count(&op_id)?;
1784            let operation_source = self.operation_source(&op_id)?;
1785            let terminal_outcome = self.terminal_outcome(&op_id)?;
1786            let completion_sequence = self.completion_sequence(&op_id);
1787            if terminal_outcome.is_some() && completion_sequence.is_none() {
1788                return Err(OpsLifecycleError::Internal(format!(
1789                    "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1790                )));
1791            }
1792            if terminal_outcome.is_none() && completion_sequence.is_some() {
1793                return Err(OpsLifecycleError::Internal(format!(
1794                    "generated op terminal authority has completion sequence for non-terminal {op_id}"
1795                )));
1796            }
1797            let terminal_buffered = terminal_outcome.is_some();
1798            let watcher_count = self
1799                .records
1800                .get(&op_id)
1801                .map(|r| r.watchers.len() as u32)
1802                .unwrap_or(0);
1803            operations.insert(
1804                op_id,
1805                OperationCanonicalState {
1806                    status,
1807                    kind,
1808                    operation_source,
1809                    peer_ready,
1810                    progress_count,
1811                    watcher_count,
1812                    terminal_outcome,
1813                    completion_sequence,
1814                    terminal_buffered,
1815                },
1816            );
1817        }
1818        let operation_specs: HashMap<OperationId, OperationSpec> = self
1819            .records
1820            .iter()
1821            .filter(|(id, _)| operations.contains_key(*id))
1822            .map(|(id, record)| {
1823                let mut spec = record.spec.clone();
1824                let operation_source = operations
1825                    .get(id)
1826                    .and_then(|state| state.operation_source.as_ref());
1827                Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1828                (id.clone(), spec)
1829            })
1830            .collect();
1831        let completed_order: VecDeque<OperationId> = self
1832            .completed_order
1833            .iter()
1834            .filter(|id| operations.contains_key(*id))
1835            .cloned()
1836            .collect();
1837        let active_count = operations
1838            .iter()
1839            .filter(|(id, state)| {
1840                matches!(
1841                    Self::operation_status_is_terminal(id, state.status),
1842                    Ok(false)
1843                )
1844            })
1845            .count();
1846        let authority_completion_entries = self.completion_feed_authority_entries()?;
1847        let published_completion_entries_by_id: HashMap<OperationId, CompletionEntry> = {
1848            let inner = self
1849                .feed_buffer
1850                .inner
1851                .read()
1852                .unwrap_or_else(std::sync::PoisonError::into_inner);
1853            let mut entries = HashMap::new();
1854            for entry in &inner.entries {
1855                if !authority_completion_entries.contains_key(&entry.operation_id) {
1856                    return Err(OpsLifecycleError::Internal(format!(
1857                        "public completion feed projection for {} has no generated authority",
1858                        entry.operation_id
1859                    )));
1860                }
1861                if entries
1862                    .insert(entry.operation_id.clone(), entry.clone())
1863                    .is_some()
1864                {
1865                    return Err(OpsLifecycleError::Internal(format!(
1866                        "public completion feed projection for {} appeared more than once",
1867                        entry.operation_id
1868                    )));
1869                }
1870            }
1871            entries
1872        };
1873        let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
1874            .iter()
1875            .map(|(operation_id, authority_entry)| {
1876                if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
1877                    if projection.seq != authority_entry.seq
1878                        || projection.kind != authority_entry.kind
1879                        || projection.terminal_outcome != authority_entry.terminal_outcome
1880                    {
1881                        return Err(OpsLifecycleError::Internal(format!(
1882                            "public completion feed projection for {operation_id} drifted from generated authority"
1883                        )));
1884                    }
1885                    return Ok(projection.clone());
1886                }
1887
1888                let display_name = self
1889                    .records
1890                    .get(operation_id)
1891                    .map(|record| record.spec.display_name.clone())
1892                    .unwrap_or_default();
1893                let completed_at_ms = self.records.get(operation_id).and_then(|record| {
1894                    record
1895                        .completed_at
1896                        .map(|completed_at| record.epoch_millis_for_instant(completed_at))
1897                });
1898
1899                Ok(CompletionEntry {
1900                    seq: authority_entry.seq,
1901                    operation_id: operation_id.clone(),
1902                    kind: authority_entry.kind,
1903                    display_name,
1904                    terminal_outcome: authority_entry.terminal_outcome.clone(),
1905                    completed_at_ms,
1906                })
1907            })
1908            .collect::<Result<_, _>>()?;
1909        completion_entries.sort_by_key(|entry| entry.seq);
1910
1911        let authority_state = RegistryCanonicalState {
1912            operations,
1913            completion_feed_entries: authority_completion_entries,
1914            completed_order,
1915            max_completed: self.max_completed,
1916            max_concurrent: self.max_concurrent,
1917            active_count,
1918            wait_request_id: self.wait_request_id.clone(),
1919            wait_operation_ids: self.wait_operation_ids()?,
1920            next_completion_seq: self.dsl.0.state().next_completion_seq,
1921        };
1922
1923        Ok(PersistedOpsSnapshot {
1924            epoch_id,
1925            authority_state,
1926            operation_specs,
1927            completion_entries,
1928            cursors: self.completion_cursor_snapshot(),
1929        })
1930    }
1931
1932    fn shell_record_mut(
1933        &mut self,
1934        id: &OperationId,
1935    ) -> Result<&mut ShellRecord, OpsLifecycleError> {
1936        self.records
1937            .get_mut(id)
1938            .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
1939    }
1940
1941    fn collect_wait_outcomes(
1942        &self,
1943        operation_ids: &[OperationId],
1944    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
1945        operation_ids
1946            .iter()
1947            .map(|operation_id| {
1948                let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
1949                    OpsLifecycleError::Internal(format!(
1950                        "wait_all completed without terminal outcome for {operation_id}"
1951                    ))
1952                })?;
1953                Ok((operation_id.clone(), outcome))
1954            })
1955            .collect()
1956    }
1957}
1958
1959impl Default for ShellState {
1960    fn default() -> Self {
1961        Self::new(DEFAULT_MAX_COMPLETED, None)
1962    }
1963}
1964
1965// ---------------------------------------------------------------------------
1966// Public configuration & registry
1967// ---------------------------------------------------------------------------
1968
1969/// Configuration for [`RuntimeOpsLifecycleRegistry`].
1970#[derive(Debug, Clone)]
1971pub struct OpsLifecycleConfig {
1972    /// Maximum number of completed operations to retain (default: 256).
1973    pub max_completed: usize,
1974    /// Maximum concurrent non-terminal operations (None = unlimited).
1975    pub max_concurrent: Option<usize>,
1976}
1977
1978impl Default for OpsLifecycleConfig {
1979    fn default() -> Self {
1980        Self {
1981            max_completed: DEFAULT_MAX_COMPLETED,
1982            max_concurrent: None,
1983        }
1984    }
1985}
1986
1987/// Per-runtime shared registry for async operation lifecycle truth.
1988///
1989/// Per-operation canonical lifecycle state is owned by the DSL authority
1990/// embedded in the shell. This struct manages I/O concerns: watcher
1991/// channels, timestamps, peer handles, snapshot assembly, FIFO eviction,
1992/// and the completion feed buffer.
1993#[derive(Debug)]
1994pub struct RuntimeOpsLifecycleRegistry {
1995    state: RwLock<ShellState>,
1996}
1997
1998#[derive(Debug, Clone)]
1999pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2000    pub operation_count: usize,
2001    pub active_count: usize,
2002    pub wait_request_id: Option<WaitRequestId>,
2003    pub pending_wait_present: bool,
2004    pub pending_wait_request_id: Option<WaitRequestId>,
2005    pub wait_operation_ids: Vec<OperationId>,
2006    pub operations: Vec<OperationLifecycleSnapshot>,
2007}
2008
2009impl Default for RuntimeOpsLifecycleRegistry {
2010    fn default() -> Self {
2011        Self {
2012            state: RwLock::new(ShellState::default()),
2013        }
2014    }
2015}
2016
2017impl RuntimeOpsLifecycleRegistry {
2018    pub fn new() -> Self {
2019        let dsl = new_ops_dsl_authority();
2020        let feed_capacity = DEFAULT_MAX_COMPLETED.saturating_mul(4).max(1024);
2021        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2022        Self {
2023            state: RwLock::new(ShellState {
2024                dsl,
2025                records: HashMap::new(),
2026                pending_wait: None,
2027                completed_order: VecDeque::new(),
2028                max_completed: DEFAULT_MAX_COMPLETED,
2029                max_concurrent: None,
2030                wait_request_id: None,
2031                feed_buffer,
2032                persist_tx: None,
2033                persist_epoch_id: None,
2034                persist_cursor_state: None,
2035            }),
2036        }
2037    }
2038
2039    pub fn with_config(config: OpsLifecycleConfig) -> Self {
2040        Self {
2041            state: RwLock::new(ShellState::new(config.max_completed, config.max_concurrent)),
2042        }
2043    }
2044
2045    fn recover_completion_feed_entry(
2046        shell: &mut ShellState,
2047        operation_id: &OperationId,
2048        entry: &CompletionFeedCanonicalState,
2049    ) -> Result<(), OpsLifecycleError> {
2050        let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2051        let terminal_outcome_kind =
2052            mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2053        let effects = shell.dsl_apply_with_effects(
2054            mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2055                operation_id: expected_operation_id.clone(),
2056                kind: mm_dsl::OperationKind::from(entry.kind),
2057                terminal_outcome: terminal_outcome_kind,
2058                terminal_payload: entry.terminal_outcome.clone(),
2059                completion_sequence: entry.seq,
2060            },
2061            "RecoverCompletionFeedEntry",
2062        )?;
2063        let recovered = effects.iter().find_map(|effect| match effect {
2064            mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2065                operation_id,
2066                seq,
2067                kind,
2068                terminal_outcome,
2069                terminal_payload,
2070            } => Some((
2071                operation_id,
2072                *seq,
2073                OperationKind::from(*kind),
2074                *terminal_outcome,
2075                terminal_payload,
2076            )),
2077            _ => None,
2078        });
2079        let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2080            return Err(OpsLifecycleError::Internal(
2081                "generated completion-feed recovery emitted no recovered entry".into(),
2082            ));
2083        };
2084        if operation_id != &expected_operation_id
2085            || seq != entry.seq
2086            || kind != entry.kind
2087            || terminal_outcome != terminal_outcome_kind
2088            || terminal_payload != &entry.terminal_outcome
2089        {
2090            return Err(OpsLifecycleError::Internal(format!(
2091                "generated completion-feed recovery drifted for {operation_id}"
2092            )));
2093        }
2094        Ok(())
2095    }
2096
2097    /// Wire a persistence channel for durable snapshot writes.
2098    ///
2099    /// After this call, terminal transitions (complete/fail/cancel/abort)
2100    /// capture a snapshot and queue it to the channel. A dedicated
2101    /// persistence task should drain the channel and write to the store.
2102    pub fn set_persistence_channel(
2103        &self,
2104        tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2105        epoch_id: meerkat_core::RuntimeEpochId,
2106        cursor_state: Arc<meerkat_core::EpochCursorState>,
2107    ) {
2108        if let Ok(mut state) = self.state.write() {
2109            state.persist_tx = Some(tx);
2110            state.persist_epoch_id = Some(epoch_id);
2111            state.persist_cursor_state = Some(cursor_state);
2112        }
2113    }
2114
2115    /// Recover from a persisted snapshot.
2116    ///
2117    /// Rebuilds DSL state (stripping non-terminal ops — only terminals
2118    /// survive recovery), creates fresh shell records from specs, and seeds
2119    /// the feed buffer only with completion entries accepted by generated
2120    /// recovery authority.
2121    pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2122        let PersistedOpsSnapshot {
2123            authority_state,
2124            operation_specs,
2125            completion_entries,
2126            cursors,
2127            ..
2128        } = snapshot;
2129        let max_completed = authority_state.max_completed;
2130        let max_concurrent = authority_state.max_concurrent;
2131        let next_completion_seq = authority_state.next_completion_seq;
2132        let authority_completion_entries = authority_state.completion_feed_entries;
2133        let authority_operations = authority_state.operations;
2134        let mut shell = ShellState::new(max_completed, max_concurrent);
2135
2136        // Replay every persisted op through generated recovery authority.
2137        // The transition accepts only terminal records with outcome and
2138        // completion-sequence witnesses. Volatile non-terminal rows are not
2139        // recovered; terminal/corrupt rows must fail closed instead of being
2140        // projected into shell/public feed state.
2141        let mut retained_ids: HashSet<OperationId> = HashSet::new();
2142        for (op_id, op_state) in authority_operations {
2143            let terminal_outcome = op_state
2144                .terminal_outcome
2145                .as_ref()
2146                .map(mm_dsl::OperationTerminalOutcomeKind::from);
2147            let terminal_payload = op_state.terminal_outcome.clone();
2148            let disposition = ShellState::recovered_operation_record_disposition(
2149                &op_id,
2150                op_state.status,
2151                op_state.kind,
2152                terminal_outcome.is_some(),
2153                terminal_payload.is_some(),
2154                op_state.completion_sequence.is_some(),
2155            )?;
2156            if disposition == RecoveredOperationRecordDisposition::Discard {
2157                continue;
2158            }
2159            if let Some(spec_source) = operation_specs
2160                .get(&op_id)
2161                .and_then(|spec| spec.operation_source.as_ref())
2162                && op_state.operation_source.as_ref() != Some(spec_source)
2163            {
2164                return Err(OpsLifecycleError::Internal(format!(
2165                    "persisted operation source mirror for {op_id} drifted from generated authority"
2166                )));
2167            }
2168            let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2169                operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2170                status: mm_dsl::OperationStatus::from(op_state.status),
2171                kind: mm_dsl::OperationKind::from(op_state.kind),
2172                source: op_state
2173                    .operation_source
2174                    .as_ref()
2175                    .map(mm_dsl::OperationSource::from_domain),
2176                peer_ready: op_state.peer_ready,
2177                progress_count: u64::from(op_state.progress_count),
2178                terminal_outcome,
2179                terminal_payload,
2180                completion_sequence: op_state.completion_sequence,
2181            };
2182            shell.dsl_apply(recovery, "RecoverOpRecord")?;
2183            let recovered_seq = shell.completion_sequence(&op_id).ok_or_else(|| {
2184                OpsLifecycleError::Internal(format!(
2185                    "generated op recovery accepted {op_id} without completion sequence"
2186                ))
2187            })?;
2188            if op_state.completion_sequence != Some(recovered_seq) {
2189                return Err(OpsLifecycleError::Internal(format!(
2190                    "generated op recovery completion sequence mismatch for {op_id}"
2191                )));
2192            }
2193            retained_ids.insert(op_id);
2194        }
2195        shell.dsl_apply(
2196            mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2197                next_completion_seq,
2198            },
2199            "RecoverOpsCompletionCursor",
2200        )?;
2201        shell.dsl_apply(
2202            mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2203                agent_applied_cursor: cursors.agent_applied_cursor,
2204                runtime_observed_cursor: cursors.runtime_observed_seq,
2205                runtime_injected_cursor: cursors.runtime_last_injected_seq,
2206            },
2207            "RecoverCompletionConsumerCursors",
2208        )?;
2209
2210        // Rebuild completed_order from generated completion-sequence truth,
2211        // never from the persisted shell ordering mirror.
2212        let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2213            .iter()
2214            .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2215            .collect();
2216        recovered_completed.sort_by_key(|(seq, _)| *seq);
2217        shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2218
2219        // Recover generated-owned feed authority for entries whose operation
2220        // record is no longer retained. Retained records already wrote their
2221        // feed authority through RecoverOpRecord above.
2222        for (operation_id, entry) in &authority_completion_entries {
2223            if !retained_ids.contains(operation_id) {
2224                Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2225            }
2226        }
2227
2228        let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2229        for (operation_id, entry) in &authority_completion_entries {
2230            let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2231                return Err(OpsLifecycleError::Internal(format!(
2232                    "persisted completion feed authority for {operation_id} was not recovered"
2233                )));
2234            };
2235            if recovered_entry != entry {
2236                return Err(OpsLifecycleError::Internal(format!(
2237                    "persisted completion feed authority drifted from generated recovery for {operation_id}"
2238                )));
2239            }
2240        }
2241
2242        // Projection rows may carry display metadata only after generated
2243        // feed authority has decided the operation id, sequence, kind, and
2244        // terminal outcome. Any semantic drift fails closed.
2245        let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2246        for entry in completion_entries {
2247            let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2248                return Err(OpsLifecycleError::Internal(format!(
2249                    "persisted completion feed projection for {} has no generated feed authority",
2250                    entry.operation_id
2251                )));
2252            };
2253            if authority_entry.seq != entry.seq
2254                || authority_entry.kind != entry.kind
2255                || authority_entry.terminal_outcome != entry.terminal_outcome
2256            {
2257                return Err(OpsLifecycleError::Internal(format!(
2258                    "persisted completion feed projection for {} drifted from generated feed authority",
2259                    entry.operation_id
2260                )));
2261            }
2262            projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2263        }
2264
2265        let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2266            canonical_feed_entries.into_iter().collect();
2267        recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2268        for (operation_id, entry) in recovered_entries {
2269            let projection = projection_entries_by_id.get(&operation_id);
2270            let display_name = operation_specs
2271                .get(&operation_id)
2272                .map(|spec| spec.display_name.clone())
2273                .or_else(|| projection.map(|entry| entry.display_name.clone()))
2274                .unwrap_or_default();
2275            let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2276            shell.feed_buffer.push(CompletionEntry {
2277                seq: entry.seq,
2278                operation_id,
2279                kind: entry.kind,
2280                display_name,
2281                terminal_outcome: entry.terminal_outcome,
2282                completed_at_ms,
2283            });
2284        }
2285
2286        // Rebuild shell records from specs (fresh timestamps, no watchers)
2287        // — only for operations still retained in the DSL state.
2288        for (op_id, spec) in operation_specs {
2289            if retained_ids.contains(&op_id) {
2290                let mut spec = spec;
2291                let operation_source = shell.operation_source(&op_id)?;
2292                ShellState::align_spec_child_session_id_to_source(
2293                    &mut spec,
2294                    operation_source.as_ref(),
2295                );
2296                shell.records.insert(
2297                    op_id,
2298                    ShellRecord {
2299                        spec,
2300                        peer_handle: None,
2301                        watchers: Vec::new(),
2302                        created_at: Instant::now(),
2303                        started_at: None,
2304                        completed_at: None,
2305                        created_at_wall: SystemTime::now(),
2306                    },
2307                );
2308            }
2309        }
2310
2311        Ok(Self {
2312            state: RwLock::new(shell),
2313        })
2314    }
2315
2316    /// Capture a serializable snapshot of the current state for persistence.
2317    ///
2318    /// Includes authority state, operation specs, completion entries, and
2319    /// generated completion-consumer cursor values.
2320    pub fn capture_persistence_snapshot(
2321        &self,
2322        epoch_id: meerkat_core::RuntimeEpochId,
2323        cursor_state: &meerkat_core::EpochCursorState,
2324    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2325        let state = self
2326            .state
2327            .read()
2328            .unwrap_or_else(std::sync::PoisonError::into_inner);
2329        state.capture_snapshot(epoch_id, cursor_state)
2330    }
2331
2332    /// Snapshot generated completion-consumer cursor state.
2333    pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2334        let state = self
2335            .state
2336            .read()
2337            .unwrap_or_else(std::sync::PoisonError::into_inner);
2338        state.completion_cursor_snapshot()
2339    }
2340
2341    /// Return a read handle to the completion feed.
2342    pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2343        let state = self
2344            .state
2345            .read()
2346            .unwrap_or_else(std::sync::PoisonError::into_inner);
2347        Arc::new(RuntimeCompletionFeed {
2348            buffer: Arc::clone(&state.feed_buffer),
2349        })
2350    }
2351
2352    /// Capture a stable diagnostic snapshot of the canonical ops lifecycle state.
2353    pub(crate) fn diagnostic_snapshot(
2354        &self,
2355    ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2356        let state = self
2357            .state
2358            .read()
2359            .unwrap_or_else(std::sync::PoisonError::into_inner);
2360        let mut operations = state
2361            .operation_ids()?
2362            .into_iter()
2363            .map(|id| state.snapshot(&id))
2364            .collect::<Result<Vec<_>, _>>()?
2365            .into_iter()
2366            .flatten()
2367            .collect::<Vec<_>>();
2368        operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2369        Ok(RuntimeOpsDiagnosticSnapshot {
2370            operation_count: state.operation_count(),
2371            active_count: state.active_count(),
2372            wait_request_id: state.wait_request_id.clone(),
2373            pending_wait_present: state.pending_wait.is_some(),
2374            pending_wait_request_id: state
2375                .pending_wait
2376                .as_ref()
2377                .map(|pending_wait| pending_wait.wait_request_id.clone()),
2378            wait_operation_ids: state.wait_operation_ids()?,
2379            operations,
2380        })
2381    }
2382
2383    fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2384        self.state
2385            .read()
2386            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2387    }
2388
2389    fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2390        self.state
2391            .write()
2392            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2393    }
2394
2395    fn cancel_wait_all_internal(
2396        &self,
2397        wait_request_id: &WaitRequestId,
2398    ) -> Result<(), OpsLifecycleError> {
2399        let mut state = self.write_state()?;
2400        match state.wait_request_id.as_ref() {
2401            Some(active) if active == wait_request_id => {
2402                // Clear the DSL barrier via the dedicated `CancelWaitAll`
2403                // transition. Unlike `SatisfyWaitAll`, it does not require
2404                // every member to be terminal (the request was dropped, not
2405                // resolved) and does not emit the `WaitAllSatisfied`.
2406                state.dsl_apply(
2407                    mm_dsl::MeerkatMachineInput::CancelWaitAll,
2408                    "CancelWaitAll(cancel)",
2409                )?;
2410                state.wait_request_id = None;
2411                if state
2412                    .pending_wait
2413                    .as_ref()
2414                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2415                {
2416                    state.pending_wait = None;
2417                }
2418                Ok(())
2419            }
2420            _ => {
2421                if state
2422                    .pending_wait
2423                    .as_ref()
2424                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2425                {
2426                    state.pending_wait = None;
2427                }
2428                Ok(())
2429            }
2430        }
2431    }
2432}
2433
2434enum WaitAllFutureState {
2435    Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2436    Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2437    Done,
2438}
2439
2440struct WaitAllFuture<'a> {
2441    registry: &'a RuntimeOpsLifecycleRegistry,
2442    wait_request_id: WaitRequestId,
2443    state: WaitAllFutureState,
2444}
2445
2446impl Future for WaitAllFuture<'_> {
2447    type Output = Result<WaitAllResult, OpsLifecycleError>;
2448
2449    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2450        match &mut self.state {
2451            WaitAllFutureState::Ready(result) => {
2452                let ready = result.take().unwrap_or_else(|| {
2453                    Err(OpsLifecycleError::Internal(
2454                        "wait_all future polled after completion".into(),
2455                    ))
2456                });
2457                self.state = WaitAllFutureState::Done;
2458                Poll::Ready(ready)
2459            }
2460            WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2461                Poll::Pending => Poll::Pending,
2462                Poll::Ready(Ok(satisfied)) => {
2463                    let outcomes = match self.registry.read_state() {
2464                        Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2465                        Err(err) => Err(err),
2466                    };
2467                    self.state = WaitAllFutureState::Done;
2468                    Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2469                        outcomes,
2470                        satisfied,
2471                    }))
2472                }
2473                Poll::Ready(Err(_)) => {
2474                    self.state = WaitAllFutureState::Done;
2475                    Poll::Ready(Err(OpsLifecycleError::Internal(
2476                        "wait_all completion channel dropped".into(),
2477                    )))
2478                }
2479            },
2480            WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2481                "wait_all future polled after completion".into(),
2482            ))),
2483        }
2484    }
2485}
2486
2487impl Drop for WaitAllFuture<'_> {
2488    fn drop(&mut self) {
2489        if matches!(self.state, WaitAllFutureState::Waiting(_))
2490            && let Err(err) = self
2491                .registry
2492                .cancel_wait_all_internal(&self.wait_request_id)
2493        {
2494            tracing::error!(
2495                wait_request_id = %self.wait_request_id,
2496                error = %err,
2497                "generated wait_all authority rejected cancellation during drop"
2498            );
2499        }
2500    }
2501}
2502
2503// ---------------------------------------------------------------------------
2504// Generated op lifecycle result-class feedback
2505// ---------------------------------------------------------------------------
2506
2507fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2508    match action {
2509        mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2510        mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2511        mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2512        mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2513        mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2514        mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2515        mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2516        mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2517        mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2518        mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2519    }
2520}
2521
2522fn op_lifecycle_rejection_error_from_effects(
2523    id: &OperationId,
2524    requested_action: mm_dsl::OpLifecycleActionKind,
2525    effects: &[mm_dsl::MeerkatMachineEffect],
2526) -> Result<OpsLifecycleError, OpsLifecycleError> {
2527    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2528    let mut rejection = None;
2529    for effect in effects {
2530        let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2531            operation_id,
2532            action,
2533            reason,
2534            status,
2535        } = effect
2536        else {
2537            continue;
2538        };
2539        if rejection.is_some() {
2540            return Err(OpsLifecycleError::Internal(
2541                "generated op lifecycle authority emitted multiple rejection results".into(),
2542            ));
2543        }
2544        if operation_id != &expected_id || *action != requested_action {
2545            return Err(OpsLifecycleError::Internal(format!(
2546                "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2547            )));
2548        }
2549        let error = match reason {
2550            mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2551                if status.is_some() {
2552                    return Err(OpsLifecycleError::Internal(
2553                        "generated op lifecycle authority emitted not-found with status".into(),
2554                    ));
2555                }
2556                OpsLifecycleError::NotFound(id.clone())
2557            }
2558            mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2559                let status = status.ok_or_else(|| {
2560                    OpsLifecycleError::Internal(
2561                        "generated op lifecycle authority emitted invalid-transition without status"
2562                            .into(),
2563                    )
2564                })?;
2565                OpsLifecycleError::InvalidTransition {
2566                    id: id.clone(),
2567                    status: OperationStatus::from(status),
2568                    action: op_lifecycle_action_label(requested_action),
2569                }
2570            }
2571            mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2572                if status.is_none() {
2573                    return Err(OpsLifecycleError::Internal(
2574                        "generated op lifecycle authority emitted peer-not-expected without status"
2575                            .into(),
2576                    ));
2577                }
2578                OpsLifecycleError::PeerNotExpected(id.clone())
2579            }
2580            mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2581                if status.is_none() {
2582                    return Err(OpsLifecycleError::Internal(
2583                        "generated op lifecycle authority emitted already-peer-ready without status"
2584                            .into(),
2585                    ));
2586                }
2587                OpsLifecycleError::AlreadyPeerReady(id.clone())
2588            }
2589        };
2590        rejection = Some(error);
2591    }
2592    rejection.ok_or_else(|| {
2593        OpsLifecycleError::Internal(
2594            "generated op lifecycle authority emitted no rejection result".into(),
2595        )
2596    })
2597}
2598
2599fn classify_generated_op_rejection(
2600    state: &mut ShellState,
2601    err: mm_dsl::MeerkatMachineTransitionError,
2602    id: &OperationId,
2603    action: mm_dsl::OpLifecycleActionKind,
2604) -> OpsLifecycleError {
2605    match err {
2606        mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2607            match state.dsl_apply_with_effects(
2608                mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2609                    operation_id: mm_dsl::OperationId::from_domain(id).0,
2610                    action,
2611                },
2612                "ResolveOpLifecycleTransitionRejection",
2613            ) {
2614                Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2615                    .unwrap_or_else(|err| err),
2616                Err(err) => err,
2617            }
2618        }
2619        other => OpsLifecycleError::Internal(format!(
2620            "DSL rejected ops transition ({}): {other:?}",
2621            op_lifecycle_action_label(action)
2622        )),
2623    }
2624}
2625
2626fn apply_op_transition(
2627    state: &mut ShellState,
2628    id: &OperationId,
2629    input: mm_dsl::MeerkatMachineInput,
2630    action: mm_dsl::OpLifecycleActionKind,
2631) -> Result<(), OpsLifecycleError> {
2632    state
2633        .dsl_apply_raw(input)
2634        .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2635}
2636
2637fn apply_terminal_op_transition_and_persist(
2638    state: &mut ShellState,
2639    id: &OperationId,
2640    input: mm_dsl::MeerkatMachineInput,
2641    action: mm_dsl::OpLifecycleActionKind,
2642) -> Result<(), OpsLifecycleError> {
2643    let previous_snapshot = state.dsl.0.snapshot();
2644    apply_op_transition(state, id, input, action)?;
2645    if let Err(err) = state.maybe_persist() {
2646        state.dsl.0.restore_snapshot(previous_snapshot);
2647        return Err(err);
2648    }
2649    Ok(())
2650}
2651
2652fn op_registration_error_from_effects(
2653    id: &OperationId,
2654    effects: &[mm_dsl::MeerkatMachineEffect],
2655) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2656    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2657    let mut admission = None;
2658    for effect in effects {
2659        let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2660            operation_id,
2661            result,
2662            reject_reason,
2663            max_concurrent_limit,
2664            active_op_count,
2665        } = effect
2666        else {
2667            continue;
2668        };
2669        if admission.is_some() {
2670            return Err(OpsLifecycleError::Internal(
2671                "generated op registration authority emitted multiple admission results".into(),
2672            ));
2673        }
2674        if operation_id != &expected_id {
2675            return Err(OpsLifecycleError::Internal(format!(
2676                "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2677            )));
2678        }
2679        admission = Some(match result {
2680            mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2681                if reject_reason.is_some() {
2682                    return Err(OpsLifecycleError::Internal(
2683                        "generated op registration authority accepted with rejection reason".into(),
2684                    ));
2685                }
2686                None
2687            }
2688            mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2689                let reason = reject_reason.ok_or_else(|| {
2690                    OpsLifecycleError::Internal(
2691                        "generated op registration authority rejected without reason".into(),
2692                    )
2693                })?;
2694                let error = match reason {
2695                    mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2696                        OpsLifecycleError::AlreadyRegistered(id.clone())
2697                    }
2698                    mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2699                        let limit = max_concurrent_limit.ok_or_else(|| {
2700                            OpsLifecycleError::Internal(
2701                                "generated op registration authority rejected capacity without limit"
2702                                    .into(),
2703                            )
2704                        })?;
2705                        OpsLifecycleError::MaxConcurrentExceeded {
2706                            limit: limit as usize,
2707                            active: *active_op_count as usize,
2708                        }
2709                    }
2710                };
2711                Some(error)
2712            }
2713        });
2714    }
2715    admission.ok_or_else(|| {
2716        OpsLifecycleError::Internal(
2717            "generated op registration authority emitted no admission result".into(),
2718        )
2719    })
2720}
2721
2722impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2723    fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2724        self.register_operation_with_admission_limit(spec, None)
2725    }
2726
2727    fn register_operation_with_admission_limit(
2728        &self,
2729        mut spec: OperationSpec,
2730        max_concurrent: Option<usize>,
2731    ) -> Result<(), OpsLifecycleError> {
2732        let mut state = self.write_state()?;
2733        let operation_id = spec.id.clone();
2734        let kind = spec.kind;
2735        let max_concurrent = max_concurrent
2736            .or(state.max_concurrent)
2737            .map(|limit| limit as u64);
2738
2739        let effects = state.dsl_apply_with_effects(
2740            mm_dsl::MeerkatMachineInput::RegisterOp {
2741                operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2742                kind: mm_dsl::OperationKind::from_domain(&kind),
2743                source: spec
2744                    .operation_source
2745                    .as_ref()
2746                    .map(mm_dsl::OperationSource::from_domain),
2747                max_concurrent,
2748            },
2749            "RegisterOp",
2750        )?;
2751        if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
2752            return Err(error);
2753        }
2754
2755        let authority_operation_source = state.operation_source(&operation_id)?;
2756        ShellState::align_spec_child_session_id_to_source(
2757            &mut spec,
2758            authority_operation_source.as_ref(),
2759        );
2760
2761        // Insert shell record.
2762        state.records.insert(operation_id, ShellRecord::new(spec));
2763        Ok(())
2764    }
2765
2766    fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2767        let mut state = self.write_state()?;
2768
2769        apply_op_transition(
2770            &mut state,
2771            id,
2772            mm_dsl::MeerkatMachineInput::StartOp {
2773                operation_id: mm_dsl::OperationId::from_domain(id).0,
2774            },
2775            mm_dsl::OpLifecycleActionKind::Start,
2776        )?;
2777
2778        // Shell concern: record the started timestamp.
2779        if let Some(shell) = state.records.get_mut(id) {
2780            shell.started_at = Some(Instant::now());
2781        }
2782        Ok(())
2783    }
2784
2785    fn provisioning_failed(
2786        &self,
2787        id: &OperationId,
2788        error: String,
2789    ) -> Result<(), OpsLifecycleError> {
2790        let mut state = self.write_state()?;
2791
2792        let terminal_outcome = OperationTerminalOutcome::Failed { error };
2793        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2794
2795        apply_terminal_op_transition_and_persist(
2796            &mut state,
2797            id,
2798            mm_dsl::MeerkatMachineInput::FailOp {
2799                operation_id: mm_dsl::OperationId::from_domain(id).0,
2800                outcome: outcome_kind,
2801                payload: terminal_outcome,
2802            },
2803            mm_dsl::OpLifecycleActionKind::Fail,
2804        )?;
2805
2806        let completion_entry = state.finalize_terminal(id)?;
2807        state.publish_completion_entry(completion_entry);
2808        Ok(())
2809    }
2810
2811    fn peer_ready(
2812        &self,
2813        id: &OperationId,
2814        peer: OperationPeerHandle,
2815    ) -> Result<(), OpsLifecycleError> {
2816        let mut state = self.write_state()?;
2817
2818        apply_op_transition(
2819            &mut state,
2820            id,
2821            mm_dsl::MeerkatMachineInput::PeerReadyOp {
2822                operation_id: mm_dsl::OperationId::from_domain(id).0,
2823            },
2824            mm_dsl::OpLifecycleActionKind::PeerReady,
2825        )?;
2826
2827        // Shell concern: store the peer handle.
2828        if let Some(shell) = state.records.get_mut(id) {
2829            shell.peer_handle = Some(peer);
2830        }
2831        Ok(())
2832    }
2833
2834    fn register_watcher(
2835        &self,
2836        id: &OperationId,
2837    ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
2838        let mut state = self.write_state()?;
2839
2840        if !state.contains(id) {
2841            return Err(OpsLifecycleError::NotFound(id.clone()));
2842        }
2843
2844        // If already terminal, return an already-resolved watch.
2845        if let Some(outcome) = state.terminal_outcome(id)? {
2846            return Ok(resolved_operation_completion_watch(outcome));
2847        }
2848
2849        // Shell concern: create the channel and store the sender.
2850        let shell = state.shell_record_mut(id)?;
2851        let (tx, rx) = tokio::sync::oneshot::channel();
2852        let watch = operation_completion_watch_from_receiver(rx);
2853        shell.watchers.push(OperationCompletionNotifier::new(tx));
2854        Ok(watch)
2855    }
2856
2857    fn report_progress(
2858        &self,
2859        id: &OperationId,
2860        _update: OperationProgressUpdate,
2861    ) -> Result<(), OpsLifecycleError> {
2862        let mut state = self.write_state()?;
2863
2864        apply_op_transition(
2865            &mut state,
2866            id,
2867            mm_dsl::MeerkatMachineInput::ProgressReportedOp {
2868                operation_id: mm_dsl::OperationId::from_domain(id).0,
2869            },
2870            mm_dsl::OpLifecycleActionKind::ProgressReported,
2871        )?;
2872        Ok(())
2873    }
2874
2875    fn complete_operation(
2876        &self,
2877        id: &OperationId,
2878        result: OperationResult,
2879    ) -> Result<(), OpsLifecycleError> {
2880        let mut state = self.write_state()?;
2881
2882        let terminal_outcome = OperationTerminalOutcome::Completed(result);
2883        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2884
2885        apply_terminal_op_transition_and_persist(
2886            &mut state,
2887            id,
2888            mm_dsl::MeerkatMachineInput::CompleteOp {
2889                operation_id: mm_dsl::OperationId::from_domain(id).0,
2890                outcome: outcome_kind,
2891                payload: terminal_outcome,
2892            },
2893            mm_dsl::OpLifecycleActionKind::Complete,
2894        )?;
2895
2896        let completion_entry = state.finalize_terminal(id)?;
2897        state.publish_completion_entry(completion_entry);
2898        Ok(())
2899    }
2900
2901    fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
2902        let mut state = self.write_state()?;
2903
2904        let terminal_outcome = OperationTerminalOutcome::Failed { error };
2905        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2906
2907        apply_terminal_op_transition_and_persist(
2908            &mut state,
2909            id,
2910            mm_dsl::MeerkatMachineInput::FailOp {
2911                operation_id: mm_dsl::OperationId::from_domain(id).0,
2912                outcome: outcome_kind,
2913                payload: terminal_outcome,
2914            },
2915            mm_dsl::OpLifecycleActionKind::Fail,
2916        )?;
2917
2918        let completion_entry = state.finalize_terminal(id)?;
2919        state.publish_completion_entry(completion_entry);
2920        Ok(())
2921    }
2922
2923    fn abort_provisioning(
2924        &self,
2925        id: &OperationId,
2926        reason: Option<String>,
2927    ) -> Result<(), OpsLifecycleError> {
2928        let mut state = self.write_state()?;
2929
2930        let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
2931        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2932
2933        apply_terminal_op_transition_and_persist(
2934            &mut state,
2935            id,
2936            mm_dsl::MeerkatMachineInput::AbortOp {
2937                operation_id: mm_dsl::OperationId::from_domain(id).0,
2938                outcome: outcome_kind,
2939                payload: terminal_outcome,
2940            },
2941            mm_dsl::OpLifecycleActionKind::Abort,
2942        )?;
2943
2944        let completion_entry = state.finalize_terminal(id)?;
2945        state.publish_completion_entry(completion_entry);
2946        Ok(())
2947    }
2948
2949    fn cancel_operation(
2950        &self,
2951        id: &OperationId,
2952        reason: Option<String>,
2953    ) -> Result<(), OpsLifecycleError> {
2954        let mut state = self.write_state()?;
2955
2956        let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
2957        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2958
2959        apply_terminal_op_transition_and_persist(
2960            &mut state,
2961            id,
2962            mm_dsl::MeerkatMachineInput::CancelOp {
2963                operation_id: mm_dsl::OperationId::from_domain(id).0,
2964                outcome: outcome_kind,
2965                payload: terminal_outcome,
2966            },
2967            mm_dsl::OpLifecycleActionKind::Cancel,
2968        )?;
2969
2970        let completion_entry = state.finalize_terminal(id)?;
2971        state.publish_completion_entry(completion_entry);
2972        Ok(())
2973    }
2974
2975    fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2976        let mut state = self.write_state()?;
2977
2978        apply_op_transition(
2979            &mut state,
2980            id,
2981            mm_dsl::MeerkatMachineInput::RetireRequestedOp {
2982                operation_id: mm_dsl::OperationId::from_domain(id).0,
2983            },
2984            mm_dsl::OpLifecycleActionKind::RetireRequested,
2985        )?;
2986        Ok(())
2987    }
2988
2989    fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2990        let mut state = self.write_state()?;
2991
2992        let terminal_outcome = OperationTerminalOutcome::Retired;
2993        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2994
2995        apply_terminal_op_transition_and_persist(
2996            &mut state,
2997            id,
2998            mm_dsl::MeerkatMachineInput::RetireCompletedOp {
2999                operation_id: mm_dsl::OperationId::from_domain(id).0,
3000                outcome: outcome_kind,
3001                payload: terminal_outcome,
3002            },
3003            mm_dsl::OpLifecycleActionKind::RetireCompleted,
3004        )?;
3005
3006        let completion_entry = state.finalize_terminal(id)?;
3007        state.publish_completion_entry(completion_entry);
3008        Ok(())
3009    }
3010
3011    fn snapshot(
3012        &self,
3013        id: &OperationId,
3014    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3015        let state = self.read_state()?;
3016        state.snapshot(id)
3017    }
3018
3019    fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3020        let state = self.read_state()?;
3021        let mut snapshots = Vec::new();
3022        for id in state.operation_ids()? {
3023            let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3024                OpsLifecycleError::Internal(format!(
3025                    "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3026                ))
3027            })?;
3028            snapshots.push(snapshot);
3029        }
3030        snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3031        Ok(snapshots)
3032    }
3033
3034    fn classify_operation_terminality(
3035        &self,
3036        id: &OperationId,
3037        status: OperationStatus,
3038    ) -> Result<bool, OpsLifecycleError> {
3039        ShellState::operation_status_is_terminal(id, status)
3040    }
3041
3042    fn classify_operation_public_result(
3043        &self,
3044        id: &OperationId,
3045    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3046        let state = self.read_state()?;
3047        let status = match state.status(id) {
3048            Some(status) => status,
3049            None if state.records.contains_key(id)
3050                || state.has_generated_operation_record_fact(id) =>
3051            {
3052                return Err(OpsLifecycleError::Internal(format!(
3053                    "generated op lifecycle authority missing status for {id}"
3054                )));
3055            }
3056            None => OperationStatus::Absent,
3057        };
3058        ShellState::operation_public_result_class(id, status)
3059    }
3060
3061    fn classify_operation_completion_wake(
3062        &self,
3063        id: &OperationId,
3064        kind: OperationKind,
3065    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3066        ShellState::operation_completion_wake_class(id, kind)
3067    }
3068
3069    fn classify_operation_transition_idempotence(
3070        &self,
3071        id: &OperationId,
3072        action: OperationLifecycleAction,
3073    ) -> Result<bool, OpsLifecycleError> {
3074        let state = self.read_state()?;
3075        let status = match state.status(id) {
3076            Some(status) => status,
3077            None if state.records.contains_key(id)
3078                || state.has_generated_operation_record_fact(id) =>
3079            {
3080                return Err(OpsLifecycleError::Internal(format!(
3081                    "generated op lifecycle authority missing status for {id}"
3082                )));
3083            }
3084            None => OperationStatus::Absent,
3085        };
3086        ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3087    }
3088
3089    fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3090        let mut state = self.write_state()?;
3091
3092        let to_terminate = state.owner_termination_targets()?;
3093
3094        for (op_id, _status) in &to_terminate {
3095            let terminal_outcome = OperationTerminalOutcome::Terminated {
3096                reason: reason.clone(),
3097            };
3098            let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3099
3100            apply_terminal_op_transition_and_persist(
3101                &mut state,
3102                op_id,
3103                mm_dsl::MeerkatMachineInput::TerminateOp {
3104                    operation_id: mm_dsl::OperationId::from_domain(op_id).0,
3105                    outcome: outcome_kind,
3106                    payload: terminal_outcome,
3107                },
3108                mm_dsl::OpLifecycleActionKind::Terminate,
3109            )?;
3110
3111            if let Some(entry) = state.finalize_terminal(op_id)? {
3112                state.publish_completion_entry(Some(entry));
3113            }
3114        }
3115        Ok(())
3116    }
3117
3118    fn collect_completed(
3119        &self,
3120    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3121        let mut state = self.write_state()?;
3122
3123        let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3124        let mut collected = Vec::with_capacity(ids.len());
3125        for id in ids {
3126            let outcome = state.terminal_outcome(&id)?;
3127            state.dsl_apply(
3128                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3129                    operation_id: mm_dsl::OperationId::from_domain(&id).0,
3130                },
3131                "CollectCompletedOp",
3132            )?;
3133            state.completed_order.retain(|queued| queued != &id);
3134            state.records.remove(&id);
3135            if let Some(outcome) = outcome {
3136                collected.push((id, outcome));
3137            }
3138        }
3139        Ok(collected)
3140    }
3141
3142    fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3143        Some(self.completion_feed_handle())
3144    }
3145
3146    fn completion_cursor(
3147        &self,
3148        consumer: CompletionCursorConsumer,
3149    ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3150        let state = self.read_state()?;
3151        Ok(Some(state.completion_cursor(consumer)))
3152    }
3153
3154    fn advance_completion_cursor(
3155        &self,
3156        consumer: CompletionCursorConsumer,
3157        cursor: CompletionSeq,
3158        projection: Option<&meerkat_core::EpochCursorState>,
3159    ) -> Result<CompletionSeq, OpsLifecycleError> {
3160        let mut state = self.write_state()?;
3161        let input = match consumer {
3162            CompletionCursorConsumer::AgentApplied => {
3163                mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3164            }
3165            CompletionCursorConsumer::RuntimeObserved => {
3166                mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3167            }
3168            CompletionCursorConsumer::RuntimeInjected => {
3169                mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3170            }
3171        };
3172        let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3173        let advanced = effects
3174            .iter()
3175            .find_map(|effect| match (consumer, effect) {
3176                (
3177                    CompletionCursorConsumer::AgentApplied,
3178                    mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3179                ) => Some(*cursor),
3180                (
3181                    CompletionCursorConsumer::RuntimeObserved,
3182                    mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3183                        cursor,
3184                    },
3185                ) => Some(*cursor),
3186                (
3187                    CompletionCursorConsumer::RuntimeInjected,
3188                    mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3189                        cursor,
3190                    },
3191                ) => Some(*cursor),
3192                _ => None,
3193            })
3194            .ok_or_else(|| {
3195                OpsLifecycleError::Internal(format!(
3196                    "generated completion cursor transition emitted no feedback for {consumer:?}"
3197                ))
3198            })?;
3199        if let Some(projection) = projection {
3200            projection.project_authorized_completion_cursor(consumer, advanced);
3201        }
3202        Ok(advanced)
3203    }
3204
3205    fn wait_all(
3206        &self,
3207        run_id: &RunId,
3208        ids: &[OperationId],
3209    ) -> std::pin::Pin<
3210        Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3211    > {
3212        let wait_request_id = WaitRequestId::new();
3213        let owned_ids = ids.to_vec();
3214
3215        let state = match self.write_state() {
3216            Ok(mut state) => {
3217                match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3218                    Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3219                        let outcomes =
3220                            state
3221                                .collect_wait_outcomes(&satisfied.operation_ids)
3222                                .map(|outcomes| WaitAllResult {
3223                                    outcomes,
3224                                    satisfied,
3225                                });
3226                        WaitAllFutureState::Ready(Some(outcomes))
3227                    }
3228                    Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3229                        if state.pending_wait.is_some() {
3230                            // Roll back the DSL barrier we just activated so the
3231                            // registry is not stuck in a wait-active state with
3232                            // no correlation oneshot to resolve. `CancelWaitAll`
3233                            // is the no-obligation clearer (members need not be
3234                            // terminal).
3235                            let rollback = state.dsl_apply(
3236                                mm_dsl::MeerkatMachineInput::CancelWaitAll,
3237                                "CancelWaitAll(rollback)",
3238                            );
3239                            return Box::pin(WaitAllFuture {
3240                                registry: self,
3241                                wait_request_id,
3242                                state: WaitAllFutureState::Ready(Some(Err(match rollback {
3243                                    Ok(()) => OpsLifecycleError::Internal(
3244                                        "wait_all started while a pending wait sender already existed"
3245                                            .into(),
3246                                    ),
3247                                    Err(err) => err,
3248                                }))),
3249                            });
3250                        }
3251                        state.wait_request_id = Some(wait_request_id.clone());
3252                        let (sender, receiver) = tokio::sync::oneshot::channel();
3253                        state.pending_wait = Some(PendingWaitState {
3254                            wait_request_id: wait_request_id.clone(),
3255                            sender,
3256                        });
3257                        WaitAllFutureState::Waiting(receiver)
3258                    }
3259                    Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3260                }
3261            }
3262            Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3263        };
3264
3265        Box::pin(WaitAllFuture {
3266            registry: self,
3267            wait_request_id,
3268            state,
3269        })
3270    }
3271}
3272
3273#[cfg(test)]
3274#[allow(clippy::unwrap_used, clippy::panic)]
3275mod tests {
3276    use super::*;
3277    use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3278    use meerkat_core::lifecycle::RunId;
3279    use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3280    use meerkat_core::types::SessionId;
3281    use std::sync::atomic::Ordering;
3282    use uuid::Uuid;
3283
3284    fn test_run_id() -> RunId {
3285        RunId(Uuid::from_u128(1))
3286    }
3287
3288    fn background_spec(name: &str) -> OperationSpec {
3289        OperationSpec {
3290            id: OperationId::new(),
3291            kind: OperationKind::BackgroundToolOp,
3292            owner_session_id: SessionId::new(),
3293            display_name: name.into(),
3294            source_label: "test".into(),
3295            operation_source: None,
3296            child_session_id: None,
3297            expect_peer_channel: false,
3298        }
3299    }
3300
3301    #[tokio::test]
3302    async fn late_watchers_resolve_immediately() {
3303        let registry = RuntimeOpsLifecycleRegistry::new();
3304        let spec = background_spec("late");
3305        let op_id = spec.id.clone();
3306        registry.register_operation(spec).unwrap();
3307        registry.provisioning_succeeded(&op_id).unwrap();
3308        registry
3309            .complete_operation(
3310                &op_id,
3311                OperationResult {
3312                    id: op_id.clone(),
3313                    content: "done".into(),
3314                    is_error: false,
3315                    duration_ms: 1,
3316                    tokens_used: 0,
3317                },
3318            )
3319            .unwrap();
3320
3321        let watch = registry.register_watcher(&op_id).unwrap();
3322        match watch
3323            .await
3324            .expect("operation completion watch should resolve")
3325        {
3326            OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3327            other => panic!("expected completed outcome, got {other:?}"),
3328        }
3329    }
3330
3331    #[tokio::test]
3332    async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3333        let (tx, rx) = tokio::sync::oneshot::channel();
3334        let watch = operation_completion_watch_from_receiver(rx);
3335        drop(tx);
3336
3337        assert_eq!(
3338            watch.await,
3339            Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3340        );
3341    }
3342
3343    #[test]
3344    fn peer_ready_requires_peer_expectation() {
3345        let registry = RuntimeOpsLifecycleRegistry::new();
3346        let spec = background_spec("no-peer");
3347        let op_id = spec.id.clone();
3348        registry.register_operation(spec).unwrap();
3349        registry.provisioning_succeeded(&op_id).unwrap();
3350
3351        let result = registry.peer_ready(
3352            &op_id,
3353            OperationPeerHandle {
3354                peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3355                trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3356                    "peer",
3357                    PeerId::new(),
3358                    "inproc://peer",
3359                )
3360                .unwrap(),
3361            },
3362        );
3363        assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3364    }
3365
3366    /// K8b pinning test: the terminal payload carried through the generated
3367    /// machine IS the typed domain outcome — no JSON codec exists in either
3368    /// direction. Every variant classifies to its matching discriminant, the
3369    /// fail-closed read returns the exact payload when the discriminant
3370    /// matches, and rejects any discriminant/variant disagreement.
3371    #[test]
3372    fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3373        let op_id = OperationId::new();
3374        let outcomes = vec![
3375            (
3376                OperationTerminalOutcome::Completed(OperationResult {
3377                    id: op_id.clone(),
3378                    content: "done".into(),
3379                    is_error: false,
3380                    duration_ms: 7,
3381                    tokens_used: 42,
3382                }),
3383                mm_dsl::OperationTerminalOutcomeKind::Completed,
3384            ),
3385            (
3386                OperationTerminalOutcome::Failed {
3387                    error: "boom".into(),
3388                },
3389                mm_dsl::OperationTerminalOutcomeKind::Failed,
3390            ),
3391            (
3392                OperationTerminalOutcome::Aborted {
3393                    reason: Some("user aborted".into()),
3394                },
3395                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3396            ),
3397            (
3398                OperationTerminalOutcome::Aborted { reason: None },
3399                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3400            ),
3401            (
3402                OperationTerminalOutcome::Cancelled {
3403                    reason: Some("cancelled".into()),
3404                },
3405                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3406            ),
3407            (
3408                OperationTerminalOutcome::Cancelled { reason: None },
3409                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3410            ),
3411            (
3412                OperationTerminalOutcome::Retired,
3413                mm_dsl::OperationTerminalOutcomeKind::Retired,
3414            ),
3415            (
3416                OperationTerminalOutcome::Terminated {
3417                    reason: "owner stopped".into(),
3418                },
3419                mm_dsl::OperationTerminalOutcomeKind::Terminated,
3420            ),
3421        ];
3422
3423        for (outcome, expected_kind) in &outcomes {
3424            assert_eq!(
3425                mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3426                *expected_kind,
3427                "typed payload {outcome:?} must classify to {expected_kind:?}"
3428            );
3429            let read = ShellState::checked_terminal_payload(
3430                *expected_kind,
3431                outcome,
3432                "test authority",
3433                "test-op",
3434            )
3435            .expect("matching discriminant must read back the exact payload");
3436            assert_eq!(&read, outcome);
3437        }
3438
3439        // Discriminant/variant disagreement fails closed.
3440        let err = ShellState::checked_terminal_payload(
3441            mm_dsl::OperationTerminalOutcomeKind::Completed,
3442            &OperationTerminalOutcome::Retired,
3443            "test authority",
3444            "test-op",
3445        )
3446        .expect_err("variant mismatch must be rejected");
3447        assert!(matches!(err, OpsLifecycleError::Internal(_)));
3448    }
3449
3450    /// K8b machine-ownership pin: a terminal payload whose variant disagrees
3451    /// with the transition's terminal kind is rejected by the generated
3452    /// machine guard (`payload_variant_matches_kind`) — not by any shell
3453    /// decode step. Drives the DSL input directly to prove the guard owns
3454    /// the invariant.
3455    #[test]
3456    fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3457        let registry = RuntimeOpsLifecycleRegistry::new();
3458        let spec = background_spec("variant-mismatch");
3459        let op_id = spec.id.clone();
3460        registry.register_operation(spec).unwrap();
3461        registry.provisioning_succeeded(&op_id).unwrap();
3462
3463        let mut state = registry.write_state().unwrap();
3464        let err = state
3465            .dsl_apply(
3466                mm_dsl::MeerkatMachineInput::CompleteOp {
3467                    operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3468                    outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3469                    // Variant mismatch: Completed kind with a Retired payload.
3470                    payload: OperationTerminalOutcome::Retired,
3471                },
3472                "CompleteOp",
3473            )
3474            .expect_err("machine must reject payload variant mismatch");
3475        // The kernel reports guard rejections without naming the guard; the
3476        // discriminating pin is the pair: identical input EXCEPT the payload
3477        // variant is guard-rejected here, then accepted below. Status
3478        // (`Running`) and discriminant (`Completed`) are identical in both
3479        // calls, so `payload_variant_matches_kind` is the only differing
3480        // guard.
3481        assert!(
3482            matches!(
3483                &err,
3484                OpsLifecycleError::Internal(message)
3485                    if message.contains("GuardRejected") && message.contains("CompleteOp")
3486            ),
3487            "expected generated guard rejection for CompleteOp, got: {err:?}"
3488        );
3489        drop(state);
3490
3491        // RetireCompletedOp requires the unit Retired payload; a data-carrying
3492        // payload is rejected by the same guard shape.
3493        registry
3494            .complete_operation(
3495                &op_id,
3496                OperationResult {
3497                    id: op_id.clone(),
3498                    content: "done".into(),
3499                    is_error: false,
3500                    duration_ms: 1,
3501                    tokens_used: 0,
3502                },
3503            )
3504            .expect("matching variant must complete");
3505    }
3506
3507    #[test]
3508    fn duplicate_registration_rejection_is_generated() {
3509        let registry = RuntimeOpsLifecycleRegistry::new();
3510        let spec = background_spec("duplicate");
3511        let op_id = spec.id.clone();
3512
3513        registry.register_operation(spec.clone()).unwrap();
3514        let result = registry.register_operation(spec);
3515
3516        assert!(matches!(
3517            result,
3518            Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3519        ));
3520    }
3521
3522    #[test]
3523    fn invalid_transition_rejection_is_generated() {
3524        let registry = RuntimeOpsLifecycleRegistry::new();
3525        let spec = background_spec("invalid-transition");
3526        let op_id = spec.id.clone();
3527        registry.register_operation(spec).unwrap();
3528
3529        let result = registry.complete_operation(
3530            &op_id,
3531            OperationResult {
3532                id: op_id.clone(),
3533                content: "too-early".into(),
3534                is_error: false,
3535                duration_ms: 1,
3536                tokens_used: 0,
3537            },
3538        );
3539
3540        assert!(matches!(
3541            result,
3542            Err(OpsLifecycleError::InvalidTransition {
3543                id,
3544                status: OperationStatus::Provisioning,
3545                action: "complete_operation",
3546            }) if id == op_id
3547        ));
3548    }
3549
3550    #[tokio::test]
3551    async fn multi_listener_completion() {
3552        let registry = RuntimeOpsLifecycleRegistry::new();
3553        let spec = background_spec("multi");
3554        let op_id = spec.id.clone();
3555        registry.register_operation(spec).unwrap();
3556        registry.provisioning_succeeded(&op_id).unwrap();
3557
3558        let watch1 = registry.register_watcher(&op_id).unwrap();
3559        let watch2 = registry.register_watcher(&op_id).unwrap();
3560        let watch3 = registry.register_watcher(&op_id).unwrap();
3561
3562        registry
3563            .complete_operation(
3564                &op_id,
3565                OperationResult {
3566                    id: op_id.clone(),
3567                    content: "multi-done".into(),
3568                    is_error: false,
3569                    duration_ms: 1,
3570                    tokens_used: 0,
3571                },
3572            )
3573            .unwrap();
3574
3575        for watch in [watch1, watch2, watch3] {
3576            match watch
3577                .await
3578                .expect("operation completion watch should resolve")
3579            {
3580                OperationTerminalOutcome::Completed(result) => {
3581                    assert_eq!(result.content, "multi-done");
3582                }
3583                other => panic!("expected completed, got {other:?}"),
3584            }
3585        }
3586    }
3587
3588    #[tokio::test]
3589    async fn wait_all_returns_all_outcomes() {
3590        let registry = RuntimeOpsLifecycleRegistry::new();
3591
3592        let spec_a = background_spec("a");
3593        let id_a = spec_a.id.clone();
3594        registry.register_operation(spec_a).unwrap();
3595        registry.provisioning_succeeded(&id_a).unwrap();
3596
3597        let spec_b = background_spec("b");
3598        let id_b = spec_b.id.clone();
3599        registry.register_operation(spec_b).unwrap();
3600        registry.provisioning_succeeded(&id_b).unwrap();
3601
3602        registry
3603            .complete_operation(
3604                &id_a,
3605                OperationResult {
3606                    id: id_a.clone(),
3607                    content: "a-done".into(),
3608                    is_error: false,
3609                    duration_ms: 1,
3610                    tokens_used: 0,
3611                },
3612            )
3613            .unwrap();
3614        registry.fail_operation(&id_b, "b-error".into()).unwrap();
3615
3616        let wait_result = registry
3617            .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3618            .await
3619            .unwrap();
3620        assert_eq!(wait_result.outcomes.len(), 2);
3621        assert_eq!(wait_result.outcomes[0].0, id_a);
3622        assert!(matches!(
3623            wait_result.outcomes[0].1,
3624            OperationTerminalOutcome::Completed(_)
3625        ));
3626        assert_eq!(wait_result.outcomes[1].0, id_b);
3627        assert!(matches!(
3628            wait_result.outcomes[1].1,
3629            OperationTerminalOutcome::Failed { .. }
3630        ));
3631        // Obligation carries the awaited IDs
3632        assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3633        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3634    }
3635
3636    /// Exercises the trait `wait_all` path (via `dyn OpsLifecycleRegistry`)
3637    /// which must submit WaitAll through the DSL for cross-machine handoff.
3638    #[tokio::test]
3639    async fn wait_all_trait_path_submits_through_authority() {
3640        let registry = RuntimeOpsLifecycleRegistry::new();
3641        let spec = background_spec("trait-wait");
3642        let op_id = spec.id.clone();
3643        registry.register_operation(spec).unwrap();
3644        registry.provisioning_succeeded(&op_id).unwrap();
3645        registry
3646            .complete_operation(
3647                &op_id,
3648                OperationResult {
3649                    id: op_id.clone(),
3650                    content: "done".into(),
3651                    is_error: false,
3652                    duration_ms: 1,
3653                    tokens_used: 0,
3654                },
3655            )
3656            .unwrap();
3657
3658        // Call through trait object to exercise the trait impl, not the inherent method.
3659        let trait_ref: &dyn OpsLifecycleRegistry = &registry;
3660        let wait_result = trait_ref
3661            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3662            .await
3663            .unwrap();
3664        assert_eq!(wait_result.outcomes.len(), 1);
3665        assert!(matches!(
3666            wait_result.outcomes[0].1,
3667            OperationTerminalOutcome::Completed(_)
3668        ));
3669        // Obligation carries the validated ID
3670        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3671        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3672        let state = registry.read_state().unwrap();
3673        assert!(
3674            !state.wait_active(),
3675            "already-satisfied wait_all must be cleared by generated satisfaction authority"
3676        );
3677        assert!(state.wait_operation_ids().unwrap().is_empty());
3678    }
3679
3680    #[tokio::test]
3681    async fn wait_all_duplicate_rejection_is_generated() {
3682        let registry = RuntimeOpsLifecycleRegistry::new();
3683        let spec = background_spec("duplicate-wait");
3684        let op_id = spec.id.clone();
3685        registry.register_operation(spec).unwrap();
3686        registry.provisioning_succeeded(&op_id).unwrap();
3687
3688        let result = registry
3689            .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3690            .await;
3691
3692        assert!(matches!(
3693            result,
3694            Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3695        ));
3696        let state = registry.read_state().unwrap();
3697        assert!(
3698            !state.wait_active(),
3699            "duplicate wait rejection must not create a shell or machine barrier"
3700        );
3701        assert!(state.wait_operation_ids().unwrap().is_empty());
3702    }
3703
3704    #[tokio::test]
3705    async fn wait_all_active_rejection_is_generated() {
3706        let registry = RuntimeOpsLifecycleRegistry::new();
3707        let spec = background_spec("active-wait");
3708        let op_id = spec.id.clone();
3709        registry.register_operation(spec).unwrap();
3710        registry.provisioning_succeeded(&op_id).unwrap();
3711
3712        let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3713        let result = registry
3714            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3715            .await;
3716
3717        assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3718        drop(active_wait);
3719        let state = registry.read_state().unwrap();
3720        assert!(!state.wait_active());
3721        assert!(state.wait_operation_ids().unwrap().is_empty());
3722    }
3723
3724    #[tokio::test]
3725    async fn wait_all_unknown_operation_rejection_is_generated() {
3726        let registry = RuntimeOpsLifecycleRegistry::new();
3727        let op_id = OperationId::new();
3728
3729        let result = registry
3730            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3731            .await;
3732
3733        assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3734        let state = registry.read_state().unwrap();
3735        assert!(!state.wait_active());
3736        assert!(state.wait_operation_ids().unwrap().is_empty());
3737    }
3738
3739    #[tokio::test]
3740    async fn wait_all_resolves_from_authority_owned_wait_request() {
3741        let registry = RuntimeOpsLifecycleRegistry::new();
3742        let run_id = test_run_id();
3743
3744        let spec = background_spec("pending");
3745        let op_id = spec.id.clone();
3746        registry.register_operation(spec).unwrap();
3747        registry.provisioning_succeeded(&op_id).unwrap();
3748
3749        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3750        tokio::pin!(wait_fut);
3751        assert!(
3752            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3753                .await
3754                .is_err()
3755        );
3756
3757        let active_wait_request_id = {
3758            let state = registry.read_state().unwrap();
3759            let wait_request_id = match state.wait_request_id.clone() {
3760                Some(wait_request_id) => wait_request_id,
3761                None => panic!("wait request should be active"),
3762            };
3763            assert_eq!(
3764                state.wait_operation_ids().unwrap().as_slice(),
3765                std::slice::from_ref(&op_id)
3766            );
3767            wait_request_id
3768        };
3769
3770        registry
3771            .complete_operation(
3772                &op_id,
3773                OperationResult {
3774                    id: op_id.clone(),
3775                    content: "done".into(),
3776                    is_error: false,
3777                    duration_ms: 1,
3778                    tokens_used: 0,
3779                },
3780            )
3781            .unwrap();
3782
3783        let wait_result = wait_fut.await.unwrap();
3784        assert_eq!(
3785            wait_result.satisfied.wait_request_id,
3786            active_wait_request_id
3787        );
3788        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
3789        assert!(matches!(
3790            wait_result.outcomes.as_slice(),
3791            [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
3792        ));
3793        assert!(registry.read_state().unwrap().wait_request_id.is_none());
3794    }
3795
3796    #[tokio::test]
3797    async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
3798        let registry = RuntimeOpsLifecycleRegistry::new();
3799        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3800        registry.set_persistence_channel(
3801            tx,
3802            meerkat_core::RuntimeEpochId::new(),
3803            Arc::new(meerkat_core::EpochCursorState::new()),
3804        );
3805        let worker = std::thread::spawn(move || {
3806            let request = rx.blocking_recv().expect("persistence request");
3807            let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
3808                "injected persistence failure".to_string(),
3809            )));
3810        });
3811
3812        let spec = background_spec("feed-after-persist");
3813        let op_id = spec.id.clone();
3814        registry.register_operation(spec).unwrap();
3815        registry.provisioning_succeeded(&op_id).unwrap();
3816        let watch = registry.register_watcher(&op_id).unwrap();
3817
3818        let err = registry
3819            .complete_operation(
3820                &op_id,
3821                OperationResult {
3822                    id: op_id.clone(),
3823                    content: "done".into(),
3824                    is_error: false,
3825                    duration_ms: 1,
3826                    tokens_used: 0,
3827                },
3828            )
3829            .expect_err("persistence failure must fail the terminal transition");
3830        assert!(
3831            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
3832            "unexpected persistence error: {err:?}"
3833        );
3834        worker.join().expect("persistence worker");
3835
3836        let feed = registry.completion_feed_handle();
3837        let batch = feed.list_since(0);
3838        assert!(
3839            batch.entries.is_empty(),
3840            "completion feed must not publish entries before durable snapshot success"
3841        );
3842        assert_eq!(batch.watermark, 0);
3843        assert!(
3844            tokio::time::timeout(std::time::Duration::from_millis(10), watch)
3845                .await
3846                .is_err(),
3847            "watchers must not resolve before durable terminal snapshot success"
3848        );
3849        assert_eq!(
3850            registry.snapshot(&op_id).unwrap().unwrap().status,
3851            OperationStatus::Running,
3852            "failed persistence must roll the generated terminal transition back"
3853        );
3854
3855        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3856        registry.set_persistence_channel(
3857            tx,
3858            meerkat_core::RuntimeEpochId::new(),
3859            Arc::new(meerkat_core::EpochCursorState::new()),
3860        );
3861        let worker = std::thread::spawn(move || {
3862            let request = rx.blocking_recv().expect("second persistence request");
3863            let _ = request.result_tx.send(Ok(()));
3864        });
3865        registry
3866            .complete_operation(
3867                &op_id,
3868                OperationResult {
3869                    id: op_id.clone(),
3870                    content: "done".into(),
3871                    is_error: false,
3872                    duration_ms: 1,
3873                    tokens_used: 0,
3874                },
3875            )
3876            .expect("rolled-back terminal transition should remain retryable");
3877        worker.join().expect("second persistence worker");
3878
3879        let batch = feed.list_since(0);
3880        assert_eq!(batch.entries.len(), 1);
3881        assert_eq!(batch.entries[0].operation_id, op_id);
3882    }
3883
3884    #[tokio::test]
3885    async fn dropping_wait_all_future_cancels_active_wait_request() {
3886        let registry = RuntimeOpsLifecycleRegistry::new();
3887        let run_id = test_run_id();
3888
3889        let spec = background_spec("cancelled-wait");
3890        let op_id = spec.id.clone();
3891        registry.register_operation(spec).unwrap();
3892        registry.provisioning_succeeded(&op_id).unwrap();
3893
3894        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3895        drop(wait_fut);
3896
3897        let state = registry.read_state().unwrap();
3898        assert!(state.wait_request_id.is_none());
3899        assert!(state.wait_operation_ids().unwrap().is_empty());
3900        assert!(!state.wait_active());
3901    }
3902
3903    /// id 101: an authority-invariant corruption surfaced during a terminal
3904    /// transition must NOT report the op terminal with a silently-hung barrier.
3905    /// The terminal call must return the typed `Internal` fault, AND the awaited
3906    /// `wait_all` future must resolve to `Err` (via the dropped-sender arm)
3907    /// instead of hanging forever.
3908    #[tokio::test]
3909    async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
3910        let registry = RuntimeOpsLifecycleRegistry::new();
3911        let run_id = test_run_id();
3912
3913        let spec = background_spec("corrupt-barrier");
3914        let op_id = spec.id.clone();
3915        registry.register_operation(spec).unwrap();
3916        registry.provisioning_succeeded(&op_id).unwrap();
3917
3918        // Activate a barrier whose waiter is still pending.
3919        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3920        tokio::pin!(wait_fut);
3921        assert!(
3922            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3923                .await
3924                .is_err(),
3925            "barrier waiter must still be pending before corruption"
3926        );
3927        assert!(registry.read_state().unwrap().wait_request_id.is_some());
3928
3929        // Corrupt the generated wait authority: an active wait_request_id with
3930        // no wait_run_id forces `try_satisfy_wait_all_authority` down its
3931        // non-GuardRejected `Internal` arm during the terminal transition.
3932        {
3933            let mut state = registry.write_state().unwrap();
3934            let mut machine_state = state.dsl.0.state().clone();
3935            machine_state.wait_run_id = None;
3936            state.dsl = DslAuthority(Box::new(
3937                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
3938            ));
3939        }
3940
3941        // The terminal transition must FAIL with the typed authority fault, not
3942        // report the op complete.
3943        let err = registry
3944            .complete_operation(
3945                &op_id,
3946                OperationResult {
3947                    id: op_id.clone(),
3948                    content: "done".into(),
3949                    is_error: false,
3950                    duration_ms: 1,
3951                    tokens_used: 0,
3952                },
3953            )
3954            .expect_err("corrupt wait authority must fail the terminal transition");
3955        assert!(
3956            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
3957            "unexpected terminal error: {err:?}"
3958        );
3959
3960        // The waiter must resolve to Err (dropped sender), not hang.
3961        let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
3962            .await
3963            .expect("waiter must resolve, not hang, after authority corruption");
3964        match waiter_result {
3965            Err(OpsLifecycleError::Internal(message)) => assert!(
3966                message.contains("wait_all completion channel dropped"),
3967                "unexpected waiter error message: {message}"
3968            ),
3969            other => panic!("expected dropped-channel Internal error, got {other:?}"),
3970        }
3971    }
3972
3973    /// id 97: a poisoned registry lock must surface as a typed `Internal` fault
3974    /// from `completion_cursor`, NOT laundered into `Ok(None)` (which means "no
3975    /// generated cursor authority") or a bare `None`.
3976    #[test]
3977    fn completion_cursor_propagates_poison_not_none() {
3978        let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
3979
3980        // Poison the registry RwLock by panicking while holding the write guard.
3981        let poison_registry = std::sync::Arc::clone(&registry);
3982        let join = std::thread::spawn(move || {
3983            let _guard = poison_registry.write_state().unwrap();
3984            panic!("intentional panic to poison ops lifecycle registry lock");
3985        });
3986        assert!(
3987            join.join().is_err(),
3988            "poisoning thread must have panicked while holding the write guard"
3989        );
3990
3991        let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
3992        let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
3993        match result {
3994            Err(OpsLifecycleError::Internal(message)) => assert!(
3995                message.contains("ops lifecycle registry poisoned"),
3996                "unexpected cursor error message: {message}"
3997            ),
3998            other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
3999        }
4000    }
4001
4002    #[test]
4003    fn terminate_owner_only_targets_non_terminal_operations() {
4004        let registry = RuntimeOpsLifecycleRegistry::new();
4005
4006        let running_spec = background_spec("running");
4007        let running_id = running_spec.id.clone();
4008        registry.register_operation(running_spec).unwrap();
4009        registry.provisioning_succeeded(&running_id).unwrap();
4010
4011        let completed_spec = background_spec("completed");
4012        let completed_id = completed_spec.id.clone();
4013        registry.register_operation(completed_spec).unwrap();
4014        registry.provisioning_succeeded(&completed_id).unwrap();
4015        registry
4016            .complete_operation(
4017                &completed_id,
4018                OperationResult {
4019                    id: completed_id.clone(),
4020                    content: "done".into(),
4021                    is_error: false,
4022                    duration_ms: 1,
4023                    tokens_used: 0,
4024                },
4025            )
4026            .unwrap();
4027
4028        registry.terminate_owner("shutdown".into()).unwrap();
4029
4030        assert!(matches!(
4031            registry.snapshot(&running_id).unwrap().unwrap().status,
4032            OperationStatus::Terminated
4033        ));
4034        assert!(matches!(
4035            registry.snapshot(&completed_id).unwrap().unwrap().status,
4036            OperationStatus::Completed
4037        ));
4038    }
4039
4040    #[test]
4041    fn collect_completed_drains_terminal_operations() {
4042        let registry = RuntimeOpsLifecycleRegistry::new();
4043
4044        let spec_a = background_spec("a");
4045        let id_a = spec_a.id.clone();
4046        registry.register_operation(spec_a).unwrap();
4047        registry.provisioning_succeeded(&id_a).unwrap();
4048        registry
4049            .complete_operation(
4050                &id_a,
4051                OperationResult {
4052                    id: id_a.clone(),
4053                    content: "done".into(),
4054                    is_error: false,
4055                    duration_ms: 1,
4056                    tokens_used: 0,
4057                },
4058            )
4059            .unwrap();
4060
4061        let spec_b = background_spec("b");
4062        let id_b = spec_b.id.clone();
4063        registry.register_operation(spec_b).unwrap();
4064
4065        let collected = registry.collect_completed().unwrap();
4066        assert_eq!(collected.len(), 1);
4067        assert_eq!(collected[0].0, id_a);
4068
4069        assert!(registry.snapshot(&id_a).unwrap().is_none());
4070        assert!(registry.snapshot(&id_b).unwrap().is_some());
4071
4072        let collected2 = registry.collect_completed().unwrap();
4073        assert!(collected2.is_empty());
4074    }
4075
4076    #[test]
4077    fn bounded_completed_retention_evicts_oldest() {
4078        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4079            max_completed: 3,
4080            max_concurrent: None,
4081        });
4082
4083        let mut ids = Vec::new();
4084        for i in 0..5 {
4085            let spec = background_spec(&format!("op-{i}"));
4086            let id = spec.id.clone();
4087            registry.register_operation(spec).unwrap();
4088            registry.provisioning_succeeded(&id).unwrap();
4089            registry
4090                .complete_operation(
4091                    &id,
4092                    OperationResult {
4093                        id: id.clone(),
4094                        content: format!("done-{i}"),
4095                        is_error: false,
4096                        duration_ms: 1,
4097                        tokens_used: 0,
4098                    },
4099                )
4100                .unwrap();
4101            ids.push(id);
4102        }
4103
4104        assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4105        assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4106        assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4107        assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4108        assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4109    }
4110
4111    #[test]
4112    fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4113        let registry = RuntimeOpsLifecycleRegistry::new();
4114
4115        let completed_spec = background_spec("completed");
4116        let completed_id = completed_spec.id.clone();
4117        registry.register_operation(completed_spec).unwrap();
4118        registry.provisioning_succeeded(&completed_id).unwrap();
4119        registry
4120            .complete_operation(
4121                &completed_id,
4122                OperationResult {
4123                    id: completed_id.clone(),
4124                    content: "done".into(),
4125                    is_error: false,
4126                    duration_ms: 1,
4127                    tokens_used: 0,
4128                },
4129            )
4130            .unwrap();
4131
4132        let running_spec = background_spec("running");
4133        let running_id = running_spec.id.clone();
4134        registry.register_operation(running_spec).unwrap();
4135        registry.provisioning_succeeded(&running_id).unwrap();
4136
4137        let cursor_state = meerkat_core::EpochCursorState::new();
4138        let snapshot = registry
4139            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4140            .unwrap();
4141        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4142
4143        assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4144        assert!(recovered.snapshot(&running_id).unwrap().is_none());
4145
4146        let collected = recovered.collect_completed().unwrap();
4147        assert_eq!(collected.len(), 1);
4148        assert_eq!(collected[0].0, completed_id);
4149    }
4150
4151    #[test]
4152    fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4153        let registry = RuntimeOpsLifecycleRegistry::new();
4154
4155        let mut spec = background_spec("capacity");
4156        spec.kind = OperationKind::BackgroundToolCapacitySlot;
4157        let operation_id = spec.id.clone();
4158        registry.register_operation(spec).unwrap();
4159        registry.provisioning_succeeded(&operation_id).unwrap();
4160        registry.mark_retired(&operation_id).unwrap();
4161
4162        assert!(registry.snapshot(&operation_id).unwrap().is_none());
4163
4164        let cursor_state = meerkat_core::EpochCursorState::new();
4165        let snapshot = registry
4166            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4167            .unwrap();
4168        assert!(
4169            !snapshot
4170                .authority_state
4171                .operations
4172                .contains_key(&operation_id)
4173        );
4174        assert!(!snapshot.operation_specs.contains_key(&operation_id));
4175        assert!(snapshot.completion_entries.is_empty());
4176
4177        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4178        assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4179    }
4180
4181    #[test]
4182    fn recovered_snapshot_uses_authority_operation_source() {
4183        let registry = RuntimeOpsLifecycleRegistry::new();
4184        let child_session_id = SessionId::new();
4185        let operation_source = OperationSource::session_child(child_session_id.clone());
4186        let spec = OperationSpec {
4187            id: OperationId::new(),
4188            kind: OperationKind::MobMemberChild,
4189            owner_session_id: SessionId::new(),
4190            display_name: "source-recovery".into(),
4191            source_label: "test".into(),
4192            operation_source: Some(operation_source.clone()),
4193            child_session_id: Some(child_session_id),
4194            expect_peer_channel: true,
4195        };
4196        let operation_id = spec.id.clone();
4197
4198        registry.register_operation(spec).unwrap();
4199        registry.provisioning_succeeded(&operation_id).unwrap();
4200        registry.mark_retired(&operation_id).unwrap();
4201
4202        let cursor_state = meerkat_core::EpochCursorState::new();
4203        let mut snapshot = registry
4204            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4205            .unwrap();
4206        assert_eq!(
4207            snapshot
4208                .authority_state
4209                .operations
4210                .get(&operation_id)
4211                .and_then(|state| state.operation_source.as_ref()),
4212            Some(&operation_source)
4213        );
4214
4215        snapshot
4216            .operation_specs
4217            .get_mut(&operation_id)
4218            .expect("persisted spec")
4219            .operation_source = None;
4220        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4221        assert_eq!(
4222            recovered
4223                .snapshot(&operation_id)
4224                .unwrap()
4225                .unwrap()
4226                .operation_source,
4227            Some(operation_source)
4228        );
4229    }
4230
4231    #[test]
4232    fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4233        let registry = RuntimeOpsLifecycleRegistry::new();
4234        let child_session_id = SessionId::new();
4235        let operation_source = OperationSource::session_child(child_session_id.clone());
4236        let spec = OperationSpec {
4237            id: OperationId::new(),
4238            kind: OperationKind::MobMemberChild,
4239            owner_session_id: SessionId::new(),
4240            display_name: "source-drift".into(),
4241            source_label: "test".into(),
4242            operation_source: Some(operation_source),
4243            child_session_id: Some(child_session_id),
4244            expect_peer_channel: true,
4245        };
4246        let operation_id = spec.id.clone();
4247
4248        registry.register_operation(spec).unwrap();
4249        registry.provisioning_succeeded(&operation_id).unwrap();
4250        registry.mark_retired(&operation_id).unwrap();
4251
4252        let cursor_state = meerkat_core::EpochCursorState::new();
4253        let mut snapshot = registry
4254            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4255            .unwrap();
4256        snapshot
4257            .operation_specs
4258            .get_mut(&operation_id)
4259            .expect("persisted spec")
4260            .operation_source = Some(OperationSource::session_child(SessionId::new()));
4261
4262        let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4263            .expect_err("source mirror drift must fail recovery");
4264        assert!(
4265            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4266            "unexpected recovery error: {err:?}"
4267        );
4268    }
4269
4270    #[test]
4271    fn persisted_authority_state_serializes_explicit_no_operation_source() {
4272        let registry = RuntimeOpsLifecycleRegistry::new();
4273
4274        let spec = background_spec("explicit-no-source");
4275        let operation_id = spec.id.clone();
4276        registry.register_operation(spec).unwrap();
4277        registry.provisioning_succeeded(&operation_id).unwrap();
4278
4279        let cursor_state = meerkat_core::EpochCursorState::new();
4280        let snapshot = registry
4281            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4282            .unwrap();
4283        let value = serde_json::to_value(&snapshot).unwrap();
4284        let operations = value
4285            .get("authority_state")
4286            .and_then(|state| state.get("operations"))
4287            .and_then(serde_json::Value::as_object)
4288            .expect("serialized authority operations");
4289        let persisted_state = operations
4290            .values()
4291            .next()
4292            .and_then(serde_json::Value::as_object)
4293            .expect("serialized operation state");
4294
4295        assert!(
4296            persisted_state
4297                .get("operation_source")
4298                .is_some_and(serde_json::Value::is_null),
4299            "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4300        );
4301
4302        let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4303        assert_eq!(
4304            recovered_snapshot
4305                .authority_state
4306                .operations
4307                .get(&operation_id)
4308                .expect("round-tripped operation")
4309                .operation_source,
4310            None
4311        );
4312    }
4313
4314    #[test]
4315    fn persisted_authority_state_rejects_missing_operation_source_fact() {
4316        let registry = RuntimeOpsLifecycleRegistry::new();
4317
4318        let spec = background_spec("missing-source-fact");
4319        registry.register_operation(spec).unwrap();
4320
4321        let cursor_state = meerkat_core::EpochCursorState::new();
4322        let snapshot = registry
4323            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4324            .unwrap();
4325        let mut value = serde_json::to_value(&snapshot).unwrap();
4326        let operations = value
4327            .get_mut("authority_state")
4328            .and_then(|state| state.get_mut("operations"))
4329            .and_then(serde_json::Value::as_object_mut)
4330            .expect("serialized authority operations");
4331        let operation_state = operations
4332            .values_mut()
4333            .next()
4334            .and_then(serde_json::Value::as_object_mut)
4335            .expect("serialized operation state");
4336        assert!(operation_state.remove("operation_source").is_some());
4337
4338        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4339            .expect_err("missing generated source fact must fail recovery snapshot decoding");
4340        assert!(
4341            err.to_string().contains("operation_source"),
4342            "unexpected decode error: {err}"
4343        );
4344    }
4345
4346    #[test]
4347    fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4348        let registry = RuntimeOpsLifecycleRegistry::new();
4349
4350        let spec = background_spec("missing-feed-authority");
4351        let operation_id = spec.id.clone();
4352        registry.register_operation(spec).unwrap();
4353        registry.provisioning_succeeded(&operation_id).unwrap();
4354        registry
4355            .complete_operation(
4356                &operation_id,
4357                OperationResult {
4358                    id: operation_id.clone(),
4359                    content: "done".into(),
4360                    is_error: false,
4361                    duration_ms: 1,
4362                    tokens_used: 0,
4363                },
4364            )
4365            .unwrap();
4366
4367        let cursor_state = meerkat_core::EpochCursorState::new();
4368        let snapshot = registry
4369            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4370            .unwrap();
4371        let mut value = serde_json::to_value(&snapshot).unwrap();
4372        let authority_state = value
4373            .get_mut("authority_state")
4374            .and_then(serde_json::Value::as_object_mut)
4375            .expect("serialized authority state");
4376        assert!(authority_state.remove("completion_feed_entries").is_some());
4377
4378        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4379            .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4380        assert!(
4381            err.to_string().contains("completion_feed_entries"),
4382            "unexpected decode error: {err}"
4383        );
4384    }
4385
4386    #[test]
4387    fn public_child_session_projection_uses_authority_operation_source() {
4388        let registry = RuntimeOpsLifecycleRegistry::new();
4389        let authority_child_session_id = SessionId::new();
4390        let stale_shell_child_session_id = SessionId::new();
4391        let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4392        let spec = OperationSpec {
4393            id: OperationId::new(),
4394            kind: OperationKind::MobMemberChild,
4395            owner_session_id: SessionId::new(),
4396            display_name: "child-projection".into(),
4397            source_label: "test".into(),
4398            operation_source: Some(operation_source),
4399            child_session_id: Some(stale_shell_child_session_id),
4400            expect_peer_channel: true,
4401        };
4402        let operation_id = spec.id.clone();
4403
4404        registry.register_operation(spec).unwrap();
4405
4406        assert_eq!(
4407            registry
4408                .snapshot(&operation_id)
4409                .unwrap()
4410                .unwrap()
4411                .child_session_id,
4412            Some(authority_child_session_id.clone())
4413        );
4414
4415        let cursor_state = meerkat_core::EpochCursorState::new();
4416        let snapshot = registry
4417            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4418            .unwrap();
4419        assert_eq!(
4420            snapshot
4421                .operation_specs
4422                .get(&operation_id)
4423                .expect("persisted spec")
4424                .child_session_id,
4425            Some(authority_child_session_id)
4426        );
4427    }
4428
4429    #[test]
4430    fn generated_terminal_payload_projection_fails_closed() {
4431        let registry = RuntimeOpsLifecycleRegistry::new();
4432
4433        let spec = background_spec("terminal-payload-drift");
4434        let operation_id = spec.id.clone();
4435        registry.register_operation(spec).unwrap();
4436        registry.provisioning_succeeded(&operation_id).unwrap();
4437        registry
4438            .complete_operation(
4439                &operation_id,
4440                OperationResult {
4441                    id: operation_id.clone(),
4442                    content: "done".into(),
4443                    is_error: false,
4444                    duration_ms: 1,
4445                    tokens_used: 0,
4446                },
4447            )
4448            .unwrap();
4449
4450        {
4451            let mut state = registry.write_state().unwrap();
4452            let mut machine_state = state.dsl.0.state().clone();
4453            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4454            machine_state
4455                .op_terminal_payload
4456                .insert(operation_id_key, OperationTerminalOutcome::Retired);
4457            state.dsl = DslAuthority(Box::new(
4458                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4459            ));
4460        }
4461
4462        let err = match registry.register_watcher(&operation_id) {
4463            Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4464            Err(err) => err,
4465        };
4466        assert!(
4467            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4468            "unexpected watcher error: {err:?}"
4469        );
4470        let err = registry
4471            .snapshot(&operation_id)
4472            .expect_err("invalid generated terminal payload must reject public snapshot");
4473        assert!(
4474            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
4475            "unexpected public snapshot error: {err:?}"
4476        );
4477
4478        let cursor_state = meerkat_core::EpochCursorState::new();
4479        let err = match registry
4480            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4481        {
4482            Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
4483            Err(err) => err,
4484        };
4485        assert!(
4486            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4487            "unexpected snapshot error: {err:?}"
4488        );
4489
4490        let err = match registry.collect_completed() {
4491            Ok(_) => panic!("invalid generated terminal payload must reject collection"),
4492            Err(err) => err,
4493        };
4494        assert!(
4495            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4496            "unexpected collection error: {err:?}"
4497        );
4498    }
4499
4500    #[test]
4501    fn generated_terminal_payload_missing_projection_fails_closed() {
4502        let registry = RuntimeOpsLifecycleRegistry::new();
4503
4504        let spec = background_spec("terminal-payload-missing");
4505        let operation_id = spec.id.clone();
4506        registry.register_operation(spec).unwrap();
4507        registry.provisioning_succeeded(&operation_id).unwrap();
4508        registry
4509            .fail_operation(&operation_id, "boom".into())
4510            .unwrap();
4511
4512        {
4513            let mut state = registry.write_state().unwrap();
4514            let mut machine_state = state.dsl.0.state().clone();
4515            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4516            machine_state.op_terminal_payload.remove(&operation_id_key);
4517            state.dsl = DslAuthority(Box::new(
4518                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4519            ));
4520        }
4521
4522        let err = match registry.register_watcher(&operation_id) {
4523            Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
4524            Err(err) => err,
4525        };
4526        assert!(
4527            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4528            "unexpected watcher error: {err:?}"
4529        );
4530        let err = registry
4531            .snapshot(&operation_id)
4532            .expect_err("missing generated terminal payload must reject public snapshot");
4533        assert!(
4534            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4535            "unexpected public snapshot error: {err:?}"
4536        );
4537    }
4538
4539    #[test]
4540    fn generated_terminal_status_without_outcome_fails_closed() {
4541        let registry = RuntimeOpsLifecycleRegistry::new();
4542
4543        let spec = background_spec("terminal-outcome-missing");
4544        let operation_id = spec.id.clone();
4545        registry.register_operation(spec).unwrap();
4546        registry.provisioning_succeeded(&operation_id).unwrap();
4547        registry
4548            .fail_operation(&operation_id, "boom".into())
4549            .unwrap();
4550
4551        {
4552            let mut state = registry.write_state().unwrap();
4553            let mut machine_state = state.dsl.0.state().clone();
4554            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4555            machine_state.op_terminal_outcomes.remove(&operation_id_key);
4556            state.dsl = DslAuthority(Box::new(
4557                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4558            ));
4559        }
4560
4561        let err = registry
4562            .snapshot(&operation_id)
4563            .expect_err("terminal status without outcome must reject public snapshot");
4564        assert!(
4565            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4566            "unexpected public snapshot error: {err:?}"
4567        );
4568
4569        let err = match registry.collect_completed() {
4570            Ok(_) => panic!("terminal status without outcome must reject collection"),
4571            Err(err) => err,
4572        };
4573        assert!(
4574            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4575            "unexpected collection error: {err:?}"
4576        );
4577    }
4578
4579    #[test]
4580    fn generated_operation_source_projection_fails_closed() {
4581        let registry = RuntimeOpsLifecycleRegistry::new();
4582        let child_session_id = SessionId::new();
4583        let operation_source = OperationSource::session_child(child_session_id.clone());
4584        let spec = OperationSpec {
4585            id: OperationId::new(),
4586            kind: OperationKind::MobMemberChild,
4587            owner_session_id: SessionId::new(),
4588            display_name: "source-authority-drift".into(),
4589            source_label: "test".into(),
4590            operation_source: Some(operation_source),
4591            child_session_id: Some(child_session_id),
4592            expect_peer_channel: true,
4593        };
4594        let operation_id = spec.id.clone();
4595
4596        registry.register_operation(spec).unwrap();
4597
4598        {
4599            let mut state = registry.write_state().unwrap();
4600            let mut machine_state = state.dsl.0.state().clone();
4601            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4602            machine_state
4603                .op_sources
4604                .get_mut(&operation_id_key)
4605                .expect("generated operation source")
4606                .session_id = None;
4607            state.dsl = DslAuthority(Box::new(
4608                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4609            ));
4610        }
4611
4612        let err = registry
4613            .snapshot(&operation_id)
4614            .expect_err("invalid generated operation source must reject public snapshot");
4615        assert!(
4616            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4617            "unexpected public snapshot error: {err:?}"
4618        );
4619        let err = registry
4620            .list_operations()
4621            .expect_err("invalid generated operation source must reject public operation list");
4622        assert!(
4623            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4624            "unexpected operation list error: {err:?}"
4625        );
4626
4627        let cursor_state = meerkat_core::EpochCursorState::new();
4628        let err = match registry
4629            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4630        {
4631            Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
4632            Err(err) => err,
4633        };
4634        assert!(
4635            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4636            "unexpected snapshot error: {err:?}"
4637        );
4638    }
4639
4640    #[test]
4641    fn generated_operation_id_projection_fails_closed() {
4642        let registry = RuntimeOpsLifecycleRegistry::new();
4643
4644        {
4645            let mut state = registry.write_state().unwrap();
4646            let mut machine_state = state.dsl.0.state().clone();
4647            machine_state.op_statuses.insert(
4648                "not-json-operation-id".into(),
4649                mm_dsl::OperationStatus::Running,
4650            );
4651            state.dsl = DslAuthority(Box::new(
4652                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4653            ));
4654        }
4655
4656        let err = registry
4657            .list_operations()
4658            .expect_err("invalid generated operation id must reject public operation list");
4659        assert!(
4660            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4661            "unexpected operation list error: {err:?}"
4662        );
4663
4664        let cursor_state = meerkat_core::EpochCursorState::new();
4665        let err = match registry
4666            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4667        {
4668            Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
4669            Err(err) => err,
4670        };
4671        assert!(
4672            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4673            "unexpected persistence snapshot error: {err:?}"
4674        );
4675    }
4676
4677    #[test]
4678    fn generated_missing_kind_projection_fails_closed() {
4679        let registry = RuntimeOpsLifecycleRegistry::new();
4680        let spec = background_spec("missing-kind");
4681        let operation_id = spec.id.clone();
4682        registry.register_operation(spec).unwrap();
4683
4684        {
4685            let mut state = registry.write_state().unwrap();
4686            let mut machine_state = state.dsl.0.state().clone();
4687            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4688            machine_state.op_kinds.remove(&operation_id_key);
4689            state.dsl = DslAuthority(Box::new(
4690                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4691            ));
4692        }
4693
4694        let err = registry
4695            .snapshot(&operation_id)
4696            .expect_err("missing generated kind must reject public snapshot");
4697        assert!(
4698            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4699            "unexpected public snapshot error: {err:?}"
4700        );
4701        let err = registry
4702            .list_operations()
4703            .expect_err("missing generated kind must reject public list");
4704        assert!(
4705            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4706            "unexpected public list error: {err:?}"
4707        );
4708
4709        let cursor_state = meerkat_core::EpochCursorState::new();
4710        let err = registry
4711            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4712            .expect_err("missing generated kind must reject persistence snapshot");
4713        assert!(
4714            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4715            "unexpected persistence snapshot error: {err:?}"
4716        );
4717    }
4718
4719    #[test]
4720    fn generated_missing_status_projection_fails_closed() {
4721        let registry = RuntimeOpsLifecycleRegistry::new();
4722        let spec = background_spec("missing-status");
4723        let operation_id = spec.id.clone();
4724        registry.register_operation(spec).unwrap();
4725
4726        {
4727            let mut state = registry.write_state().unwrap();
4728            let mut machine_state = state.dsl.0.state().clone();
4729            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4730            machine_state.op_statuses.remove(&operation_id_key);
4731            state.dsl = DslAuthority(Box::new(
4732                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4733            ));
4734        }
4735
4736        let err = registry
4737            .snapshot(&operation_id)
4738            .expect_err("missing generated status must reject public snapshot");
4739        assert!(
4740            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4741            "unexpected public snapshot error: {err:?}"
4742        );
4743        let err = registry
4744            .list_operations()
4745            .expect_err("missing generated status must reject public list");
4746        assert!(
4747            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4748            "unexpected public list error: {err:?}"
4749        );
4750        let err = registry
4751            .classify_operation_public_result(&operation_id)
4752            .expect_err("missing generated status must reject public-result classification");
4753        assert!(
4754            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4755            "unexpected public-result error: {err:?}"
4756        );
4757    }
4758
4759    #[test]
4760    fn generated_retiring_public_result_remains_running_until_terminal() {
4761        let registry = RuntimeOpsLifecycleRegistry::new();
4762        let spec = background_spec("retiring-public-result");
4763        let operation_id = spec.id.clone();
4764        registry.register_operation(spec).unwrap();
4765        registry.provisioning_succeeded(&operation_id).unwrap();
4766        registry.request_retire(&operation_id).unwrap();
4767
4768        let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
4769        assert_eq!(snapshot.status, OperationStatus::Retiring);
4770        assert!(snapshot.terminal_outcome.is_none());
4771        assert!(!snapshot.terminal);
4772        assert_eq!(
4773            snapshot.public_result_class,
4774            OperationPublicResultClass::Running
4775        );
4776        assert_eq!(
4777            registry
4778                .classify_operation_public_result(&operation_id)
4779                .unwrap(),
4780            OperationPublicResultClass::Running
4781        );
4782    }
4783
4784    #[test]
4785    fn generated_missing_peer_ready_projection_fails_closed() {
4786        let registry = RuntimeOpsLifecycleRegistry::new();
4787        let spec = background_spec("missing-peer-ready");
4788        let operation_id = spec.id.clone();
4789        registry.register_operation(spec).unwrap();
4790
4791        {
4792            let mut state = registry.write_state().unwrap();
4793            let mut machine_state = state.dsl.0.state().clone();
4794            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4795            machine_state.op_peer_ready.remove(&operation_id_key);
4796            state.dsl = DslAuthority(Box::new(
4797                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4798            ));
4799        }
4800
4801        let err = registry
4802            .snapshot(&operation_id)
4803            .expect_err("missing generated peer-ready fact must reject public snapshot");
4804        assert!(
4805            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4806            "unexpected public snapshot error: {err:?}"
4807        );
4808
4809        let cursor_state = meerkat_core::EpochCursorState::new();
4810        let err = registry
4811            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4812            .expect_err("missing generated peer-ready fact must reject persistence snapshot");
4813        assert!(
4814            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4815            "unexpected persistence snapshot error: {err:?}"
4816        );
4817    }
4818
4819    #[test]
4820    fn generated_missing_progress_count_projection_fails_closed() {
4821        let registry = RuntimeOpsLifecycleRegistry::new();
4822        let spec = background_spec("missing-progress-count");
4823        let operation_id = spec.id.clone();
4824        registry.register_operation(spec).unwrap();
4825
4826        {
4827            let mut state = registry.write_state().unwrap();
4828            let mut machine_state = state.dsl.0.state().clone();
4829            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4830            machine_state.op_progress_counts.remove(&operation_id_key);
4831            state.dsl = DslAuthority(Box::new(
4832                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4833            ));
4834        }
4835
4836        let err = registry
4837            .snapshot(&operation_id)
4838            .expect_err("missing generated progress count must reject public snapshot");
4839        assert!(
4840            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4841            "unexpected public snapshot error: {err:?}"
4842        );
4843
4844        let cursor_state = meerkat_core::EpochCursorState::new();
4845        let err = registry
4846            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4847            .expect_err("missing generated progress count must reject persistence snapshot");
4848        assert!(
4849            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4850            "unexpected persistence snapshot error: {err:?}"
4851        );
4852    }
4853
4854    #[test]
4855    fn generated_terminal_sequence_missing_persistence_fails_closed() {
4856        let registry = RuntimeOpsLifecycleRegistry::new();
4857        let spec = background_spec("terminal-sequence-missing");
4858        let operation_id = spec.id.clone();
4859        registry.register_operation(spec).unwrap();
4860        registry.provisioning_succeeded(&operation_id).unwrap();
4861        registry
4862            .complete_operation(
4863                &operation_id,
4864                OperationResult {
4865                    id: operation_id.clone(),
4866                    content: "done".into(),
4867                    is_error: false,
4868                    duration_ms: 1,
4869                    tokens_used: 0,
4870                },
4871            )
4872            .unwrap();
4873
4874        {
4875            let mut state = registry.write_state().unwrap();
4876            let mut machine_state = state.dsl.0.state().clone();
4877            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4878            machine_state.op_completion_seq.remove(&operation_id_key);
4879            state.dsl = DslAuthority(Box::new(
4880                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4881            ));
4882        }
4883
4884        let cursor_state = meerkat_core::EpochCursorState::new();
4885        let err = registry
4886            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4887            .expect_err("missing generated terminal sequence must reject persistence snapshot");
4888        assert!(
4889            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
4890            "unexpected persistence snapshot error: {err:?}"
4891        );
4892    }
4893
4894    #[test]
4895    fn generated_record_without_shell_projection_fails_closed() {
4896        let registry = RuntimeOpsLifecycleRegistry::new();
4897        let spec = background_spec("missing-shell-record");
4898        let operation_id = spec.id.clone();
4899        registry.register_operation(spec).unwrap();
4900
4901        {
4902            let mut state = registry.write_state().unwrap();
4903            state.records.remove(&operation_id);
4904        }
4905
4906        let err = registry
4907            .snapshot(&operation_id)
4908            .expect_err("generated operation without shell record must reject public snapshot");
4909        assert!(
4910            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
4911            "unexpected public snapshot error: {err:?}"
4912        );
4913        let err = registry
4914            .list_operations()
4915            .expect_err("generated operation without shell record must reject public list");
4916        assert!(
4917            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
4918            "unexpected public list error: {err:?}"
4919        );
4920    }
4921
4922    #[test]
4923    fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
4924        let registry = RuntimeOpsLifecycleRegistry::new();
4925        let child_session_id = SessionId::new();
4926        let operation_source = OperationSource::session_child(child_session_id.clone());
4927        let spec = OperationSpec {
4928            id: OperationId::new(),
4929            kind: OperationKind::MobMemberChild,
4930            owner_session_id: SessionId::new(),
4931            display_name: "child-drift".into(),
4932            source_label: "test".into(),
4933            operation_source: Some(operation_source),
4934            child_session_id: Some(child_session_id.clone()),
4935            expect_peer_channel: true,
4936        };
4937        let operation_id = spec.id.clone();
4938
4939        registry.register_operation(spec).unwrap();
4940        registry.provisioning_succeeded(&operation_id).unwrap();
4941        registry.mark_retired(&operation_id).unwrap();
4942
4943        let cursor_state = meerkat_core::EpochCursorState::new();
4944        let mut snapshot = registry
4945            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4946            .unwrap();
4947        snapshot
4948            .operation_specs
4949            .get_mut(&operation_id)
4950            .expect("persisted spec")
4951            .child_session_id = Some(SessionId::new());
4952
4953        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4954        assert_eq!(
4955            recovered
4956                .snapshot(&operation_id)
4957                .unwrap()
4958                .unwrap()
4959                .child_session_id,
4960            Some(child_session_id)
4961        );
4962    }
4963
4964    #[test]
4965    fn completion_wake_class_is_generated_by_operation_kind() {
4966        let registry = RuntimeOpsLifecycleRegistry::new();
4967        let operation_id = OperationId::new();
4968
4969        assert_eq!(
4970            registry
4971                .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
4972                .unwrap(),
4973            OperationCompletionWakeClass::Wake
4974        );
4975        assert_eq!(
4976            registry
4977                .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
4978                .unwrap(),
4979            OperationCompletionWakeClass::Ignore
4980        );
4981        assert_eq!(
4982            registry
4983                .classify_operation_completion_wake(
4984                    &operation_id,
4985                    OperationKind::BackgroundToolCapacitySlot,
4986                )
4987                .unwrap(),
4988            OperationCompletionWakeClass::Ignore
4989        );
4990    }
4991
4992    #[test]
4993    fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
4994        let registry = RuntimeOpsLifecycleRegistry::new();
4995
4996        let running_spec = background_spec("running");
4997        let running_id = running_spec.id.clone();
4998        registry.register_operation(running_spec.clone()).unwrap();
4999        registry.provisioning_succeeded(&running_id).unwrap();
5000
5001        let cursor_state = meerkat_core::EpochCursorState::new();
5002        let mut snapshot = registry
5003            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5004            .unwrap();
5005        snapshot.completion_entries.push(CompletionEntry {
5006            seq: 1,
5007            operation_id: running_id.clone(),
5008            kind: running_spec.kind,
5009            display_name: running_spec.display_name,
5010            terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5011                id: running_id,
5012                content: "phantom".into(),
5013                is_error: false,
5014                duration_ms: 1,
5015                tokens_used: 0,
5016            }),
5017            completed_at_ms: None,
5018        });
5019
5020        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5021            Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5022            Err(err) => err,
5023        };
5024        assert!(
5025            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5026            "unexpected recovery error: {err:?}"
5027        );
5028    }
5029
5030    #[test]
5031    fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5032        let registry = RuntimeOpsLifecycleRegistry::new();
5033        let phantom_id = OperationId::new();
5034        {
5035            let state = registry.read_state().unwrap();
5036            state.feed_buffer.push(CompletionEntry {
5037                seq: 1,
5038                operation_id: phantom_id.clone(),
5039                kind: OperationKind::BackgroundToolOp,
5040                display_name: "phantom".into(),
5041                terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5042                    id: phantom_id,
5043                    content: "phantom".into(),
5044                    is_error: false,
5045                    duration_ms: 1,
5046                    tokens_used: 0,
5047                }),
5048                completed_at_ms: None,
5049            });
5050        }
5051
5052        let cursor_state = meerkat_core::EpochCursorState::new();
5053        let err = match registry
5054            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5055        {
5056            Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5057            Err(err) => err,
5058        };
5059        assert!(
5060            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5061            "unexpected capture error: {err:?}"
5062        );
5063    }
5064
5065    #[test]
5066    fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5067        let registry = RuntimeOpsLifecycleRegistry::new();
5068
5069        let spec = background_spec("terminal");
5070        let operation_id = spec.id.clone();
5071        registry.register_operation(spec).unwrap();
5072        registry.provisioning_succeeded(&operation_id).unwrap();
5073        registry
5074            .complete_operation(
5075                &operation_id,
5076                OperationResult {
5077                    id: operation_id.clone(),
5078                    content: "done".into(),
5079                    is_error: false,
5080                    duration_ms: 1,
5081                    tokens_used: 0,
5082                },
5083            )
5084            .unwrap();
5085
5086        let cursor_state = meerkat_core::EpochCursorState::new();
5087        let mut snapshot = registry
5088            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5089            .unwrap();
5090        let phantom_id = OperationId::new();
5091        let phantom_result = OperationResult {
5092            id: phantom_id.clone(),
5093            content: "phantom".into(),
5094            is_error: false,
5095            duration_ms: 1,
5096            tokens_used: 0,
5097        };
5098        let phantom_entry = CompletionFeedCanonicalState {
5099            seq: snapshot.authority_state.next_completion_seq + 1,
5100            kind: OperationKind::BackgroundToolOp,
5101            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5102        };
5103        snapshot
5104            .authority_state
5105            .completion_feed_entries
5106            .insert(phantom_id.clone(), phantom_entry);
5107        snapshot.completion_entries.push(CompletionEntry {
5108            seq: snapshot.authority_state.next_completion_seq + 1,
5109            operation_id: phantom_id.clone(),
5110            kind: OperationKind::BackgroundToolOp,
5111            display_name: "phantom".into(),
5112            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5113            completed_at_ms: None,
5114        });
5115
5116        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5117            Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5118            Err(err) => err,
5119        };
5120        assert!(
5121            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5122            "unexpected recovery error: {err:?}"
5123        );
5124    }
5125
5126    #[test]
5127    fn recovered_completed_order_uses_generated_completion_sequences() {
5128        let registry = RuntimeOpsLifecycleRegistry::new();
5129
5130        let spec_a = background_spec("a");
5131        let id_a = spec_a.id.clone();
5132        registry.register_operation(spec_a).unwrap();
5133        registry.provisioning_succeeded(&id_a).unwrap();
5134        registry
5135            .complete_operation(
5136                &id_a,
5137                OperationResult {
5138                    id: id_a.clone(),
5139                    content: "a".into(),
5140                    is_error: false,
5141                    duration_ms: 1,
5142                    tokens_used: 0,
5143                },
5144            )
5145            .unwrap();
5146
5147        let spec_b = background_spec("b");
5148        let id_b = spec_b.id.clone();
5149        registry.register_operation(spec_b).unwrap();
5150        registry.provisioning_succeeded(&id_b).unwrap();
5151        registry
5152            .complete_operation(
5153                &id_b,
5154                OperationResult {
5155                    id: id_b.clone(),
5156                    content: "b".into(),
5157                    is_error: false,
5158                    duration_ms: 1,
5159                    tokens_used: 0,
5160                },
5161            )
5162            .unwrap();
5163
5164        let cursor_state = meerkat_core::EpochCursorState::new();
5165        let mut snapshot = registry
5166            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5167            .unwrap();
5168        snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5169
5170        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5171        let collected = recovered.collect_completed().unwrap();
5172
5173        assert_eq!(collected[0].0, id_a);
5174        assert_eq!(collected[1].0, id_b);
5175    }
5176
5177    #[test]
5178    fn max_concurrent_enforcement() {
5179        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5180            max_completed: DEFAULT_MAX_COMPLETED,
5181            max_concurrent: Some(2),
5182        });
5183
5184        let spec_a = background_spec("a");
5185        let id_a = spec_a.id.clone();
5186        registry.register_operation(spec_a).unwrap();
5187
5188        let spec_b = background_spec("b");
5189        registry.register_operation(spec_b).unwrap();
5190
5191        let spec_c = background_spec("c");
5192        let result = registry.register_operation(spec_c);
5193        assert!(matches!(
5194            result,
5195            Err(OpsLifecycleError::MaxConcurrentExceeded {
5196                limit: 2,
5197                active: 2,
5198            })
5199        ));
5200
5201        registry.provisioning_succeeded(&id_a).unwrap();
5202        registry
5203            .complete_operation(
5204                &id_a,
5205                OperationResult {
5206                    id: id_a.clone(),
5207                    content: "done".into(),
5208                    is_error: false,
5209                    duration_ms: 1,
5210                    tokens_used: 0,
5211                },
5212            )
5213            .unwrap();
5214
5215        let spec_d = background_spec("d");
5216        assert!(registry.register_operation(spec_d).is_ok());
5217    }
5218
5219    #[test]
5220    fn snapshot_includes_timestamps() {
5221        let registry = RuntimeOpsLifecycleRegistry::new();
5222        let spec = background_spec("timed");
5223        let op_id = spec.id.clone();
5224        registry.register_operation(spec).unwrap();
5225
5226        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5227        assert!(snap1.created_at_ms > 0);
5228        assert!(snap1.started_at_ms.is_none());
5229        assert!(snap1.completed_at_ms.is_none());
5230        assert!(snap1.elapsed_ms.is_none());
5231
5232        registry.provisioning_succeeded(&op_id).unwrap();
5233        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5234        assert!(snap2.started_at_ms.is_some());
5235        assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5236
5237        registry
5238            .complete_operation(
5239                &op_id,
5240                OperationResult {
5241                    id: op_id.clone(),
5242                    content: "done".into(),
5243                    is_error: false,
5244                    duration_ms: 1,
5245                    tokens_used: 0,
5246                },
5247            )
5248            .unwrap();
5249        let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5250        assert!(snap3.completed_at_ms.is_some());
5251        assert!(snap3.elapsed_ms.is_some());
5252        assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5253    }
5254
5255    #[test]
5256    fn snapshot_includes_peer_handle() {
5257        let registry = RuntimeOpsLifecycleRegistry::new();
5258        let child_session_id = SessionId::new();
5259        let spec = OperationSpec {
5260            id: OperationId::new(),
5261            kind: OperationKind::MobMemberChild,
5262            owner_session_id: SessionId::new(),
5263            display_name: "peer-test".into(),
5264            source_label: "test".into(),
5265            operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5266            child_session_id: Some(child_session_id),
5267            expect_peer_channel: true,
5268        };
5269        let op_id = spec.id.clone();
5270        registry.register_operation(spec).unwrap();
5271        registry.provisioning_succeeded(&op_id).unwrap();
5272
5273        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5274        assert!(snap1.peer_handle.is_none());
5275
5276        let handle = OperationPeerHandle {
5277            peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5278            trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5279                "member-x",
5280                PeerId::new(),
5281                "inproc://x",
5282            )
5283            .unwrap(),
5284        };
5285        registry.peer_ready(&op_id, handle).unwrap();
5286
5287        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5288        assert_eq!(
5289            snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5290            "member-x"
5291        );
5292    }
5293}