Skip to main content

meerkat_runtime/
ops_lifecycle.rs

1//! In-memory runtime implementation of the shared async-operation lifecycle seam.
2//!
3//! Per-operation canonical lifecycle state lives in the MeerkatMachine DSL
4//! authority (`op_statuses`, `op_terminal_outcomes`, `op_kinds`,
5//! `op_sources`, `op_peer_ready`, `op_progress_counts`, `active_op_count`,
6//! `wait_active`, `wait_operation_ids`). This shell layer owns pure mechanics: watcher
7//! channels, timestamps, peer handles, snapshot assembly, FIFO eviction
8//! bookkeeping, the completion feed buffer, and typed delivery of generated
9//! admission/rejection feedback.
10//!
11//! Per-transition legality ("is `CompleteOp` legal on a `Provisioning` op?")
12//! is NOT owned by the shell — it lives in the DSL's `from_status_valid`
13//! guards on each op-lifecycle transition. The shell's only job on a
14//! `GuardRejected` rejection is to ask the generated rejection resolver for
15//! the public result class.
16
17use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24    CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31    CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32    OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33    OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34    OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35    OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36    WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43// ---------------------------------------------------------------------------
44// Serde-only persisted canonical state shells
45// ---------------------------------------------------------------------------
46//
47// These structures preserve the persisted fact shape of `PersistedOpsSnapshot`.
48// They are pure serde shells — no methods beyond read-only field accessors,
49// no authority behavior. Optional generated facts that can be explicitly
50// absent still serialize as present `null` so recovery can distinguish
51// generated "none" from a missing persisted fact.
52
53fn deserialize_required_operation_source<'de, D>(
54    deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57    D: serde::Deserializer<'de>,
58{
59    <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62/// Canonical per-operation state as captured in persisted snapshots.
63#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65    status: OperationStatus,
66    kind: OperationKind,
67    #[serde(deserialize_with = "deserialize_required_operation_source")]
68    operation_source: Option<OperationSource>,
69    peer_ready: bool,
70    progress_count: u32,
71    watcher_count: u32,
72    terminal_outcome: Option<OperationTerminalOutcome>,
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    completion_sequence: Option<CompletionSeq>,
75    terminal_buffered: bool,
76}
77
78/// Generated-owned public completion feed fact captured from DSL authority.
79#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81    seq: CompletionSeq,
82    kind: OperationKind,
83    terminal_outcome: OperationTerminalOutcome,
84}
85
86/// Canonical registry-level state as captured in persisted snapshots.
87#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89    operations: HashMap<OperationId, OperationCanonicalState>,
90    completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91    completed_order: VecDeque<OperationId>,
92    max_completed: usize,
93    max_concurrent: Option<usize>,
94    active_count: usize,
95    wait_request_id: Option<WaitRequestId>,
96    wait_operation_ids: Vec<OperationId>,
97    next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101    /// Maximum completed operations retained at capture time.
102    pub fn max_completed(&self) -> usize {
103        self.max_completed
104    }
105
106    /// Maximum concurrent non-terminal operations at capture time.
107    pub fn max_concurrent(&self) -> Option<usize> {
108        self.max_concurrent
109    }
110
111    /// Number of operations captured in the snapshot.
112    pub fn operation_count(&self) -> usize {
113        self.operations.len()
114    }
115
116    /// Number of generated-owned public completion feed entries captured.
117    pub fn completion_feed_count(&self) -> usize {
118        self.completion_feed_entries.len()
119    }
120}
121
122// ---------------------------------------------------------------------------
123// Serializable snapshot for persistence
124// ---------------------------------------------------------------------------
125
126/// Serializable snapshot of the ops lifecycle registry state.
127///
128/// Captured on terminal transitions for durable persistence. Contains
129/// canonical state, operation specs, persisted completion feed entries, and
130/// consumer cursor values. Wire format preserved verbatim from legacy
131/// runtime versions for backward compatibility.
132#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134    /// Epoch identity at capture time.
135    pub epoch_id: meerkat_core::RuntimeEpochId,
136    /// Canonical machine-owned state at capture time.
137    pub authority_state: RegistryCanonicalState,
138    /// Per-operation specs for shell record reconstruction.
139    pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140    /// Persisted completion feed projection metadata. Canonical feed truth is
141    /// captured in `authority_state.completion_feed_entries`.
142    pub completion_entries: Vec<CompletionEntry>,
143    /// Consumer cursor snapshot at capture time.
144    pub cursors: meerkat_core::EpochCursorSnapshot,
145}
146
147#[derive(Debug)]
148pub struct OpsLifecyclePersistenceRequest {
149    snapshot: PersistedOpsSnapshot,
150    result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
151}
152
153impl OpsLifecyclePersistenceRequest {
154    pub fn snapshot(&self) -> &PersistedOpsSnapshot {
155        &self.snapshot
156    }
157
158    pub fn complete(self, result: Result<(), OpsLifecycleError>) {
159        let _ = self.result_tx.send(result);
160    }
161}
162
163// ---------------------------------------------------------------------------
164// Concrete completion feed buffer
165// ---------------------------------------------------------------------------
166
167/// Shared inner state of the completion feed buffer.
168///
169/// Protected by the registry's `RwLock<ShellState>` for writes, and by its
170/// own `RwLock` for reads by external consumers (agent boundary, idle wake).
171#[derive(Debug)]
172struct FeedBufferInner {
173    entries: VecDeque<CompletionEntry>,
174    watermark: CompletionSeq,
175    max_retained: usize,
176}
177
178/// Shared completion feed buffer owned by the runtime registry.
179///
180/// The registry writes entries under its own write lock. External consumers
181/// read through the [`RuntimeCompletionFeed`] handle.
182#[derive(Debug)]
183struct FeedBuffer {
184    inner: RwLock<FeedBufferInner>,
185    /// Atomic mirror of watermark for lock-free `watermark()` reads.
186    watermark_atomic: AtomicU64,
187    /// Notifies all waiters when new entries are appended.
188    notify: tokio::sync::Notify,
189}
190
191impl FeedBuffer {
192    fn new(max_retained: usize) -> Self {
193        Self {
194            inner: RwLock::new(FeedBufferInner {
195                entries: VecDeque::new(),
196                watermark: 0,
197                max_retained,
198            }),
199            watermark_atomic: AtomicU64::new(0),
200            notify: tokio::sync::Notify::new(),
201        }
202    }
203
204    fn push(&self, entry: CompletionEntry) {
205        let mut inner = self
206            .inner
207            .write()
208            .unwrap_or_else(std::sync::PoisonError::into_inner);
209        let seq = entry.seq;
210        inner.entries.push_back(entry);
211        inner.watermark = seq;
212
213        // Evict oldest if over capacity.
214        while inner.entries.len() > inner.max_retained {
215            inner.entries.pop_front();
216        }
217
218        drop(inner);
219
220        self.watermark_atomic.store(seq, Ordering::Release);
221        self.notify.notify_waiters();
222    }
223}
224
225/// Read-only handle to the runtime completion feed.
226///
227/// Implements [`CompletionFeed`] for external consumers. Obtained via
228/// [`RuntimeOpsLifecycleRegistry::completion_feed()`].
229#[derive(Debug, Clone)]
230pub struct RuntimeCompletionFeed {
231    buffer: Arc<FeedBuffer>,
232}
233
234impl CompletionFeed for RuntimeCompletionFeed {
235    fn watermark(&self) -> CompletionSeq {
236        self.buffer.watermark_atomic.load(Ordering::Acquire)
237    }
238
239    fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
240        let inner = self
241            .buffer
242            .inner
243            .read()
244            .unwrap_or_else(std::sync::PoisonError::into_inner);
245        let entries: Vec<CompletionEntry> = inner
246            .entries
247            .iter()
248            .filter(|e| e.seq > after_seq)
249            .cloned()
250            .collect();
251        let watermark = inner.watermark;
252        CompletionBatch { entries, watermark }
253    }
254
255    fn wait_for_advance(
256        &self,
257        after_seq: CompletionSeq,
258    ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
259        Box::pin(async move {
260            loop {
261                // Register the waiter BEFORE reading the watermark.
262                // notify_waiters() in push() only wakes already-registered
263                // listeners — if we read first and push() lands between the
264                // read and notified().await, the wake is lost.
265                let notified = self.buffer.notify.notified();
266                let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
267                if current > after_seq {
268                    return current;
269                }
270                notified.await;
271            }
272        })
273    }
274}
275
276// ---------------------------------------------------------------------------
277// Shell-only per-operation record (not part of canonical machine state)
278// ---------------------------------------------------------------------------
279
280#[derive(Debug)]
281struct OperationCompletionNotifier {
282    tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
283}
284
285impl OperationCompletionNotifier {
286    fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
287        Self { tx }
288    }
289
290    fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
291        let _ = self.tx.send(outcome.clone());
292    }
293}
294
295fn operation_completion_watch_from_receiver(
296    rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
297) -> OperationCompletionWatch {
298    Box::pin(async move {
299        rx.await
300            .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
301    })
302}
303
304fn resolved_operation_completion_watch(
305    outcome: OperationTerminalOutcome,
306) -> OperationCompletionWatch {
307    Box::pin(async move { Ok(outcome) })
308}
309
310/// Shell-owned data for a single operation. Canonical lifecycle state lives in
311/// the DSL authority; this struct holds I/O concerns that the DSL has no
312/// knowledge of.
313#[derive(Debug)]
314struct ShellRecord {
315    spec: OperationSpec,
316    peer_handle: Option<OperationPeerHandle>,
317    /// Private waiter plumbing. Notifiers are drained only from
318    /// `finalize_terminal()` after the generated authority has accepted and
319    /// stored a terminal outcome.
320    watchers: Vec<OperationCompletionNotifier>,
321    // Monotonic timestamps for elapsed computation
322    created_at: Instant,
323    started_at: Option<Instant>,
324    completed_at: Option<Instant>,
325    // Wall-clock anchor captured at creation for epoch millis
326    created_at_wall: SystemTime,
327}
328
329#[derive(Debug)]
330struct PendingWaitState {
331    wait_request_id: WaitRequestId,
332    sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
333}
334
335enum WaitAllAuthorityPlan {
336    AlreadySatisfied(WaitAllSatisfied),
337    ActivateBarrier,
338}
339
340#[derive(Debug, Clone, Copy, PartialEq, Eq)]
341enum RecoveredOperationRecordDisposition {
342    Retain,
343    Discard,
344}
345
346impl ShellRecord {
347    fn new(spec: OperationSpec) -> Self {
348        Self {
349            spec,
350            peer_handle: None,
351            watchers: Vec::new(),
352            created_at: Instant::now(),
353            started_at: None,
354            completed_at: None,
355            created_at_wall: SystemTime::now(),
356        }
357    }
358
359    fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
360        wall_anchor
361            .duration_since(UNIX_EPOCH)
362            .map(|d| d.as_millis() as u64)
363            .unwrap_or(0)
364    }
365
366    fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
367        // Compute wall time for a given instant using the wall-clock anchor:
368        // wall_time = created_at_wall + (instant - created_at)
369        let offset = instant.saturating_duration_since(self.created_at);
370        let wall = self.created_at_wall + offset;
371        Self::epoch_millis(&wall)
372    }
373
374    /// Notify all watchers with the given terminal outcome and drain the list.
375    fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
376        for watcher in std::mem::take(&mut self.watchers) {
377            watcher.notify_after_generated_terminal(outcome);
378        }
379    }
380
381    /// Mark the completion timestamp.
382    fn mark_completed(&mut self) {
383        self.completed_at = Some(Instant::now());
384    }
385}
386
387// ---------------------------------------------------------------------------
388// Combined shell state: DSL authority + shell records
389// ---------------------------------------------------------------------------
390
391#[derive(Debug)]
392struct ShellState {
393    /// DSL authority — sole source of truth for per-op canonical state.
394    dsl: DslAuthority,
395    /// Shell-owned per-operation records (specs, watchers, timestamps, peer handles).
396    records: HashMap<OperationId, ShellRecord>,
397    /// Pending wait-all coordination (oneshot channel).
398    pending_wait: Option<PendingWaitState>,
399    /// FIFO ordering of completed operation IDs for bounded eviction.
400    completed_order: VecDeque<OperationId>,
401    /// Maximum completed operations to retain.
402    max_completed: usize,
403    /// Maximum concurrent non-terminal operations (None = unlimited).
404    max_concurrent: Option<usize>,
405    /// Oneshot correlation id for the currently-pending `wait_all` future.
406    ///
407    /// Barrier membership (`wait_operation_ids`) and activation (`wait_active`)
408    /// are DSL-owned. This field is pure transport mechanics — the identity
409    /// the oneshot sender is tagged with so `Drop` can correlate cancellation.
410    wait_request_id: Option<WaitRequestId>,
411    /// Shared feed buffer for completion events.
412    feed_buffer: Arc<FeedBuffer>,
413    /// Persistence channel for durable snapshot writes (set via `set_persistence_channel`).
414    persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
415    /// Epoch ID for persistence snapshots.
416    persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
417    /// Shared cursor state for persistence snapshots.
418    persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
419    /// Mechanical epoch fence closed by canonical owner teardown.
420    ///
421    /// Generated authority terminalizes each known operation. This bit closes
422    /// the shell admission door so detached callbacks that retained an Arc to
423    /// the registry cannot create or mutate operations after unregister.
424    owner_retired: bool,
425}
426
427/// Wrapper around the DSL authority that provides `Debug` output.
428///
429/// The generated `MeerkatMachineAuthority` does not derive `Debug`, but
430/// `ShellState` requires it. This wrapper delegates to the inner state's
431/// `Debug` impl.
432struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
433
434impl std::fmt::Debug for DslAuthority {
435    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
436        f.debug_struct("DslAuthority")
437            .field("state", self.0.state())
438            .finish()
439    }
440}
441
442/// Create a DSL authority initialized through generated authority. Per-op
443/// transitions guard only on `op_statuses.contains_key(operation_id)`, so the
444/// phase stays in `Idle` permanently (they all `to Idle`).
445fn new_ops_dsl_authority() -> DslAuthority {
446    DslAuthority(Box::new(
447        crate::meerkat_machine::dsl_authority::new_initialized_authority(
448            "ops lifecycle DSL Initialize must be accepted",
449        ),
450    ))
451}
452
453impl ShellState {
454    fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
455        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
456        let dsl = new_ops_dsl_authority();
457        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
458        let feed_capacity = max_completed.saturating_mul(4).max(1024);
459        tracing::info!(
460            feed_capacity,
461            "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
462        );
463        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
464        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
465        Self {
466            dsl,
467            records: HashMap::new(),
468            pending_wait: None,
469            completed_order: VecDeque::new(),
470            max_completed,
471            max_concurrent,
472            wait_request_id: None,
473            // Feed buffer is larger than max_completed to absorb bursts.
474            // Entries are only evicted by buffer capacity, not by consumer cursor,
475            // so the buffer must be large enough that consumers drain before
476            // the oldest entry is evicted.
477            feed_buffer,
478            persist_tx: None,
479            persist_epoch_id: None,
480            persist_cursor_state: None,
481            owner_retired: false,
482        }
483    }
484
485    fn ensure_owner_active(&self) -> Result<(), OpsLifecycleError> {
486        if self.owner_retired {
487            Err(OpsLifecycleError::OwnerRetired)
488        } else {
489            Ok(())
490        }
491    }
492
493    /// Apply a DSL input, mapping transition errors into
494    /// [`OpsLifecycleError::Internal`]. Callers that need to distinguish
495    /// guard rejections (legal-transition violations) from internal desync
496    /// should use [`Self::dsl_apply_raw`] and classify the error themselves;
497    /// this helper is for DSL inputs whose preconditions the caller has
498    /// already fully validated (e.g., `RequestWaitAll`, `SatisfyWaitAll`).
499    fn dsl_apply(
500        &mut self,
501        input: mm_dsl::MeerkatMachineInput,
502        context: &str,
503    ) -> Result<(), OpsLifecycleError> {
504        self.dsl_apply_raw(input).map_err(|err| {
505            OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
506        })
507    }
508
509    /// Apply a DSL input, returning the raw kernel-level rejection so callers
510    /// can distinguish `GuardRejected` (a legitimate legality violation, e.g.,
511    /// `complete_operation` on a `Provisioning` op) from
512    /// `NoMatchingTransition` (a shell/DSL desync). Op-lifecycle entry points
513    /// feed guard rejections into generated rejection feedback before surfacing
514    /// public result classes.
515    fn dsl_apply_raw(
516        &mut self,
517        input: mm_dsl::MeerkatMachineInput,
518    ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
519        mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
520    }
521
522    fn dsl_apply_with_effects(
523        &mut self,
524        input: mm_dsl::MeerkatMachineInput,
525        context: &str,
526    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
527        let transition =
528            mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
529                OpsLifecycleError::Internal(format!(
530                    "DSL rejected ops transition ({context}): {err:?}"
531                ))
532            })?;
533        Ok(transition.into_effects())
534    }
535
536    /// Fail-closed read of a typed terminal payload entry: the payload IS the
537    /// domain [`OperationTerminalOutcome`] (K8b fold — no JSON codec), but the
538    /// shell still refuses to surface a payload whose variant disagrees with
539    /// the recorded discriminant.
540    fn checked_terminal_payload(
541        kind: mm_dsl::OperationTerminalOutcomeKind,
542        payload: &OperationTerminalOutcome,
543        authority: &str,
544        operation_id: &str,
545    ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
546        if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
547            return Err(OpsLifecycleError::Internal(format!(
548                "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
549            )));
550        }
551        Ok(payload.clone())
552    }
553
554    /// Read the DSL operation status for `id`, or `None` if not registered.
555    fn status(&self, id: &OperationId) -> Option<OperationStatus> {
556        let id_key = mm_dsl::OperationId::from_domain(id).0;
557        self.dsl
558            .0
559            .state()
560            .op_statuses
561            .get(&id_key)
562            .copied()
563            .map(OperationStatus::from)
564    }
565
566    fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
567        self.status(id).ok_or_else(|| {
568            OpsLifecycleError::Internal(format!(
569                "generated op lifecycle authority missing status for {id}"
570            ))
571        })
572    }
573
574    /// Read the DSL operation kind for `id`, or `None` if not registered.
575    fn kind(&self, id: &OperationId) -> Option<OperationKind> {
576        let id_key = mm_dsl::OperationId::from_domain(id).0;
577        self.dsl
578            .0
579            .state()
580            .op_kinds
581            .get(&id_key)
582            .copied()
583            .map(OperationKind::from)
584    }
585
586    fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
587        self.kind(id).ok_or_else(|| {
588            OpsLifecycleError::Internal(format!(
589                "generated op lifecycle authority missing kind for {id}"
590            ))
591        })
592    }
593
594    fn operation_source(
595        &self,
596        id: &OperationId,
597    ) -> Result<Option<OperationSource>, OpsLifecycleError> {
598        let id_key = mm_dsl::OperationId::from_domain(id).0;
599        self.dsl
600            .0
601            .state()
602            .op_sources
603            .get(&id_key)
604            .map(|source| {
605                source.to_domain().map_err(|error| {
606                    OpsLifecycleError::Internal(format!(
607                        "generated operation source authority has invalid source for {id}: {error}"
608                    ))
609                })
610            })
611            .transpose()
612    }
613
614    fn child_session_id_from_operation_source(
615        operation_source: Option<&OperationSource>,
616    ) -> Option<SessionId> {
617        match operation_source {
618            Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
619            Some(OperationSource::BackendPeer { .. }) | None => None,
620        }
621    }
622
623    fn align_spec_child_session_id_to_source(
624        spec: &mut OperationSpec,
625        operation_source: Option<&OperationSource>,
626    ) {
627        spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
628    }
629
630    /// Read the peer-ready flag for `id`.
631    fn peer_ready(&self, id: &OperationId) -> Option<bool> {
632        let id_key = mm_dsl::OperationId::from_domain(id).0;
633        self.dsl.0.state().op_peer_ready.get(&id_key).copied()
634    }
635
636    fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
637        self.peer_ready(id).ok_or_else(|| {
638            OpsLifecycleError::Internal(format!(
639                "generated op peer wiring authority missing peer-ready fact for {id}"
640            ))
641        })
642    }
643
644    /// Read the progress counter for `id`.
645    fn progress_count(&self, id: &OperationId) -> Option<u32> {
646        let id_key = mm_dsl::OperationId::from_domain(id).0;
647        self.dsl
648            .0
649            .state()
650            .op_progress_counts
651            .get(&id_key)
652            .map(|v| (*v).min(u32::MAX as u64) as u32)
653    }
654
655    fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
656        self.progress_count(id).ok_or_else(|| {
657            OpsLifecycleError::Internal(format!(
658                "generated op progress authority missing progress count for {id}"
659            ))
660        })
661    }
662
663    /// Read the terminal outcome for `id` by pairing the DSL's typed
664    /// discriminant with the companion payload JSON. Returns `None` when the
665    /// op has no recorded terminal discriminant.
666    fn terminal_outcome(
667        &self,
668        id: &OperationId,
669    ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
670        let id_key = mm_dsl::OperationId::from_domain(id).0;
671        let state = self.dsl.0.state();
672        let status = self.status(id);
673        let terminal = match status {
674            Some(status) => Self::operation_status_is_terminal(id, status)?,
675            None => false,
676        };
677        let kind = state.op_terminal_outcomes.get(&id_key).copied();
678        let Some(kind) = kind else {
679            if terminal {
680                return Err(OpsLifecycleError::Internal(format!(
681                    "generated op terminal authority missing terminal outcome for {id}"
682                )));
683            }
684            return Ok(None);
685        };
686        if !terminal {
687            return Err(OpsLifecycleError::Internal(format!(
688                "generated op terminal authority has terminal outcome for non-terminal {id}"
689            )));
690        }
691        let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
692            OpsLifecycleError::Internal(format!(
693                "generated op terminal authority missing terminal payload for {id}"
694            ))
695        })?;
696        Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
697            .map(Some)
698    }
699
700    /// Whether the operation is currently tracked in DSL state.
701    fn contains(&self, id: &OperationId) -> bool {
702        let id_key = mm_dsl::OperationId::from_domain(id).0;
703        self.dsl.0.state().op_statuses.contains_key(&id_key)
704    }
705
706    /// Number of non-terminal operations (derived from DSL state).
707    fn active_count(&self) -> usize {
708        self.dsl.0.state().active_op_count as usize
709    }
710
711    /// Number of operations currently tracked (including terminal).
712    fn operation_count(&self) -> usize {
713        self.dsl.0.state().op_statuses.len()
714    }
715
716    /// Iterate over all tracked operation IDs (DSL keys converted to domain).
717    fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
718        let mut ids = BTreeSet::new();
719        let state = self.dsl.0.state();
720        Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
721        Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
722        Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
723        Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
724        Self::collect_operation_id_keys(
725            &mut ids,
726            "op_progress_counts",
727            state.op_progress_counts.keys(),
728        )?;
729        Self::collect_operation_id_keys(
730            &mut ids,
731            "op_terminal_outcomes",
732            state.op_terminal_outcomes.keys(),
733        )?;
734        Self::collect_operation_id_keys(
735            &mut ids,
736            "op_terminal_payload",
737            state.op_terminal_payload.keys(),
738        )?;
739        Self::collect_operation_id_keys(
740            &mut ids,
741            "op_completion_seq",
742            state.op_completion_seq.keys(),
743        )?;
744        ids.extend(self.records.keys().cloned());
745        Ok(ids.into_iter().collect())
746    }
747
748    fn collect_operation_id_keys<'a, I>(
749        ids: &mut BTreeSet<OperationId>,
750        field: &str,
751        keys: I,
752    ) -> Result<(), OpsLifecycleError>
753    where
754        I: IntoIterator<Item = &'a String>,
755    {
756        for key in keys {
757            let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
758                OpsLifecycleError::Internal(format!(
759                    "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
760                ))
761            })?;
762            ids.insert(id);
763        }
764        Ok(())
765    }
766
767    fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
768        let id_key = mm_dsl::OperationId::from_domain(id).0;
769        let state = self.dsl.0.state();
770        state.op_statuses.contains_key(&id_key)
771            || state.op_kinds.contains_key(&id_key)
772            || state.op_sources.contains_key(&id_key)
773            || state.op_peer_ready.contains_key(&id_key)
774            || state.op_progress_counts.contains_key(&id_key)
775            || state.op_terminal_outcomes.contains_key(&id_key)
776            || state.op_terminal_payload.contains_key(&id_key)
777            || state.op_completion_seq.contains_key(&id_key)
778    }
779
780    /// Read the DSL-minted completion sequence for a terminal operation.
781    fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
782        let id_key = mm_dsl::OperationId::from_domain(id).0;
783        self.dsl.0.state().op_completion_seq.get(&id_key).copied()
784    }
785
786    fn completion_feed_authority_entries(
787        &self,
788    ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
789        let state = self.dsl.0.state();
790        let sequence_keys: BTreeSet<String> =
791            state.completion_feed_sequences.keys().cloned().collect();
792        let companion_domains: [(&str, BTreeSet<String>); 3] = [
793            (
794                "completion_feed_kinds",
795                state.completion_feed_kinds.keys().cloned().collect(),
796            ),
797            (
798                "completion_feed_terminal_outcomes",
799                state
800                    .completion_feed_terminal_outcomes
801                    .keys()
802                    .cloned()
803                    .collect(),
804            ),
805            (
806                "completion_feed_terminal_payload",
807                state
808                    .completion_feed_terminal_payload
809                    .keys()
810                    .cloned()
811                    .collect(),
812            ),
813        ];
814        for (field, keys) in companion_domains {
815            if keys != sequence_keys {
816                return Err(OpsLifecycleError::Internal(format!(
817                    "generated completion feed authority has mismatched {field} domain"
818                )));
819            }
820        }
821
822        let mut entries = HashMap::new();
823        for (id_key, seq) in &state.completion_feed_sequences {
824            if !state.completion_sequence_claims.contains(seq) {
825                return Err(OpsLifecycleError::Internal(format!(
826                    "generated completion feed authority sequence {seq} for {id_key} is not claimed"
827                )));
828            }
829            let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
830                OpsLifecycleError::Internal(format!(
831                    "generated completion feed authority used invalid operation id key {id_key}: {error}"
832                ))
833            })?;
834            let kind = state
835                .completion_feed_kinds
836                .get(id_key)
837                .copied()
838                .map(OperationKind::from)
839                .ok_or_else(|| {
840                    OpsLifecycleError::Internal(format!(
841                        "generated completion feed authority missing kind for {id_key}"
842                    ))
843                })?;
844            let outcome_kind = state
845                .completion_feed_terminal_outcomes
846                .get(id_key)
847                .copied()
848                .ok_or_else(|| {
849                    OpsLifecycleError::Internal(format!(
850                        "generated completion feed authority missing terminal outcome for {id_key}"
851                    ))
852                })?;
853            let payload = state
854                .completion_feed_terminal_payload
855                .get(id_key)
856                .ok_or_else(|| {
857                    OpsLifecycleError::Internal(format!(
858                        "generated completion feed authority missing terminal payload for {id_key}"
859                    ))
860                })?;
861            let terminal_outcome = Self::checked_terminal_payload(
862                outcome_kind,
863                payload,
864                "generated completion feed authority",
865                id_key,
866            )?;
867            entries.insert(
868                operation_id,
869                CompletionFeedCanonicalState {
870                    seq: *seq,
871                    kind,
872                    terminal_outcome,
873                },
874            );
875        }
876        Ok(entries)
877    }
878
879    fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
880        let state = self.dsl.0.state();
881        match consumer {
882            CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
883            CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
884            CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
885        }
886    }
887
888    fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
889        meerkat_core::EpochCursorSnapshot {
890            agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
891            runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
892            runtime_last_injected_seq: self
893                .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
894        }
895    }
896
897    /// Build a snapshot from DSL state + shell record.
898    fn snapshot(
899        &self,
900        id: &OperationId,
901    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
902        let Some(shell) = self.records.get(id) else {
903            if self.has_generated_operation_record_fact(id) {
904                return Err(OpsLifecycleError::Internal(format!(
905                    "generated op lifecycle authority has operation facts without shell projection record for {id}"
906                )));
907            }
908            return Ok(None);
909        };
910        let kind = self.require_kind(id)?;
911        let status = self.require_status(id)?;
912        let terminal = Self::operation_status_is_terminal(id, status)?;
913        let public_result_class = Self::operation_public_result_class(id, status)?;
914        let peer_ready = self.require_peer_ready(id)?;
915        let progress_count = self.require_progress_count(id)?;
916        let operation_source = self.operation_source(id)?;
917        let terminal_outcome = self.terminal_outcome(id)?;
918
919        let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
920        let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
921        let completed_at_ms = shell
922            .completed_at
923            .map(|i| shell.epoch_millis_for_instant(i));
924        let elapsed_ms = shell.completed_at.map(|completed| {
925            completed
926                .saturating_duration_since(shell.created_at)
927                .as_millis() as u64
928        });
929
930        Ok(Some(OperationLifecycleSnapshot {
931            id: shell.spec.id.clone(),
932            kind,
933            display_name: shell.spec.display_name.clone(),
934            child_session_id: Self::child_session_id_from_operation_source(
935                operation_source.as_ref(),
936            ),
937            operation_source,
938            status,
939            terminal,
940            public_result_class,
941            peer_ready,
942            progress_count,
943            watcher_count: shell.watchers.len() as u32,
944            terminal_outcome,
945            peer_handle: shell.peer_handle.clone(),
946            created_at_ms,
947            started_at_ms,
948            completed_at_ms,
949            elapsed_ms,
950        }))
951    }
952
953    /// Emit shell-side mechanics for a terminal transition: notify watchers,
954    /// push generated-authorized CompletionEntry rows, retain in FIFO, evict
955    /// as needed. Called AFTER the DSL transition has already persisted the
956    /// terminal status + outcome.
957    fn finalize_terminal(
958        &mut self,
959        id: &OperationId,
960    ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
961        let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
962            OpsLifecycleError::Internal(format!(
963                "generated op terminal transition did not mint terminal outcome for {id}"
964            ))
965        })?;
966        let kind = self.require_kind(id)?;
967
968        // Notify watchers and mark completion timestamp.
969        if let Some(shell) = self.records.get_mut(id) {
970            shell.notify_watchers(&outcome);
971            shell.mark_completed();
972        }
973
974        if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
975        {
976            self.dsl_apply(
977                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
978                    operation_id: mm_dsl::OperationId::from_domain(id).0,
979                },
980                "CollectCompletedOp",
981            )?;
982            self.records.remove(id);
983            self.completed_order.retain(|queued| queued != id);
984            return Ok(None);
985        }
986
987        let mut completion_entry = None;
988        if Self::operation_completion_feed_class(id, kind)?
989            == mm_dsl::OperationCompletionFeedClass::Emit
990        {
991            let feed_authority = self
992                .completion_feed_authority_entries()?
993                .remove(id)
994                .ok_or_else(|| {
995                    OpsLifecycleError::Internal(format!(
996                        "generated op terminal transition did not mint completion feed authority for {id}"
997                    ))
998                })?;
999            if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
1000                return Err(OpsLifecycleError::Internal(format!(
1001                    "generated completion feed authority drifted from terminal op authority for {id}"
1002                )));
1003            }
1004            let seq = self.completion_sequence(id).ok_or_else(|| {
1005                OpsLifecycleError::Internal(format!(
1006                    "generated op terminal transition did not mint completion sequence for {id}"
1007                ))
1008            })?;
1009            if feed_authority.seq != seq {
1010                return Err(OpsLifecycleError::Internal(format!(
1011                    "generated completion feed authority sequence drifted for {id}"
1012                )));
1013            }
1014            let display_name = self
1015                .records
1016                .get(id)
1017                .map(|r| r.spec.display_name.clone())
1018                .unwrap_or_default();
1019            let completed_at_ms = self
1020                .records
1021                .get(id)
1022                .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1023            completion_entry = Some(CompletionEntry {
1024                seq: feed_authority.seq,
1025                operation_id: id.clone(),
1026                kind: feed_authority.kind,
1027                display_name,
1028                terminal_outcome: feed_authority.terminal_outcome,
1029                completed_at_ms,
1030            });
1031        }
1032
1033        // FIFO retention + eviction.
1034        self.completed_order.push_back(id.clone());
1035        while self.completed_order.len() > self.max_completed {
1036            if let Some(evicted) = self.completed_order.pop_front() {
1037                self.dsl_apply(
1038                    mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1039                        operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1040                    },
1041                    "EvictCompletedOp",
1042                )?;
1043                self.records.remove(&evicted);
1044            }
1045        }
1046
1047        // Satisfy a pending wait request if all its ops are now terminal. On
1048        // authority-invariant corruption this propagates the typed fault (and
1049        // drops the barrier oneshot so the waiter resolves to Err) instead of
1050        // reporting the op terminal with a silently-hung barrier.
1051        self.maybe_satisfy_wait()?;
1052        Ok(completion_entry)
1053    }
1054
1055    fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1056        if let Some(entry) = entry {
1057            self.feed_buffer.push(entry);
1058        }
1059    }
1060
1061    /// Read barrier membership from DSL state (sole owner).
1062    fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1063        self.dsl
1064            .0
1065            .state()
1066            .wait_operation_ids
1067            .iter()
1068            .map(|key| {
1069                serde_json::from_str::<OperationId>(key).map_err(|error| {
1070                    OpsLifecycleError::Internal(format!(
1071                        "generated wait operation identity authority used invalid operation id key {key}: {error}"
1072                    ))
1073                })
1074            })
1075            .collect()
1076    }
1077
1078    /// Whether the DSL has a barrier wait active.
1079    #[cfg(test)]
1080    fn wait_active(&self) -> bool {
1081        self.dsl.0.state().wait_active
1082    }
1083
1084    fn wait_all_satisfied_from_effects(
1085        effects: &[mm_dsl::MeerkatMachineEffect],
1086    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1087        let mut satisfied = None;
1088        for effect in effects {
1089            let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1090                wait_request_id,
1091                run_id,
1092                operation_ids,
1093            } = effect
1094            else {
1095                continue;
1096            };
1097            if satisfied.is_some() {
1098                return Err(OpsLifecycleError::Internal(
1099                    "generated wait_all authority emitted multiple satisfaction effects".into(),
1100                ));
1101            }
1102            let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1103                OpsLifecycleError::Internal(format!(
1104                    "generated wait_all authority emitted invalid wait request id '{}': {err}",
1105                    wait_request_id.0
1106                ))
1107            })?;
1108            let mut ids = Vec::with_capacity(operation_ids.len());
1109            for operation_id in operation_ids {
1110                ids.push(
1111                    serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1112                        OpsLifecycleError::Internal(format!(
1113                            "generated wait_all authority emitted invalid operation id '{}': {err}",
1114                            operation_id.0
1115                        ))
1116                    })?,
1117                );
1118            }
1119            satisfied = Some(WaitAllSatisfied {
1120                wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1121                run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1122                    OpsLifecycleError::Internal(format!(
1123                        "generated wait_all authority emitted invalid run id '{}': {err}",
1124                        run_id.0
1125                    ))
1126                })?),
1127                operation_ids: ids,
1128            });
1129        }
1130        Ok(satisfied)
1131    }
1132
1133    fn parse_wait_all_operation_id(
1134        raw: &str,
1135        context: &str,
1136    ) -> Result<OperationId, OpsLifecycleError> {
1137        serde_json::from_str::<OperationId>(raw).map_err(|err| {
1138            OpsLifecycleError::Internal(format!(
1139                "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1140            ))
1141        })
1142    }
1143
1144    fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1145        let mut seen = HashSet::new();
1146        operation_ids
1147            .iter()
1148            .find(|operation_id| !seen.insert((*operation_id).clone()))
1149            .cloned()
1150    }
1151
1152    fn wait_all_admission_error_from_effects(
1153        wait_request_id: &WaitRequestId,
1154        effects: &[mm_dsl::MeerkatMachineEffect],
1155    ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1156        let mut admission = None;
1157        for effect in effects {
1158            let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1159                wait_request_id: resolved_wait_request_id,
1160                result,
1161                reject_reason,
1162                rejected_operation_id,
1163            } = effect
1164            else {
1165                continue;
1166            };
1167            if admission.is_some() {
1168                return Err(OpsLifecycleError::Internal(
1169                    "generated wait_all authority emitted multiple admission results".into(),
1170                ));
1171            }
1172            let resolved_uuid =
1173                uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1174                    OpsLifecycleError::Internal(format!(
1175                        "generated wait_all authority emitted invalid wait request id '{}': {err}",
1176                        resolved_wait_request_id.0
1177                    ))
1178                })?;
1179            let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1180            if &resolved_wait_request_id != wait_request_id {
1181                return Err(OpsLifecycleError::Internal(format!(
1182                    "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1183                )));
1184            }
1185            admission = Some(match result {
1186                mm_dsl::WaitAllAdmissionResultKind::Accept => {
1187                    if reject_reason.is_some() || rejected_operation_id.is_some() {
1188                        return Err(OpsLifecycleError::Internal(
1189                            "generated wait_all authority accepted with rejection payload".into(),
1190                        ));
1191                    }
1192                    None
1193                }
1194                mm_dsl::WaitAllAdmissionResultKind::Reject => {
1195                    let reason = reject_reason.ok_or_else(|| {
1196                        OpsLifecycleError::Internal(
1197                            "generated wait_all authority rejected without reason".into(),
1198                        )
1199                    })?;
1200                    let error = match reason {
1201                        mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1202                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1203                                OpsLifecycleError::Internal(
1204                                    "generated wait_all authority rejected duplicate without operation id"
1205                                        .into(),
1206                                )
1207                            })?;
1208                            OpsLifecycleError::DuplicateWaitOperation(
1209                                Self::parse_wait_all_operation_id(raw, "duplicate")?,
1210                            )
1211                        }
1212                        mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1213                            if rejected_operation_id.is_some() {
1214                                return Err(OpsLifecycleError::Internal(
1215                                    "generated wait_all authority rejected active wait with operation id"
1216                                        .into(),
1217                                ));
1218                            }
1219                            OpsLifecycleError::WaitAlreadyActive
1220                        }
1221                        mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1222                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1223                                OpsLifecycleError::Internal(
1224                                    "generated wait_all authority rejected missing operation without operation id"
1225                                        .into(),
1226                                )
1227                            })?;
1228                            OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1229                                raw, "missing",
1230                            )?)
1231                        }
1232                    };
1233                    Some(error)
1234                }
1235            });
1236        }
1237        admission.ok_or_else(|| {
1238            OpsLifecycleError::Internal(
1239                "generated wait_all authority emitted no admission result".into(),
1240            )
1241        })
1242    }
1243
1244    fn resolve_wait_all_admission(
1245        &mut self,
1246        wait_request_id: &WaitRequestId,
1247        operation_ids: &[OperationId],
1248        dsl_ids: &BTreeSet<String>,
1249        dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1250        operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1251        operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1252    ) -> Result<(), OpsLifecycleError> {
1253        let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1254            .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1255        let not_found = operation_ids
1256            .iter()
1257            .find(|operation_id| !self.contains(operation_id))
1258            .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1259        let dsl_id_sequence: Vec<String> = operation_ids
1260            .iter()
1261            .map(|id| mm_dsl::OperationId::from_domain(id).0)
1262            .collect();
1263        let effects = self.dsl_apply_with_effects(
1264            mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1265                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1266                operation_id_sequence: dsl_id_sequence,
1267                operation_ids: dsl_ids.clone(),
1268                operation_id_tokens: dsl_id_tokens.clone(),
1269                operation_token_by_id: operation_token_by_id.clone(),
1270                operation_id_by_token: operation_id_by_token.clone(),
1271                duplicate_operation_id: duplicate,
1272                not_found_operation_id: not_found,
1273            },
1274            "ResolveWaitAllAdmission",
1275        )?;
1276        if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1277        {
1278            return Err(error);
1279        }
1280        Ok(())
1281    }
1282
1283    fn try_satisfy_wait_all_authority(
1284        &mut self,
1285    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1286        let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1287            return Ok(None);
1288        };
1289        let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1290            return Err(OpsLifecycleError::Internal(
1291                "generated wait_all authority has active wait without run id".into(),
1292            ));
1293        };
1294        let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1295        let transition = match mm_dsl::MeerkatMachineMutator::apply(
1296            &mut *self.dsl.0,
1297            mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1298                wait_request_id: dsl_wait_request_id,
1299                run_id: dsl_run_id,
1300                operation_id_tokens: dsl_operation_id_tokens,
1301            },
1302        ) {
1303            Ok(transition) => transition,
1304            Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1305            Err(err) => {
1306                return Err(OpsLifecycleError::Internal(format!(
1307                    "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1308                )));
1309            }
1310        };
1311        Self::wait_all_satisfied_from_effects(transition.effects())?
1312            .ok_or_else(|| {
1313                OpsLifecycleError::Internal(
1314                    "generated wait_all authority accepted satisfaction without effect".into(),
1315                )
1316            })
1317            .map(Some)
1318    }
1319
1320    fn begin_wait_all_authority(
1321        &mut self,
1322        run_id: &RunId,
1323        wait_request_id: &WaitRequestId,
1324        operation_ids: &[OperationId],
1325    ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1326        let mut dsl_ids = BTreeSet::new();
1327        let mut dsl_id_tokens = BTreeSet::new();
1328        let mut operation_token_by_id = BTreeMap::new();
1329        let mut operation_id_by_token = BTreeMap::new();
1330        for id in operation_ids {
1331            let token = mm_dsl::OperationId::from_domain(id);
1332            let raw_id = token.0.clone();
1333            dsl_ids.insert(raw_id.clone());
1334            dsl_id_tokens.insert(token.clone());
1335            operation_token_by_id.insert(raw_id.clone(), token.clone());
1336            operation_id_by_token.insert(token, raw_id);
1337        }
1338        self.resolve_wait_all_admission(
1339            wait_request_id,
1340            operation_ids,
1341            &dsl_ids,
1342            &dsl_id_tokens,
1343            &operation_token_by_id,
1344            &operation_id_by_token,
1345        )?;
1346        self.dsl_apply(
1347            mm_dsl::MeerkatMachineInput::RequestWaitAll {
1348                run_id: mm_dsl::RunId::from_domain(run_id),
1349                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1350                operation_id_sequence: operation_ids
1351                    .iter()
1352                    .map(|id| mm_dsl::OperationId::from_domain(id).0)
1353                    .collect(),
1354                operation_ids: dsl_ids,
1355                operation_id_tokens: dsl_id_tokens,
1356                operation_token_by_id,
1357                operation_id_by_token,
1358            },
1359            "RequestWaitAll",
1360        )?;
1361        if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1362            return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1363        }
1364        Ok(WaitAllAuthorityPlan::ActivateBarrier)
1365    }
1366
1367    fn owner_termination_targets(
1368        &self,
1369    ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1370        let mut targets = Vec::new();
1371        for id in self.operation_ids()? {
1372            let status = self.require_status(&id)?;
1373            if !Self::operation_status_is_terminal(&id, status)? {
1374                targets.push((id, status));
1375            }
1376        }
1377        Ok(targets)
1378    }
1379
1380    fn operation_status_is_terminal(
1381        operation_id: &OperationId,
1382        status: OperationStatus,
1383    ) -> Result<bool, OpsLifecycleError> {
1384        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1385        let effects = Self::apply_stateless_classifier(
1386            mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1387                operation_id: operation_id_key.clone(),
1388                status: mm_dsl::OperationStatus::from(status),
1389            },
1390            "ClassifyOperationTerminality",
1391        )?;
1392        let mut terminal = None;
1393        for effect in effects {
1394            match effect {
1395                mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1396                    if operation_id == operation_id_key =>
1397                {
1398                    terminal = Some(true);
1399                }
1400                mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1401                    if operation_id == operation_id_key =>
1402                {
1403                    terminal = Some(false);
1404                }
1405                other => {
1406                    return Err(OpsLifecycleError::Internal(format!(
1407                        "unexpected generated operation terminality effect: {other:?}"
1408                    )));
1409                }
1410            }
1411        }
1412        terminal.ok_or_else(|| {
1413            OpsLifecycleError::Internal(format!(
1414                "generated operation terminality authority emitted no effect for {operation_id}"
1415            ))
1416        })
1417    }
1418
1419    fn operation_public_result_class(
1420        operation_id: &OperationId,
1421        status: OperationStatus,
1422    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1423        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1424        let effects = Self::apply_stateless_classifier(
1425            mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1426                operation_id: operation_id_key.clone(),
1427                status: mm_dsl::OperationStatus::from(status),
1428            },
1429            "ClassifyOperationPublicResult",
1430        )?;
1431        let mut result = None;
1432        for effect in effects {
1433            match effect {
1434                mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1435                    operation_id,
1436                    result: classified,
1437                } if operation_id == operation_id_key => {
1438                    result = Some(OperationPublicResultClass::from(classified));
1439                }
1440                other => {
1441                    return Err(OpsLifecycleError::Internal(format!(
1442                        "unexpected generated operation public-result effect: {other:?}"
1443                    )));
1444                }
1445            }
1446        }
1447        result.ok_or_else(|| {
1448            OpsLifecycleError::Internal(format!(
1449                "generated operation public-result authority emitted no effect for {operation_id}"
1450            ))
1451        })
1452    }
1453
1454    fn operation_transition_rejection_is_idempotent(
1455        operation_id: &OperationId,
1456        action: OperationLifecycleAction,
1457        status: OperationStatus,
1458    ) -> Result<bool, OpsLifecycleError> {
1459        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1460        let action = mm_dsl::OpLifecycleActionKind::from(action);
1461        let status = mm_dsl::OperationStatus::from(status);
1462        let effects = Self::apply_stateless_classifier(
1463            mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1464                operation_id: operation_id_key.clone(),
1465                action,
1466                status,
1467            },
1468            "ClassifyOperationTransitionIdempotence",
1469        )?;
1470        let mut idempotent = None;
1471        for effect in effects {
1472            match effect {
1473                mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1474                    operation_id,
1475                    action: effect_action,
1476                    status: effect_status,
1477                } if operation_id == operation_id_key
1478                    && effect_action == action
1479                    && effect_status == status =>
1480                {
1481                    idempotent = Some(true);
1482                }
1483                mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1484                    operation_id,
1485                    action: effect_action,
1486                    status: effect_status,
1487                } if operation_id == operation_id_key
1488                    && effect_action == action
1489                    && effect_status == status =>
1490                {
1491                    idempotent = Some(false);
1492                }
1493                other => {
1494                    return Err(OpsLifecycleError::Internal(format!(
1495                        "unexpected generated operation transition-idempotence effect: {other:?}"
1496                    )));
1497                }
1498            }
1499        }
1500        idempotent.ok_or_else(|| {
1501            OpsLifecycleError::Internal(format!(
1502                "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1503            ))
1504        })
1505    }
1506
1507    fn operation_completion_feed_class(
1508        operation_id: &OperationId,
1509        kind: OperationKind,
1510    ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1511        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1512        let kind = mm_dsl::OperationKind::from(kind);
1513        let effects = Self::apply_stateless_classifier(
1514            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1515                operation_id: operation_id_key.clone(),
1516                kind,
1517            },
1518            "ClassifyOperationCompletionFeed",
1519        )?;
1520        let mut class = None;
1521        for effect in effects {
1522            match effect {
1523                mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1524                    operation_id,
1525                    result,
1526                } if operation_id == operation_id_key => {
1527                    class = Some(result);
1528                }
1529                other => {
1530                    return Err(OpsLifecycleError::Internal(format!(
1531                        "unexpected generated operation completion-feed effect: {other:?}"
1532                    )));
1533                }
1534            }
1535        }
1536        class.ok_or_else(|| {
1537            OpsLifecycleError::Internal(format!(
1538                "generated operation completion-feed authority emitted no effect for {operation_id}"
1539            ))
1540        })
1541    }
1542
1543    fn operation_completion_wake_class(
1544        operation_id: &OperationId,
1545        kind: OperationKind,
1546    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1547        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1548        let kind = mm_dsl::OperationKind::from(kind);
1549        let effects = Self::apply_stateless_classifier(
1550            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1551                operation_id: operation_id_key.clone(),
1552                kind,
1553            },
1554            "ClassifyOperationCompletionWake",
1555        )?;
1556        let mut class = None;
1557        for effect in effects {
1558            match effect {
1559                mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1560                    operation_id,
1561                    result,
1562                } if operation_id == operation_id_key => {
1563                    class = Some(OperationCompletionWakeClass::from(result));
1564                }
1565                other => {
1566                    return Err(OpsLifecycleError::Internal(format!(
1567                        "unexpected generated operation completion-wake effect: {other:?}"
1568                    )));
1569                }
1570            }
1571        }
1572        class.ok_or_else(|| {
1573            OpsLifecycleError::Internal(format!(
1574                "generated operation completion-wake authority emitted no effect for {operation_id}"
1575            ))
1576        })
1577    }
1578
1579    fn operation_durability_class(
1580        operation_id: &OperationId,
1581        kind: OperationKind,
1582    ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1583        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1584        let kind = mm_dsl::OperationKind::from(kind);
1585        let effects = Self::apply_stateless_classifier(
1586            mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1587                operation_id: operation_id_key.clone(),
1588                kind,
1589            },
1590            "ClassifyOperationDurability",
1591        )?;
1592        let mut class = None;
1593        for effect in effects {
1594            match effect {
1595                mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1596                    operation_id,
1597                    result,
1598                } if operation_id == operation_id_key => {
1599                    class = Some(result);
1600                }
1601                other => {
1602                    return Err(OpsLifecycleError::Internal(format!(
1603                        "unexpected generated operation durability effect: {other:?}"
1604                    )));
1605                }
1606            }
1607        }
1608        class.ok_or_else(|| {
1609            OpsLifecycleError::Internal(format!(
1610                "generated operation durability authority emitted no effect for {operation_id}"
1611            ))
1612        })
1613    }
1614
1615    fn recovered_operation_record_disposition(
1616        operation_id: &OperationId,
1617        status: OperationStatus,
1618        kind: OperationKind,
1619        terminal_outcome_present: bool,
1620        terminal_payload_present: bool,
1621        completion_sequence_present: bool,
1622    ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1623        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1624        let effects = Self::apply_stateless_classifier(
1625            mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1626                operation_id: operation_id_key.clone(),
1627                status: mm_dsl::OperationStatus::from(status),
1628                kind: mm_dsl::OperationKind::from(kind),
1629                terminal_outcome_present,
1630                terminal_payload_present,
1631                completion_sequence_present,
1632            },
1633            "ClassifyRecoveredOperationRecord",
1634        )?;
1635        let mut disposition = None;
1636        for effect in effects {
1637            match effect {
1638                mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1639                    if operation_id == operation_id_key =>
1640                {
1641                    disposition = Some(RecoveredOperationRecordDisposition::Retain);
1642                }
1643                mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1644                    if operation_id == operation_id_key =>
1645                {
1646                    disposition = Some(RecoveredOperationRecordDisposition::Discard);
1647                }
1648                other => {
1649                    return Err(OpsLifecycleError::Internal(format!(
1650                        "unexpected generated recovered-operation classification effect: {other:?}"
1651                    )));
1652                }
1653            }
1654        }
1655        disposition.ok_or_else(|| {
1656            OpsLifecycleError::Internal(format!(
1657                "generated recovered-operation classifier emitted no effect for {operation_id}"
1658            ))
1659        })
1660    }
1661
1662    fn apply_stateless_classifier(
1663        input: mm_dsl::MeerkatMachineInput,
1664        label: &'static str,
1665    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1666        let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1667            "ops stateless classifier Initialize must be accepted",
1668        );
1669        let transition =
1670            mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1671                OpsLifecycleError::Internal(format!(
1672                    "DSL rejected ops transition ({label}): {err:?}"
1673                ))
1674            })?;
1675        Ok(transition.into_effects())
1676    }
1677
1678    /// Check whether a pending barrier wait is now satisfied and resolve it.
1679    ///
1680    /// Barrier membership and the "all members terminal" decision both live
1681    /// in the DSL: `wait_operation_ids` carries the set, `wait_active`
1682    /// signals a pending barrier, and `SatisfyWaitAll`'s
1683    /// `all_members_terminal` guard owns the fixed-point test. The shell
1684    /// echoes the DSL-owned request id and typed operation tokens into
1685    /// `SatisfyWaitAll` so the transition can clear the barrier before
1686    /// rendering the handoff effect. It still fires idempotently on every
1687    /// terminal transition and lets the DSL guard reject early firings as a
1688    /// no-op. On acceptance (transition returns `Ok`), the shell selects the
1689    /// correlated oneshot and delivers the `WaitAllSatisfied` obligation
1690    /// token.
1691    ///
1692    /// `wait_request_id` is the shell-owned oneshot correlation id that
1693    /// selects which sender to notify; when the DSL barrier satisfies
1694    /// without a live correlation (post-recovery, or duplicate resolution),
1695    /// the oneshot simply remains pending. That benign no-correlation case is
1696    /// the only path that leaves the oneshot pending: on authority-invariant
1697    /// corruption the pending sender is dropped so the waiter's
1698    /// `WaitAllFuture` resolves to `Err` and the typed
1699    /// [`OpsLifecycleError`] propagates to the terminal transition caller
1700    /// instead of reporting the op complete with a silently-hung barrier.
1701    fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1702        let satisfied = match self.try_satisfy_wait_all_authority() {
1703            Ok(Some(satisfied)) => satisfied,
1704            Ok(None) => return Ok(()),
1705            Err(err) => {
1706                // Authority-invariant corruption: do NOT report the op complete
1707                // with a silently-pending barrier oneshot. Drop the sender so
1708                // the waiter's `WaitAllFuture` resolves to `Err` via its
1709                // `Poll::Ready(Err(_))` arm (the same mechanism
1710                // `cancel_wait_all_internal` uses), then propagate the typed
1711                // fault. The invariant is already broken, so we do not attempt
1712                // a `CancelWaitAll` rollback on the corrupt machine.
1713                if let Some(pending) = self.pending_wait.take() {
1714                    drop(pending.sender);
1715                }
1716                self.wait_request_id = None;
1717                return Err(err);
1718            }
1719        };
1720        let shell_wait_id = self.wait_request_id.take();
1721        if shell_wait_id
1722            .as_ref()
1723            .is_some_and(|id| id != &satisfied.wait_request_id)
1724        {
1725            tracing::error!(
1726                shell_wait_request_id = ?shell_wait_id,
1727                authority_wait_request_id = %satisfied.wait_request_id,
1728                "generated wait_all authority satisfied a different wait request"
1729            );
1730        }
1731        if let Some(pending) = self.pending_wait.take() {
1732            if pending.wait_request_id == satisfied.wait_request_id {
1733                let _ = pending.sender.send(satisfied);
1734            } else if let Some(shell_wait_id) = shell_wait_id {
1735                tracing::error!(
1736                    shell_wait_request_id = %shell_wait_id,
1737                    pending_wait_request_id = %pending.wait_request_id,
1738                    authority_wait_request_id = %satisfied.wait_request_id,
1739                    "generated wait_all authority satisfied without a matching pending waiter"
1740                );
1741            }
1742        }
1743        Ok(())
1744    }
1745
1746    /// Persist a terminal snapshot if a persistence channel is wired.
1747    ///
1748    /// Called after terminal transitions. Captures authority + entries + cursors under the write
1749    /// lock (caller already holds it), submits a persistence request, and waits for the worker's
1750    /// durable-store result. Returning success only means the snapshot write itself succeeded.
1751    fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1752        let (tx, epoch_id, cursor_state) = match (
1753            &self.persist_tx,
1754            &self.persist_epoch_id,
1755            &self.persist_cursor_state,
1756        ) {
1757            (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1758            _ => return Ok(()),
1759        };
1760
1761        let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1762        let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1763        let request = OpsLifecyclePersistenceRequest {
1764            snapshot,
1765            result_tx,
1766        };
1767
1768        tx.send(request).map_err(|_| {
1769            OpsLifecycleError::Internal(
1770                "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1771                    .into(),
1772            )
1773        })?;
1774        result_rx.recv().map_err(|_| {
1775            OpsLifecycleError::Internal(
1776                "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1777                    .into(),
1778            )
1779        })?
1780    }
1781
1782    /// Capture the full persisted snapshot for the current state.
1783    fn capture_snapshot(
1784        &self,
1785        epoch_id: meerkat_core::RuntimeEpochId,
1786        _cursor_state: &meerkat_core::EpochCursorState,
1787    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1788        let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1789        for op_id in self.operation_ids()? {
1790            let status = self.require_status(&op_id)?;
1791            let kind = self.require_kind(&op_id)?;
1792            if Self::operation_durability_class(&op_id, kind)?
1793                != mm_dsl::OperationDurabilityClass::Retain
1794            {
1795                continue;
1796            }
1797            let peer_ready = self.require_peer_ready(&op_id)?;
1798            let progress_count = self.require_progress_count(&op_id)?;
1799            let operation_source = self.operation_source(&op_id)?;
1800            let terminal_outcome = self.terminal_outcome(&op_id)?;
1801            let completion_sequence = self.completion_sequence(&op_id);
1802            if terminal_outcome.is_some() && completion_sequence.is_none() {
1803                return Err(OpsLifecycleError::Internal(format!(
1804                    "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1805                )));
1806            }
1807            if terminal_outcome.is_none() && completion_sequence.is_some() {
1808                return Err(OpsLifecycleError::Internal(format!(
1809                    "generated op terminal authority has completion sequence for non-terminal {op_id}"
1810                )));
1811            }
1812            let terminal_buffered = terminal_outcome.is_some();
1813            let watcher_count = self
1814                .records
1815                .get(&op_id)
1816                .map(|r| r.watchers.len() as u32)
1817                .unwrap_or(0);
1818            operations.insert(
1819                op_id,
1820                OperationCanonicalState {
1821                    status,
1822                    kind,
1823                    operation_source,
1824                    peer_ready,
1825                    progress_count,
1826                    watcher_count,
1827                    terminal_outcome,
1828                    completion_sequence,
1829                    terminal_buffered,
1830                },
1831            );
1832        }
1833        let operation_specs: HashMap<OperationId, OperationSpec> = self
1834            .records
1835            .iter()
1836            .filter(|(id, _)| operations.contains_key(*id))
1837            .map(|(id, record)| {
1838                let mut spec = record.spec.clone();
1839                let operation_source = operations
1840                    .get(id)
1841                    .and_then(|state| state.operation_source.as_ref());
1842                Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1843                (id.clone(), spec)
1844            })
1845            .collect();
1846        let completed_order: VecDeque<OperationId> = self
1847            .completed_order
1848            .iter()
1849            .filter(|id| operations.contains_key(*id))
1850            .cloned()
1851            .collect();
1852        let active_count = operations
1853            .iter()
1854            .filter(|(id, state)| {
1855                matches!(
1856                    Self::operation_status_is_terminal(id, state.status),
1857                    Ok(false)
1858                )
1859            })
1860            .count();
1861        let authority_completion_entries = self.completion_feed_authority_entries()?;
1862        let published_completion_entries_by_id: HashMap<OperationId, CompletionEntry> = {
1863            let inner = self
1864                .feed_buffer
1865                .inner
1866                .read()
1867                .unwrap_or_else(std::sync::PoisonError::into_inner);
1868            let mut entries = HashMap::new();
1869            for entry in &inner.entries {
1870                if !authority_completion_entries.contains_key(&entry.operation_id) {
1871                    return Err(OpsLifecycleError::Internal(format!(
1872                        "public completion feed projection for {} has no generated authority",
1873                        entry.operation_id
1874                    )));
1875                }
1876                if entries
1877                    .insert(entry.operation_id.clone(), entry.clone())
1878                    .is_some()
1879                {
1880                    return Err(OpsLifecycleError::Internal(format!(
1881                        "public completion feed projection for {} appeared more than once",
1882                        entry.operation_id
1883                    )));
1884                }
1885            }
1886            entries
1887        };
1888        let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
1889            .iter()
1890            .map(|(operation_id, authority_entry)| {
1891                if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
1892                    if projection.seq != authority_entry.seq
1893                        || projection.kind != authority_entry.kind
1894                        || projection.terminal_outcome != authority_entry.terminal_outcome
1895                    {
1896                        return Err(OpsLifecycleError::Internal(format!(
1897                            "public completion feed projection for {operation_id} drifted from generated authority"
1898                        )));
1899                    }
1900                    return Ok(projection.clone());
1901                }
1902
1903                let display_name = self
1904                    .records
1905                    .get(operation_id)
1906                    .map(|record| record.spec.display_name.clone())
1907                    .unwrap_or_default();
1908                let completed_at_ms = self.records.get(operation_id).and_then(|record| {
1909                    record
1910                        .completed_at
1911                        .map(|completed_at| record.epoch_millis_for_instant(completed_at))
1912                });
1913
1914                Ok(CompletionEntry {
1915                    seq: authority_entry.seq,
1916                    operation_id: operation_id.clone(),
1917                    kind: authority_entry.kind,
1918                    display_name,
1919                    terminal_outcome: authority_entry.terminal_outcome.clone(),
1920                    completed_at_ms,
1921                })
1922            })
1923            .collect::<Result<_, _>>()?;
1924        completion_entries.sort_by_key(|entry| entry.seq);
1925
1926        let authority_state = RegistryCanonicalState {
1927            operations,
1928            completion_feed_entries: authority_completion_entries,
1929            completed_order,
1930            max_completed: self.max_completed,
1931            max_concurrent: self.max_concurrent,
1932            active_count,
1933            wait_request_id: self.wait_request_id.clone(),
1934            wait_operation_ids: self.wait_operation_ids()?,
1935            next_completion_seq: self.dsl.0.state().next_completion_seq,
1936        };
1937
1938        Ok(PersistedOpsSnapshot {
1939            epoch_id,
1940            authority_state,
1941            operation_specs,
1942            completion_entries,
1943            cursors: self.completion_cursor_snapshot(),
1944        })
1945    }
1946
1947    fn shell_record_mut(
1948        &mut self,
1949        id: &OperationId,
1950    ) -> Result<&mut ShellRecord, OpsLifecycleError> {
1951        self.records
1952            .get_mut(id)
1953            .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
1954    }
1955
1956    fn collect_wait_outcomes(
1957        &self,
1958        operation_ids: &[OperationId],
1959    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
1960        operation_ids
1961            .iter()
1962            .map(|operation_id| {
1963                let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
1964                    OpsLifecycleError::Internal(format!(
1965                        "wait_all completed without terminal outcome for {operation_id}"
1966                    ))
1967                })?;
1968                Ok((operation_id.clone(), outcome))
1969            })
1970            .collect()
1971    }
1972}
1973
1974impl Default for ShellState {
1975    fn default() -> Self {
1976        Self::new(DEFAULT_MAX_COMPLETED, None)
1977    }
1978}
1979
1980// ---------------------------------------------------------------------------
1981// Public configuration & registry
1982// ---------------------------------------------------------------------------
1983
1984/// Configuration for [`RuntimeOpsLifecycleRegistry`].
1985#[derive(Debug, Clone)]
1986pub struct OpsLifecycleConfig {
1987    /// Maximum number of completed operations to retain (default: 256).
1988    pub max_completed: usize,
1989    /// Maximum concurrent non-terminal operations (None = unlimited).
1990    pub max_concurrent: Option<usize>,
1991}
1992
1993impl Default for OpsLifecycleConfig {
1994    fn default() -> Self {
1995        Self {
1996            max_completed: DEFAULT_MAX_COMPLETED,
1997            max_concurrent: None,
1998        }
1999    }
2000}
2001
2002/// Per-runtime shared registry for async operation lifecycle truth.
2003///
2004/// Per-operation canonical lifecycle state is owned by the DSL authority
2005/// embedded in the shell. This struct manages I/O concerns: watcher
2006/// channels, timestamps, peer handles, snapshot assembly, FIFO eviction,
2007/// and the completion feed buffer.
2008#[derive(Debug)]
2009pub struct RuntimeOpsLifecycleRegistry {
2010    state: RwLock<ShellState>,
2011}
2012
2013#[derive(Debug, Clone)]
2014pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2015    pub operation_count: usize,
2016    pub active_count: usize,
2017    pub wait_request_id: Option<WaitRequestId>,
2018    pub pending_wait_present: bool,
2019    pub pending_wait_request_id: Option<WaitRequestId>,
2020    pub wait_operation_ids: Vec<OperationId>,
2021    pub operations: Vec<OperationLifecycleSnapshot>,
2022}
2023
2024impl Default for RuntimeOpsLifecycleRegistry {
2025    fn default() -> Self {
2026        Self {
2027            state: RwLock::new(ShellState::default()),
2028        }
2029    }
2030}
2031
2032impl RuntimeOpsLifecycleRegistry {
2033    pub fn new() -> Self {
2034        let dsl = new_ops_dsl_authority();
2035        let feed_capacity = DEFAULT_MAX_COMPLETED.saturating_mul(4).max(1024);
2036        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2037        Self {
2038            state: RwLock::new(ShellState {
2039                dsl,
2040                records: HashMap::new(),
2041                pending_wait: None,
2042                completed_order: VecDeque::new(),
2043                max_completed: DEFAULT_MAX_COMPLETED,
2044                max_concurrent: None,
2045                wait_request_id: None,
2046                feed_buffer,
2047                persist_tx: None,
2048                persist_epoch_id: None,
2049                persist_cursor_state: None,
2050                owner_retired: false,
2051            }),
2052        }
2053    }
2054
2055    pub fn with_config(config: OpsLifecycleConfig) -> Self {
2056        Self {
2057            state: RwLock::new(ShellState::new(config.max_completed, config.max_concurrent)),
2058        }
2059    }
2060
2061    fn recover_completion_feed_entry(
2062        shell: &mut ShellState,
2063        operation_id: &OperationId,
2064        entry: &CompletionFeedCanonicalState,
2065    ) -> Result<(), OpsLifecycleError> {
2066        let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2067        let terminal_outcome_kind =
2068            mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2069        let effects = shell.dsl_apply_with_effects(
2070            mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2071                operation_id: expected_operation_id.clone(),
2072                kind: mm_dsl::OperationKind::from(entry.kind),
2073                terminal_outcome: terminal_outcome_kind,
2074                terminal_payload: entry.terminal_outcome.clone(),
2075                completion_sequence: entry.seq,
2076            },
2077            "RecoverCompletionFeedEntry",
2078        )?;
2079        let recovered = effects.iter().find_map(|effect| match effect {
2080            mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2081                operation_id,
2082                seq,
2083                kind,
2084                terminal_outcome,
2085                terminal_payload,
2086            } => Some((
2087                operation_id,
2088                *seq,
2089                OperationKind::from(*kind),
2090                *terminal_outcome,
2091                terminal_payload,
2092            )),
2093            _ => None,
2094        });
2095        let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2096            return Err(OpsLifecycleError::Internal(
2097                "generated completion-feed recovery emitted no recovered entry".into(),
2098            ));
2099        };
2100        if operation_id != &expected_operation_id
2101            || seq != entry.seq
2102            || kind != entry.kind
2103            || terminal_outcome != terminal_outcome_kind
2104            || terminal_payload != &entry.terminal_outcome
2105        {
2106            return Err(OpsLifecycleError::Internal(format!(
2107                "generated completion-feed recovery drifted for {operation_id}"
2108            )));
2109        }
2110        Ok(())
2111    }
2112
2113    /// Wire a persistence channel for durable snapshot writes.
2114    ///
2115    /// After this call, terminal transitions (complete/fail/cancel/abort)
2116    /// capture a snapshot and queue it to the channel. A dedicated
2117    /// persistence task should drain the channel and write to the store.
2118    pub fn set_persistence_channel(
2119        &self,
2120        tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2121        epoch_id: meerkat_core::RuntimeEpochId,
2122        cursor_state: Arc<meerkat_core::EpochCursorState>,
2123    ) {
2124        if let Ok(mut state) = self.state.write() {
2125            if state.owner_retired {
2126                return;
2127            }
2128            state.persist_tx = Some(tx);
2129            state.persist_epoch_id = Some(epoch_id);
2130            state.persist_cursor_state = Some(cursor_state);
2131        }
2132    }
2133
2134    /// Terminalize every live operation, persist those generated transitions,
2135    /// then close both lifecycle admission and the persistence producer.
2136    ///
2137    /// The write lock excludes every callback that retained this registry.
2138    /// Each terminal transition waits for its durable worker acknowledgement;
2139    /// taking the sender afterward therefore leaves no queued write behind.
2140    /// The unregister coordinator must join the owned worker before publishing
2141    /// final lifecycle deletion in the RuntimeStore.
2142    pub(crate) fn retire_owner_for_unregister(
2143        &self,
2144        reason: String,
2145    ) -> Result<(), OpsLifecycleError> {
2146        let closed_sender = {
2147            let mut state = self.write_state()?;
2148            if state.owner_retired {
2149                return Ok(());
2150            }
2151            terminate_owner_locked(&mut state, &reason)?;
2152            state.owner_retired = true;
2153            state.persist_epoch_id = None;
2154            state.persist_cursor_state = None;
2155            state.persist_tx.take()
2156        };
2157        drop(closed_sender);
2158        Ok(())
2159    }
2160
2161    /// Recover from a persisted snapshot.
2162    ///
2163    /// Rebuilds DSL state (stripping non-terminal ops — only terminals
2164    /// survive recovery), creates fresh shell records from specs, and seeds
2165    /// the feed buffer only with completion entries accepted by generated
2166    /// recovery authority.
2167    pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2168        let PersistedOpsSnapshot {
2169            authority_state,
2170            operation_specs,
2171            completion_entries,
2172            cursors,
2173            ..
2174        } = snapshot;
2175        let max_completed = authority_state.max_completed;
2176        let max_concurrent = authority_state.max_concurrent;
2177        let next_completion_seq = authority_state.next_completion_seq;
2178        let authority_completion_entries = authority_state.completion_feed_entries;
2179        let authority_operations = authority_state.operations;
2180        let mut shell = ShellState::new(max_completed, max_concurrent);
2181
2182        // Replay every persisted op through generated recovery authority.
2183        // The transition accepts only terminal records with outcome and
2184        // completion-sequence witnesses. Volatile non-terminal rows are not
2185        // recovered; terminal/corrupt rows must fail closed instead of being
2186        // projected into shell/public feed state.
2187        let mut retained_ids: HashSet<OperationId> = HashSet::new();
2188        for (op_id, op_state) in authority_operations {
2189            let terminal_outcome = op_state
2190                .terminal_outcome
2191                .as_ref()
2192                .map(mm_dsl::OperationTerminalOutcomeKind::from);
2193            let terminal_payload = op_state.terminal_outcome.clone();
2194            let disposition = ShellState::recovered_operation_record_disposition(
2195                &op_id,
2196                op_state.status,
2197                op_state.kind,
2198                terminal_outcome.is_some(),
2199                terminal_payload.is_some(),
2200                op_state.completion_sequence.is_some(),
2201            )?;
2202            if disposition == RecoveredOperationRecordDisposition::Discard {
2203                continue;
2204            }
2205            if let Some(spec_source) = operation_specs
2206                .get(&op_id)
2207                .and_then(|spec| spec.operation_source.as_ref())
2208                && op_state.operation_source.as_ref() != Some(spec_source)
2209            {
2210                return Err(OpsLifecycleError::Internal(format!(
2211                    "persisted operation source mirror for {op_id} drifted from generated authority"
2212                )));
2213            }
2214            let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2215                operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2216                status: mm_dsl::OperationStatus::from(op_state.status),
2217                kind: mm_dsl::OperationKind::from(op_state.kind),
2218                source: op_state
2219                    .operation_source
2220                    .as_ref()
2221                    .map(mm_dsl::OperationSource::from_domain),
2222                peer_ready: op_state.peer_ready,
2223                progress_count: u64::from(op_state.progress_count),
2224                terminal_outcome,
2225                terminal_payload,
2226                completion_sequence: op_state.completion_sequence,
2227            };
2228            shell.dsl_apply(recovery, "RecoverOpRecord")?;
2229            let recovered_seq = shell.completion_sequence(&op_id).ok_or_else(|| {
2230                OpsLifecycleError::Internal(format!(
2231                    "generated op recovery accepted {op_id} without completion sequence"
2232                ))
2233            })?;
2234            if op_state.completion_sequence != Some(recovered_seq) {
2235                return Err(OpsLifecycleError::Internal(format!(
2236                    "generated op recovery completion sequence mismatch for {op_id}"
2237                )));
2238            }
2239            retained_ids.insert(op_id);
2240        }
2241        shell.dsl_apply(
2242            mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2243                next_completion_seq,
2244            },
2245            "RecoverOpsCompletionCursor",
2246        )?;
2247        shell.dsl_apply(
2248            mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2249                agent_applied_cursor: cursors.agent_applied_cursor,
2250                runtime_observed_cursor: cursors.runtime_observed_seq,
2251                runtime_injected_cursor: cursors.runtime_last_injected_seq,
2252            },
2253            "RecoverCompletionConsumerCursors",
2254        )?;
2255
2256        // Rebuild completed_order from generated completion-sequence truth,
2257        // never from the persisted shell ordering mirror.
2258        let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2259            .iter()
2260            .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2261            .collect();
2262        recovered_completed.sort_by_key(|(seq, _)| *seq);
2263        shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2264
2265        // Recover generated-owned feed authority for entries whose operation
2266        // record is no longer retained. Retained records already wrote their
2267        // feed authority through RecoverOpRecord above.
2268        for (operation_id, entry) in &authority_completion_entries {
2269            if !retained_ids.contains(operation_id) {
2270                Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2271            }
2272        }
2273
2274        let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2275        for (operation_id, entry) in &authority_completion_entries {
2276            let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2277                return Err(OpsLifecycleError::Internal(format!(
2278                    "persisted completion feed authority for {operation_id} was not recovered"
2279                )));
2280            };
2281            if recovered_entry != entry {
2282                return Err(OpsLifecycleError::Internal(format!(
2283                    "persisted completion feed authority drifted from generated recovery for {operation_id}"
2284                )));
2285            }
2286        }
2287
2288        // Projection rows may carry display metadata only after generated
2289        // feed authority has decided the operation id, sequence, kind, and
2290        // terminal outcome. Any semantic drift fails closed.
2291        let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2292        for entry in completion_entries {
2293            let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2294                return Err(OpsLifecycleError::Internal(format!(
2295                    "persisted completion feed projection for {} has no generated feed authority",
2296                    entry.operation_id
2297                )));
2298            };
2299            if authority_entry.seq != entry.seq
2300                || authority_entry.kind != entry.kind
2301                || authority_entry.terminal_outcome != entry.terminal_outcome
2302            {
2303                return Err(OpsLifecycleError::Internal(format!(
2304                    "persisted completion feed projection for {} drifted from generated feed authority",
2305                    entry.operation_id
2306                )));
2307            }
2308            projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2309        }
2310
2311        let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2312            canonical_feed_entries.into_iter().collect();
2313        recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2314        for (operation_id, entry) in recovered_entries {
2315            let projection = projection_entries_by_id.get(&operation_id);
2316            let display_name = operation_specs
2317                .get(&operation_id)
2318                .map(|spec| spec.display_name.clone())
2319                .or_else(|| projection.map(|entry| entry.display_name.clone()))
2320                .unwrap_or_default();
2321            let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2322            shell.feed_buffer.push(CompletionEntry {
2323                seq: entry.seq,
2324                operation_id,
2325                kind: entry.kind,
2326                display_name,
2327                terminal_outcome: entry.terminal_outcome,
2328                completed_at_ms,
2329            });
2330        }
2331
2332        // Rebuild shell records from specs (fresh timestamps, no watchers)
2333        // — only for operations still retained in the DSL state.
2334        for (op_id, spec) in operation_specs {
2335            if retained_ids.contains(&op_id) {
2336                let mut spec = spec;
2337                let operation_source = shell.operation_source(&op_id)?;
2338                ShellState::align_spec_child_session_id_to_source(
2339                    &mut spec,
2340                    operation_source.as_ref(),
2341                );
2342                shell.records.insert(
2343                    op_id,
2344                    ShellRecord {
2345                        spec,
2346                        peer_handle: None,
2347                        watchers: Vec::new(),
2348                        created_at: Instant::now(),
2349                        started_at: None,
2350                        completed_at: None,
2351                        created_at_wall: SystemTime::now(),
2352                    },
2353                );
2354            }
2355        }
2356
2357        Ok(Self {
2358            state: RwLock::new(shell),
2359        })
2360    }
2361
2362    /// Capture a serializable snapshot of the current state for persistence.
2363    ///
2364    /// Includes authority state, operation specs, completion entries, and
2365    /// generated completion-consumer cursor values.
2366    pub fn capture_persistence_snapshot(
2367        &self,
2368        epoch_id: meerkat_core::RuntimeEpochId,
2369        cursor_state: &meerkat_core::EpochCursorState,
2370    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2371        let state = self
2372            .state
2373            .read()
2374            .unwrap_or_else(std::sync::PoisonError::into_inner);
2375        state.capture_snapshot(epoch_id, cursor_state)
2376    }
2377
2378    /// Snapshot generated completion-consumer cursor state.
2379    pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2380        let state = self
2381            .state
2382            .read()
2383            .unwrap_or_else(std::sync::PoisonError::into_inner);
2384        state.completion_cursor_snapshot()
2385    }
2386
2387    /// Return a read handle to the completion feed.
2388    pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2389        let state = self
2390            .state
2391            .read()
2392            .unwrap_or_else(std::sync::PoisonError::into_inner);
2393        Arc::new(RuntimeCompletionFeed {
2394            buffer: Arc::clone(&state.feed_buffer),
2395        })
2396    }
2397
2398    /// Capture a stable diagnostic snapshot of the canonical ops lifecycle state.
2399    pub(crate) fn diagnostic_snapshot(
2400        &self,
2401    ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2402        let state = self
2403            .state
2404            .read()
2405            .unwrap_or_else(std::sync::PoisonError::into_inner);
2406        let mut operations = state
2407            .operation_ids()?
2408            .into_iter()
2409            .map(|id| state.snapshot(&id))
2410            .collect::<Result<Vec<_>, _>>()?
2411            .into_iter()
2412            .flatten()
2413            .collect::<Vec<_>>();
2414        operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2415        Ok(RuntimeOpsDiagnosticSnapshot {
2416            operation_count: state.operation_count(),
2417            active_count: state.active_count(),
2418            wait_request_id: state.wait_request_id.clone(),
2419            pending_wait_present: state.pending_wait.is_some(),
2420            pending_wait_request_id: state
2421                .pending_wait
2422                .as_ref()
2423                .map(|pending_wait| pending_wait.wait_request_id.clone()),
2424            wait_operation_ids: state.wait_operation_ids()?,
2425            operations,
2426        })
2427    }
2428
2429    fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2430        self.state
2431            .read()
2432            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2433    }
2434
2435    fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2436        self.state
2437            .write()
2438            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2439    }
2440
2441    fn cancel_wait_all_internal(
2442        &self,
2443        wait_request_id: &WaitRequestId,
2444    ) -> Result<(), OpsLifecycleError> {
2445        let mut state = self.write_state()?;
2446        match state.wait_request_id.as_ref() {
2447            Some(active) if active == wait_request_id => {
2448                // Clear the DSL barrier via the dedicated `CancelWaitAll`
2449                // transition. Unlike `SatisfyWaitAll`, it does not require
2450                // every member to be terminal (the request was dropped, not
2451                // resolved) and does not emit the `WaitAllSatisfied`.
2452                state.dsl_apply(
2453                    mm_dsl::MeerkatMachineInput::CancelWaitAll,
2454                    "CancelWaitAll(cancel)",
2455                )?;
2456                state.wait_request_id = None;
2457                if state
2458                    .pending_wait
2459                    .as_ref()
2460                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2461                {
2462                    state.pending_wait = None;
2463                }
2464                Ok(())
2465            }
2466            _ => {
2467                if state
2468                    .pending_wait
2469                    .as_ref()
2470                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2471                {
2472                    state.pending_wait = None;
2473                }
2474                Ok(())
2475            }
2476        }
2477    }
2478}
2479
2480enum WaitAllFutureState {
2481    Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2482    Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2483    Done,
2484}
2485
2486struct WaitAllFuture<'a> {
2487    registry: &'a RuntimeOpsLifecycleRegistry,
2488    wait_request_id: WaitRequestId,
2489    state: WaitAllFutureState,
2490}
2491
2492impl Future for WaitAllFuture<'_> {
2493    type Output = Result<WaitAllResult, OpsLifecycleError>;
2494
2495    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2496        match &mut self.state {
2497            WaitAllFutureState::Ready(result) => {
2498                let ready = result.take().unwrap_or_else(|| {
2499                    Err(OpsLifecycleError::Internal(
2500                        "wait_all future polled after completion".into(),
2501                    ))
2502                });
2503                self.state = WaitAllFutureState::Done;
2504                Poll::Ready(ready)
2505            }
2506            WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2507                Poll::Pending => Poll::Pending,
2508                Poll::Ready(Ok(satisfied)) => {
2509                    let outcomes = match self.registry.read_state() {
2510                        Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2511                        Err(err) => Err(err),
2512                    };
2513                    self.state = WaitAllFutureState::Done;
2514                    Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2515                        outcomes,
2516                        satisfied,
2517                    }))
2518                }
2519                Poll::Ready(Err(_)) => {
2520                    self.state = WaitAllFutureState::Done;
2521                    Poll::Ready(Err(OpsLifecycleError::Internal(
2522                        "wait_all completion channel dropped".into(),
2523                    )))
2524                }
2525            },
2526            WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2527                "wait_all future polled after completion".into(),
2528            ))),
2529        }
2530    }
2531}
2532
2533impl Drop for WaitAllFuture<'_> {
2534    fn drop(&mut self) {
2535        if matches!(self.state, WaitAllFutureState::Waiting(_))
2536            && let Err(err) = self
2537                .registry
2538                .cancel_wait_all_internal(&self.wait_request_id)
2539        {
2540            tracing::error!(
2541                wait_request_id = %self.wait_request_id,
2542                error = %err,
2543                "generated wait_all authority rejected cancellation during drop"
2544            );
2545        }
2546    }
2547}
2548
2549// ---------------------------------------------------------------------------
2550// Generated op lifecycle result-class feedback
2551// ---------------------------------------------------------------------------
2552
2553fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2554    match action {
2555        mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2556        mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2557        mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2558        mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2559        mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2560        mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2561        mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2562        mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2563        mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2564        mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2565    }
2566}
2567
2568fn op_lifecycle_rejection_error_from_effects(
2569    id: &OperationId,
2570    requested_action: mm_dsl::OpLifecycleActionKind,
2571    effects: &[mm_dsl::MeerkatMachineEffect],
2572) -> Result<OpsLifecycleError, OpsLifecycleError> {
2573    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2574    let mut rejection = None;
2575    for effect in effects {
2576        let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2577            operation_id,
2578            action,
2579            reason,
2580            status,
2581        } = effect
2582        else {
2583            continue;
2584        };
2585        if rejection.is_some() {
2586            return Err(OpsLifecycleError::Internal(
2587                "generated op lifecycle authority emitted multiple rejection results".into(),
2588            ));
2589        }
2590        if operation_id != &expected_id || *action != requested_action {
2591            return Err(OpsLifecycleError::Internal(format!(
2592                "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2593            )));
2594        }
2595        let error = match reason {
2596            mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2597                if status.is_some() {
2598                    return Err(OpsLifecycleError::Internal(
2599                        "generated op lifecycle authority emitted not-found with status".into(),
2600                    ));
2601                }
2602                OpsLifecycleError::NotFound(id.clone())
2603            }
2604            mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2605                let status = status.ok_or_else(|| {
2606                    OpsLifecycleError::Internal(
2607                        "generated op lifecycle authority emitted invalid-transition without status"
2608                            .into(),
2609                    )
2610                })?;
2611                OpsLifecycleError::InvalidTransition {
2612                    id: id.clone(),
2613                    status: OperationStatus::from(status),
2614                    action: op_lifecycle_action_label(requested_action),
2615                }
2616            }
2617            mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2618                if status.is_none() {
2619                    return Err(OpsLifecycleError::Internal(
2620                        "generated op lifecycle authority emitted peer-not-expected without status"
2621                            .into(),
2622                    ));
2623                }
2624                OpsLifecycleError::PeerNotExpected(id.clone())
2625            }
2626            mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2627                if status.is_none() {
2628                    return Err(OpsLifecycleError::Internal(
2629                        "generated op lifecycle authority emitted already-peer-ready without status"
2630                            .into(),
2631                    ));
2632                }
2633                OpsLifecycleError::AlreadyPeerReady(id.clone())
2634            }
2635        };
2636        rejection = Some(error);
2637    }
2638    rejection.ok_or_else(|| {
2639        OpsLifecycleError::Internal(
2640            "generated op lifecycle authority emitted no rejection result".into(),
2641        )
2642    })
2643}
2644
2645fn classify_generated_op_rejection(
2646    state: &mut ShellState,
2647    err: mm_dsl::MeerkatMachineTransitionError,
2648    id: &OperationId,
2649    action: mm_dsl::OpLifecycleActionKind,
2650) -> OpsLifecycleError {
2651    match err {
2652        mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2653            match state.dsl_apply_with_effects(
2654                mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2655                    operation_id: mm_dsl::OperationId::from_domain(id).0,
2656                    action,
2657                },
2658                "ResolveOpLifecycleTransitionRejection",
2659            ) {
2660                Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2661                    .unwrap_or_else(|err| err),
2662                Err(err) => err,
2663            }
2664        }
2665        other => OpsLifecycleError::Internal(format!(
2666            "DSL rejected ops transition ({}): {other:?}",
2667            op_lifecycle_action_label(action)
2668        )),
2669    }
2670}
2671
2672fn apply_op_transition(
2673    state: &mut ShellState,
2674    id: &OperationId,
2675    input: mm_dsl::MeerkatMachineInput,
2676    action: mm_dsl::OpLifecycleActionKind,
2677) -> Result<(), OpsLifecycleError> {
2678    state.ensure_owner_active()?;
2679    state
2680        .dsl_apply_raw(input)
2681        .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2682}
2683
2684fn apply_terminal_op_transition_and_persist(
2685    state: &mut ShellState,
2686    id: &OperationId,
2687    input: mm_dsl::MeerkatMachineInput,
2688    action: mm_dsl::OpLifecycleActionKind,
2689) -> Result<(), OpsLifecycleError> {
2690    let previous_snapshot = state.dsl.0.snapshot();
2691    apply_op_transition(state, id, input, action)?;
2692    if let Err(err) = state.maybe_persist() {
2693        state.dsl.0.restore_snapshot(previous_snapshot);
2694        return Err(err);
2695    }
2696    Ok(())
2697}
2698
2699fn terminate_owner_locked(state: &mut ShellState, reason: &str) -> Result<(), OpsLifecycleError> {
2700    state.ensure_owner_active()?;
2701    let to_terminate = state.owner_termination_targets()?;
2702
2703    for (op_id, _status) in &to_terminate {
2704        let terminal_outcome = OperationTerminalOutcome::Terminated {
2705            reason: reason.to_owned(),
2706        };
2707        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2708
2709        apply_terminal_op_transition_and_persist(
2710            state,
2711            op_id,
2712            mm_dsl::MeerkatMachineInput::TerminateOp {
2713                operation_id: mm_dsl::OperationId::from_domain(op_id).0,
2714                outcome: outcome_kind,
2715                payload: terminal_outcome,
2716            },
2717            mm_dsl::OpLifecycleActionKind::Terminate,
2718        )?;
2719
2720        if let Some(entry) = state.finalize_terminal(op_id)? {
2721            state.publish_completion_entry(Some(entry));
2722        }
2723    }
2724    Ok(())
2725}
2726
2727fn op_registration_error_from_effects(
2728    id: &OperationId,
2729    effects: &[mm_dsl::MeerkatMachineEffect],
2730) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2731    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2732    let mut admission = None;
2733    for effect in effects {
2734        let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2735            operation_id,
2736            result,
2737            reject_reason,
2738            max_concurrent_limit,
2739            active_op_count,
2740        } = effect
2741        else {
2742            continue;
2743        };
2744        if admission.is_some() {
2745            return Err(OpsLifecycleError::Internal(
2746                "generated op registration authority emitted multiple admission results".into(),
2747            ));
2748        }
2749        if operation_id != &expected_id {
2750            return Err(OpsLifecycleError::Internal(format!(
2751                "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2752            )));
2753        }
2754        admission = Some(match result {
2755            mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2756                if reject_reason.is_some() {
2757                    return Err(OpsLifecycleError::Internal(
2758                        "generated op registration authority accepted with rejection reason".into(),
2759                    ));
2760                }
2761                None
2762            }
2763            mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2764                let reason = reject_reason.ok_or_else(|| {
2765                    OpsLifecycleError::Internal(
2766                        "generated op registration authority rejected without reason".into(),
2767                    )
2768                })?;
2769                let error = match reason {
2770                    mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2771                        OpsLifecycleError::AlreadyRegistered(id.clone())
2772                    }
2773                    mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2774                        let limit = max_concurrent_limit.ok_or_else(|| {
2775                            OpsLifecycleError::Internal(
2776                                "generated op registration authority rejected capacity without limit"
2777                                    .into(),
2778                            )
2779                        })?;
2780                        OpsLifecycleError::MaxConcurrentExceeded {
2781                            limit: limit as usize,
2782                            active: *active_op_count as usize,
2783                        }
2784                    }
2785                };
2786                Some(error)
2787            }
2788        });
2789    }
2790    admission.ok_or_else(|| {
2791        OpsLifecycleError::Internal(
2792            "generated op registration authority emitted no admission result".into(),
2793        )
2794    })
2795}
2796
2797impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2798    fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2799        self.register_operation_with_admission_limit(spec, None)
2800    }
2801
2802    fn register_operation_with_admission_limit(
2803        &self,
2804        mut spec: OperationSpec,
2805        max_concurrent: Option<usize>,
2806    ) -> Result<(), OpsLifecycleError> {
2807        let mut state = self.write_state()?;
2808        state.ensure_owner_active()?;
2809        let operation_id = spec.id.clone();
2810        let kind = spec.kind;
2811        let max_concurrent = max_concurrent
2812            .or(state.max_concurrent)
2813            .map(|limit| limit as u64);
2814
2815        let effects = state.dsl_apply_with_effects(
2816            mm_dsl::MeerkatMachineInput::RegisterOp {
2817                operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2818                kind: mm_dsl::OperationKind::from_domain(&kind),
2819                source: spec
2820                    .operation_source
2821                    .as_ref()
2822                    .map(mm_dsl::OperationSource::from_domain),
2823                max_concurrent,
2824            },
2825            "RegisterOp",
2826        )?;
2827        if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
2828            return Err(error);
2829        }
2830
2831        let authority_operation_source = state.operation_source(&operation_id)?;
2832        ShellState::align_spec_child_session_id_to_source(
2833            &mut spec,
2834            authority_operation_source.as_ref(),
2835        );
2836
2837        // Insert shell record.
2838        state.records.insert(operation_id, ShellRecord::new(spec));
2839        Ok(())
2840    }
2841
2842    fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
2843        let mut state = self.write_state()?;
2844
2845        apply_op_transition(
2846            &mut state,
2847            id,
2848            mm_dsl::MeerkatMachineInput::StartOp {
2849                operation_id: mm_dsl::OperationId::from_domain(id).0,
2850            },
2851            mm_dsl::OpLifecycleActionKind::Start,
2852        )?;
2853
2854        // Shell concern: record the started timestamp.
2855        if let Some(shell) = state.records.get_mut(id) {
2856            shell.started_at = Some(Instant::now());
2857        }
2858        Ok(())
2859    }
2860
2861    fn provisioning_failed(
2862        &self,
2863        id: &OperationId,
2864        error: String,
2865    ) -> Result<(), OpsLifecycleError> {
2866        let mut state = self.write_state()?;
2867
2868        let terminal_outcome = OperationTerminalOutcome::Failed { error };
2869        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2870
2871        apply_terminal_op_transition_and_persist(
2872            &mut state,
2873            id,
2874            mm_dsl::MeerkatMachineInput::FailOp {
2875                operation_id: mm_dsl::OperationId::from_domain(id).0,
2876                outcome: outcome_kind,
2877                payload: terminal_outcome,
2878            },
2879            mm_dsl::OpLifecycleActionKind::Fail,
2880        )?;
2881
2882        let completion_entry = state.finalize_terminal(id)?;
2883        state.publish_completion_entry(completion_entry);
2884        Ok(())
2885    }
2886
2887    fn peer_ready(
2888        &self,
2889        id: &OperationId,
2890        peer: OperationPeerHandle,
2891    ) -> Result<(), OpsLifecycleError> {
2892        let mut state = self.write_state()?;
2893
2894        apply_op_transition(
2895            &mut state,
2896            id,
2897            mm_dsl::MeerkatMachineInput::PeerReadyOp {
2898                operation_id: mm_dsl::OperationId::from_domain(id).0,
2899            },
2900            mm_dsl::OpLifecycleActionKind::PeerReady,
2901        )?;
2902
2903        // Shell concern: store the peer handle.
2904        if let Some(shell) = state.records.get_mut(id) {
2905            shell.peer_handle = Some(peer);
2906        }
2907        Ok(())
2908    }
2909
2910    fn register_watcher(
2911        &self,
2912        id: &OperationId,
2913    ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
2914        let mut state = self.write_state()?;
2915
2916        if !state.contains(id) {
2917            return Err(OpsLifecycleError::NotFound(id.clone()));
2918        }
2919
2920        // If already terminal, return an already-resolved watch.
2921        if let Some(outcome) = state.terminal_outcome(id)? {
2922            return Ok(resolved_operation_completion_watch(outcome));
2923        }
2924
2925        // Shell concern: create the channel and store the sender.
2926        let shell = state.shell_record_mut(id)?;
2927        let (tx, rx) = tokio::sync::oneshot::channel();
2928        let watch = operation_completion_watch_from_receiver(rx);
2929        shell.watchers.push(OperationCompletionNotifier::new(tx));
2930        Ok(watch)
2931    }
2932
2933    fn report_progress(
2934        &self,
2935        id: &OperationId,
2936        _update: OperationProgressUpdate,
2937    ) -> Result<(), OpsLifecycleError> {
2938        let mut state = self.write_state()?;
2939
2940        apply_op_transition(
2941            &mut state,
2942            id,
2943            mm_dsl::MeerkatMachineInput::ProgressReportedOp {
2944                operation_id: mm_dsl::OperationId::from_domain(id).0,
2945            },
2946            mm_dsl::OpLifecycleActionKind::ProgressReported,
2947        )?;
2948        Ok(())
2949    }
2950
2951    fn complete_operation(
2952        &self,
2953        id: &OperationId,
2954        result: OperationResult,
2955    ) -> Result<(), OpsLifecycleError> {
2956        let mut state = self.write_state()?;
2957
2958        let terminal_outcome = OperationTerminalOutcome::Completed(result);
2959        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2960
2961        apply_terminal_op_transition_and_persist(
2962            &mut state,
2963            id,
2964            mm_dsl::MeerkatMachineInput::CompleteOp {
2965                operation_id: mm_dsl::OperationId::from_domain(id).0,
2966                outcome: outcome_kind,
2967                payload: terminal_outcome,
2968            },
2969            mm_dsl::OpLifecycleActionKind::Complete,
2970        )?;
2971
2972        let completion_entry = state.finalize_terminal(id)?;
2973        state.publish_completion_entry(completion_entry);
2974        Ok(())
2975    }
2976
2977    fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
2978        let mut state = self.write_state()?;
2979
2980        let terminal_outcome = OperationTerminalOutcome::Failed { error };
2981        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2982
2983        apply_terminal_op_transition_and_persist(
2984            &mut state,
2985            id,
2986            mm_dsl::MeerkatMachineInput::FailOp {
2987                operation_id: mm_dsl::OperationId::from_domain(id).0,
2988                outcome: outcome_kind,
2989                payload: terminal_outcome,
2990            },
2991            mm_dsl::OpLifecycleActionKind::Fail,
2992        )?;
2993
2994        let completion_entry = state.finalize_terminal(id)?;
2995        state.publish_completion_entry(completion_entry);
2996        Ok(())
2997    }
2998
2999    fn abort_provisioning(
3000        &self,
3001        id: &OperationId,
3002        reason: Option<String>,
3003    ) -> Result<(), OpsLifecycleError> {
3004        let mut state = self.write_state()?;
3005
3006        let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
3007        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3008
3009        apply_terminal_op_transition_and_persist(
3010            &mut state,
3011            id,
3012            mm_dsl::MeerkatMachineInput::AbortOp {
3013                operation_id: mm_dsl::OperationId::from_domain(id).0,
3014                outcome: outcome_kind,
3015                payload: terminal_outcome,
3016            },
3017            mm_dsl::OpLifecycleActionKind::Abort,
3018        )?;
3019
3020        let completion_entry = state.finalize_terminal(id)?;
3021        state.publish_completion_entry(completion_entry);
3022        Ok(())
3023    }
3024
3025    fn cancel_operation(
3026        &self,
3027        id: &OperationId,
3028        reason: Option<String>,
3029    ) -> Result<(), OpsLifecycleError> {
3030        let mut state = self.write_state()?;
3031
3032        let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
3033        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3034
3035        apply_terminal_op_transition_and_persist(
3036            &mut state,
3037            id,
3038            mm_dsl::MeerkatMachineInput::CancelOp {
3039                operation_id: mm_dsl::OperationId::from_domain(id).0,
3040                outcome: outcome_kind,
3041                payload: terminal_outcome,
3042            },
3043            mm_dsl::OpLifecycleActionKind::Cancel,
3044        )?;
3045
3046        let completion_entry = state.finalize_terminal(id)?;
3047        state.publish_completion_entry(completion_entry);
3048        Ok(())
3049    }
3050
3051    fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3052        let mut state = self.write_state()?;
3053
3054        apply_op_transition(
3055            &mut state,
3056            id,
3057            mm_dsl::MeerkatMachineInput::RetireRequestedOp {
3058                operation_id: mm_dsl::OperationId::from_domain(id).0,
3059            },
3060            mm_dsl::OpLifecycleActionKind::RetireRequested,
3061        )?;
3062        Ok(())
3063    }
3064
3065    fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3066        let mut state = self.write_state()?;
3067
3068        let terminal_outcome = OperationTerminalOutcome::Retired;
3069        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3070
3071        apply_terminal_op_transition_and_persist(
3072            &mut state,
3073            id,
3074            mm_dsl::MeerkatMachineInput::RetireCompletedOp {
3075                operation_id: mm_dsl::OperationId::from_domain(id).0,
3076                outcome: outcome_kind,
3077                payload: terminal_outcome,
3078            },
3079            mm_dsl::OpLifecycleActionKind::RetireCompleted,
3080        )?;
3081
3082        let completion_entry = state.finalize_terminal(id)?;
3083        state.publish_completion_entry(completion_entry);
3084        Ok(())
3085    }
3086
3087    fn snapshot(
3088        &self,
3089        id: &OperationId,
3090    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3091        let state = self.read_state()?;
3092        state.snapshot(id)
3093    }
3094
3095    fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3096        let state = self.read_state()?;
3097        let mut snapshots = Vec::new();
3098        for id in state.operation_ids()? {
3099            let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3100                OpsLifecycleError::Internal(format!(
3101                    "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3102                ))
3103            })?;
3104            snapshots.push(snapshot);
3105        }
3106        snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3107        Ok(snapshots)
3108    }
3109
3110    fn classify_operation_terminality(
3111        &self,
3112        id: &OperationId,
3113        status: OperationStatus,
3114    ) -> Result<bool, OpsLifecycleError> {
3115        ShellState::operation_status_is_terminal(id, status)
3116    }
3117
3118    fn classify_operation_public_result(
3119        &self,
3120        id: &OperationId,
3121    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3122        let state = self.read_state()?;
3123        let status = match state.status(id) {
3124            Some(status) => status,
3125            None if state.records.contains_key(id)
3126                || state.has_generated_operation_record_fact(id) =>
3127            {
3128                return Err(OpsLifecycleError::Internal(format!(
3129                    "generated op lifecycle authority missing status for {id}"
3130                )));
3131            }
3132            None => OperationStatus::Absent,
3133        };
3134        ShellState::operation_public_result_class(id, status)
3135    }
3136
3137    fn classify_operation_completion_wake(
3138        &self,
3139        id: &OperationId,
3140        kind: OperationKind,
3141    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3142        ShellState::operation_completion_wake_class(id, kind)
3143    }
3144
3145    fn classify_operation_transition_idempotence(
3146        &self,
3147        id: &OperationId,
3148        action: OperationLifecycleAction,
3149    ) -> Result<bool, OpsLifecycleError> {
3150        let state = self.read_state()?;
3151        let status = match state.status(id) {
3152            Some(status) => status,
3153            None if state.records.contains_key(id)
3154                || state.has_generated_operation_record_fact(id) =>
3155            {
3156                return Err(OpsLifecycleError::Internal(format!(
3157                    "generated op lifecycle authority missing status for {id}"
3158                )));
3159            }
3160            None => OperationStatus::Absent,
3161        };
3162        ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3163    }
3164
3165    fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3166        let mut state = self.write_state()?;
3167        if state.owner_retired {
3168            return Ok(());
3169        }
3170        terminate_owner_locked(&mut state, &reason)
3171    }
3172
3173    fn collect_completed(
3174        &self,
3175    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3176        let mut state = self.write_state()?;
3177        state.ensure_owner_active()?;
3178
3179        let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3180        let mut collected = Vec::with_capacity(ids.len());
3181        for id in ids {
3182            let outcome = state.terminal_outcome(&id)?;
3183            state.dsl_apply(
3184                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3185                    operation_id: mm_dsl::OperationId::from_domain(&id).0,
3186                },
3187                "CollectCompletedOp",
3188            )?;
3189            state.completed_order.retain(|queued| queued != &id);
3190            state.records.remove(&id);
3191            if let Some(outcome) = outcome {
3192                collected.push((id, outcome));
3193            }
3194        }
3195        Ok(collected)
3196    }
3197
3198    fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3199        Some(self.completion_feed_handle())
3200    }
3201
3202    fn completion_cursor(
3203        &self,
3204        consumer: CompletionCursorConsumer,
3205    ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3206        let state = self.read_state()?;
3207        Ok(Some(state.completion_cursor(consumer)))
3208    }
3209
3210    fn advance_completion_cursor(
3211        &self,
3212        consumer: CompletionCursorConsumer,
3213        cursor: CompletionSeq,
3214        projection: Option<&meerkat_core::EpochCursorState>,
3215    ) -> Result<CompletionSeq, OpsLifecycleError> {
3216        let mut state = self.write_state()?;
3217        state.ensure_owner_active()?;
3218        let input = match consumer {
3219            CompletionCursorConsumer::AgentApplied => {
3220                mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3221            }
3222            CompletionCursorConsumer::RuntimeObserved => {
3223                mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3224            }
3225            CompletionCursorConsumer::RuntimeInjected => {
3226                mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3227            }
3228        };
3229        let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3230        let advanced = effects
3231            .iter()
3232            .find_map(|effect| match (consumer, effect) {
3233                (
3234                    CompletionCursorConsumer::AgentApplied,
3235                    mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3236                ) => Some(*cursor),
3237                (
3238                    CompletionCursorConsumer::RuntimeObserved,
3239                    mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3240                        cursor,
3241                    },
3242                ) => Some(*cursor),
3243                (
3244                    CompletionCursorConsumer::RuntimeInjected,
3245                    mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3246                        cursor,
3247                    },
3248                ) => Some(*cursor),
3249                _ => None,
3250            })
3251            .ok_or_else(|| {
3252                OpsLifecycleError::Internal(format!(
3253                    "generated completion cursor transition emitted no feedback for {consumer:?}"
3254                ))
3255            })?;
3256        if let Some(projection) = projection {
3257            projection.project_authorized_completion_cursor(consumer, advanced);
3258        }
3259        Ok(advanced)
3260    }
3261
3262    fn wait_all(
3263        &self,
3264        run_id: &RunId,
3265        ids: &[OperationId],
3266    ) -> std::pin::Pin<
3267        Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3268    > {
3269        let wait_request_id = WaitRequestId::new();
3270        let owned_ids = ids.to_vec();
3271
3272        let state = match self.write_state() {
3273            Ok(mut state) => {
3274                if let Err(error) = state.ensure_owner_active() {
3275                    return Box::pin(WaitAllFuture {
3276                        registry: self,
3277                        wait_request_id,
3278                        state: WaitAllFutureState::Ready(Some(Err(error))),
3279                    });
3280                }
3281                match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3282                    Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3283                        let outcomes =
3284                            state
3285                                .collect_wait_outcomes(&satisfied.operation_ids)
3286                                .map(|outcomes| WaitAllResult {
3287                                    outcomes,
3288                                    satisfied,
3289                                });
3290                        WaitAllFutureState::Ready(Some(outcomes))
3291                    }
3292                    Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3293                        if state.pending_wait.is_some() {
3294                            // Roll back the DSL barrier we just activated so the
3295                            // registry is not stuck in a wait-active state with
3296                            // no correlation oneshot to resolve. `CancelWaitAll`
3297                            // is the no-obligation clearer (members need not be
3298                            // terminal).
3299                            let rollback = state.dsl_apply(
3300                                mm_dsl::MeerkatMachineInput::CancelWaitAll,
3301                                "CancelWaitAll(rollback)",
3302                            );
3303                            return Box::pin(WaitAllFuture {
3304                                registry: self,
3305                                wait_request_id,
3306                                state: WaitAllFutureState::Ready(Some(Err(match rollback {
3307                                    Ok(()) => OpsLifecycleError::Internal(
3308                                        "wait_all started while a pending wait sender already existed"
3309                                            .into(),
3310                                    ),
3311                                    Err(err) => err,
3312                                }))),
3313                            });
3314                        }
3315                        state.wait_request_id = Some(wait_request_id.clone());
3316                        let (sender, receiver) = tokio::sync::oneshot::channel();
3317                        state.pending_wait = Some(PendingWaitState {
3318                            wait_request_id: wait_request_id.clone(),
3319                            sender,
3320                        });
3321                        WaitAllFutureState::Waiting(receiver)
3322                    }
3323                    Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3324                }
3325            }
3326            Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3327        };
3328
3329        Box::pin(WaitAllFuture {
3330            registry: self,
3331            wait_request_id,
3332            state,
3333        })
3334    }
3335}
3336
3337#[cfg(test)]
3338#[allow(clippy::unwrap_used, clippy::panic)]
3339mod tests {
3340    use super::*;
3341    use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3342    use meerkat_core::lifecycle::RunId;
3343    use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3344    use meerkat_core::types::SessionId;
3345    use std::sync::atomic::Ordering;
3346    use uuid::Uuid;
3347
3348    fn test_run_id() -> RunId {
3349        RunId(Uuid::from_u128(1))
3350    }
3351
3352    fn background_spec(name: &str) -> OperationSpec {
3353        OperationSpec {
3354            id: OperationId::new(),
3355            kind: OperationKind::BackgroundToolOp,
3356            owner_session_id: SessionId::new(),
3357            display_name: name.into(),
3358            source_label: "test".into(),
3359            operation_source: None,
3360            child_session_id: None,
3361            expect_peer_channel: false,
3362        }
3363    }
3364
3365    #[tokio::test]
3366    async fn late_watchers_resolve_immediately() {
3367        let registry = RuntimeOpsLifecycleRegistry::new();
3368        let spec = background_spec("late");
3369        let op_id = spec.id.clone();
3370        registry.register_operation(spec).unwrap();
3371        registry.provisioning_succeeded(&op_id).unwrap();
3372        registry
3373            .complete_operation(
3374                &op_id,
3375                OperationResult {
3376                    id: op_id.clone(),
3377                    content: "done".into(),
3378                    is_error: false,
3379                    duration_ms: 1,
3380                    tokens_used: 0,
3381                },
3382            )
3383            .unwrap();
3384
3385        let watch = registry.register_watcher(&op_id).unwrap();
3386        match watch
3387            .await
3388            .expect("operation completion watch should resolve")
3389        {
3390            OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3391            other => panic!("expected completed outcome, got {other:?}"),
3392        }
3393    }
3394
3395    #[tokio::test]
3396    async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3397        let (tx, rx) = tokio::sync::oneshot::channel();
3398        let watch = operation_completion_watch_from_receiver(rx);
3399        drop(tx);
3400
3401        assert_eq!(
3402            watch.await,
3403            Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3404        );
3405    }
3406
3407    #[test]
3408    fn peer_ready_requires_peer_expectation() {
3409        let registry = RuntimeOpsLifecycleRegistry::new();
3410        let spec = background_spec("no-peer");
3411        let op_id = spec.id.clone();
3412        registry.register_operation(spec).unwrap();
3413        registry.provisioning_succeeded(&op_id).unwrap();
3414
3415        let result = registry.peer_ready(
3416            &op_id,
3417            OperationPeerHandle {
3418                peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3419                trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3420                    "peer",
3421                    PeerId::new(),
3422                    "inproc://peer",
3423                )
3424                .unwrap(),
3425            },
3426        );
3427        assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3428    }
3429
3430    /// K8b pinning test: the terminal payload carried through the generated
3431    /// machine IS the typed domain outcome — no JSON codec exists in either
3432    /// direction. Every variant classifies to its matching discriminant, the
3433    /// fail-closed read returns the exact payload when the discriminant
3434    /// matches, and rejects any discriminant/variant disagreement.
3435    #[test]
3436    fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3437        let op_id = OperationId::new();
3438        let outcomes = vec![
3439            (
3440                OperationTerminalOutcome::Completed(OperationResult {
3441                    id: op_id.clone(),
3442                    content: "done".into(),
3443                    is_error: false,
3444                    duration_ms: 7,
3445                    tokens_used: 42,
3446                }),
3447                mm_dsl::OperationTerminalOutcomeKind::Completed,
3448            ),
3449            (
3450                OperationTerminalOutcome::Failed {
3451                    error: "boom".into(),
3452                },
3453                mm_dsl::OperationTerminalOutcomeKind::Failed,
3454            ),
3455            (
3456                OperationTerminalOutcome::Aborted {
3457                    reason: Some("user aborted".into()),
3458                },
3459                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3460            ),
3461            (
3462                OperationTerminalOutcome::Aborted { reason: None },
3463                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3464            ),
3465            (
3466                OperationTerminalOutcome::Cancelled {
3467                    reason: Some("cancelled".into()),
3468                },
3469                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3470            ),
3471            (
3472                OperationTerminalOutcome::Cancelled { reason: None },
3473                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3474            ),
3475            (
3476                OperationTerminalOutcome::Retired,
3477                mm_dsl::OperationTerminalOutcomeKind::Retired,
3478            ),
3479            (
3480                OperationTerminalOutcome::Terminated {
3481                    reason: "owner stopped".into(),
3482                },
3483                mm_dsl::OperationTerminalOutcomeKind::Terminated,
3484            ),
3485        ];
3486
3487        for (outcome, expected_kind) in &outcomes {
3488            assert_eq!(
3489                mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3490                *expected_kind,
3491                "typed payload {outcome:?} must classify to {expected_kind:?}"
3492            );
3493            let read = ShellState::checked_terminal_payload(
3494                *expected_kind,
3495                outcome,
3496                "test authority",
3497                "test-op",
3498            )
3499            .expect("matching discriminant must read back the exact payload");
3500            assert_eq!(&read, outcome);
3501        }
3502
3503        // Discriminant/variant disagreement fails closed.
3504        let err = ShellState::checked_terminal_payload(
3505            mm_dsl::OperationTerminalOutcomeKind::Completed,
3506            &OperationTerminalOutcome::Retired,
3507            "test authority",
3508            "test-op",
3509        )
3510        .expect_err("variant mismatch must be rejected");
3511        assert!(matches!(err, OpsLifecycleError::Internal(_)));
3512    }
3513
3514    /// K8b machine-ownership pin: a terminal payload whose variant disagrees
3515    /// with the transition's terminal kind is rejected by the generated
3516    /// machine guard (`payload_variant_matches_kind`) — not by any shell
3517    /// decode step. Drives the DSL input directly to prove the guard owns
3518    /// the invariant.
3519    #[test]
3520    fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3521        let registry = RuntimeOpsLifecycleRegistry::new();
3522        let spec = background_spec("variant-mismatch");
3523        let op_id = spec.id.clone();
3524        registry.register_operation(spec).unwrap();
3525        registry.provisioning_succeeded(&op_id).unwrap();
3526
3527        let mut state = registry.write_state().unwrap();
3528        let err = state
3529            .dsl_apply(
3530                mm_dsl::MeerkatMachineInput::CompleteOp {
3531                    operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3532                    outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3533                    // Variant mismatch: Completed kind with a Retired payload.
3534                    payload: OperationTerminalOutcome::Retired,
3535                },
3536                "CompleteOp",
3537            )
3538            .expect_err("machine must reject payload variant mismatch");
3539        // The kernel reports guard rejections without naming the guard; the
3540        // discriminating pin is the pair: identical input EXCEPT the payload
3541        // variant is guard-rejected here, then accepted below. Status
3542        // (`Running`) and discriminant (`Completed`) are identical in both
3543        // calls, so `payload_variant_matches_kind` is the only differing
3544        // guard.
3545        assert!(
3546            matches!(
3547                &err,
3548                OpsLifecycleError::Internal(message)
3549                    if message.contains("GuardRejected") && message.contains("CompleteOp")
3550            ),
3551            "expected generated guard rejection for CompleteOp, got: {err:?}"
3552        );
3553        drop(state);
3554
3555        // RetireCompletedOp requires the unit Retired payload; a data-carrying
3556        // payload is rejected by the same guard shape.
3557        registry
3558            .complete_operation(
3559                &op_id,
3560                OperationResult {
3561                    id: op_id.clone(),
3562                    content: "done".into(),
3563                    is_error: false,
3564                    duration_ms: 1,
3565                    tokens_used: 0,
3566                },
3567            )
3568            .expect("matching variant must complete");
3569    }
3570
3571    #[test]
3572    fn duplicate_registration_rejection_is_generated() {
3573        let registry = RuntimeOpsLifecycleRegistry::new();
3574        let spec = background_spec("duplicate");
3575        let op_id = spec.id.clone();
3576
3577        registry.register_operation(spec.clone()).unwrap();
3578        let result = registry.register_operation(spec);
3579
3580        assert!(matches!(
3581            result,
3582            Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3583        ));
3584    }
3585
3586    #[test]
3587    fn invalid_transition_rejection_is_generated() {
3588        let registry = RuntimeOpsLifecycleRegistry::new();
3589        let spec = background_spec("invalid-transition");
3590        let op_id = spec.id.clone();
3591        registry.register_operation(spec).unwrap();
3592
3593        let result = registry.complete_operation(
3594            &op_id,
3595            OperationResult {
3596                id: op_id.clone(),
3597                content: "too-early".into(),
3598                is_error: false,
3599                duration_ms: 1,
3600                tokens_used: 0,
3601            },
3602        );
3603
3604        assert!(matches!(
3605            result,
3606            Err(OpsLifecycleError::InvalidTransition {
3607                id,
3608                status: OperationStatus::Provisioning,
3609                action: "complete_operation",
3610            }) if id == op_id
3611        ));
3612    }
3613
3614    #[tokio::test]
3615    async fn multi_listener_completion() {
3616        let registry = RuntimeOpsLifecycleRegistry::new();
3617        let spec = background_spec("multi");
3618        let op_id = spec.id.clone();
3619        registry.register_operation(spec).unwrap();
3620        registry.provisioning_succeeded(&op_id).unwrap();
3621
3622        let watch1 = registry.register_watcher(&op_id).unwrap();
3623        let watch2 = registry.register_watcher(&op_id).unwrap();
3624        let watch3 = registry.register_watcher(&op_id).unwrap();
3625
3626        registry
3627            .complete_operation(
3628                &op_id,
3629                OperationResult {
3630                    id: op_id.clone(),
3631                    content: "multi-done".into(),
3632                    is_error: false,
3633                    duration_ms: 1,
3634                    tokens_used: 0,
3635                },
3636            )
3637            .unwrap();
3638
3639        for watch in [watch1, watch2, watch3] {
3640            match watch
3641                .await
3642                .expect("operation completion watch should resolve")
3643            {
3644                OperationTerminalOutcome::Completed(result) => {
3645                    assert_eq!(result.content, "multi-done");
3646                }
3647                other => panic!("expected completed, got {other:?}"),
3648            }
3649        }
3650    }
3651
3652    #[tokio::test]
3653    async fn wait_all_returns_all_outcomes() {
3654        let registry = RuntimeOpsLifecycleRegistry::new();
3655
3656        let spec_a = background_spec("a");
3657        let id_a = spec_a.id.clone();
3658        registry.register_operation(spec_a).unwrap();
3659        registry.provisioning_succeeded(&id_a).unwrap();
3660
3661        let spec_b = background_spec("b");
3662        let id_b = spec_b.id.clone();
3663        registry.register_operation(spec_b).unwrap();
3664        registry.provisioning_succeeded(&id_b).unwrap();
3665
3666        registry
3667            .complete_operation(
3668                &id_a,
3669                OperationResult {
3670                    id: id_a.clone(),
3671                    content: "a-done".into(),
3672                    is_error: false,
3673                    duration_ms: 1,
3674                    tokens_used: 0,
3675                },
3676            )
3677            .unwrap();
3678        registry.fail_operation(&id_b, "b-error".into()).unwrap();
3679
3680        let wait_result = registry
3681            .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3682            .await
3683            .unwrap();
3684        assert_eq!(wait_result.outcomes.len(), 2);
3685        assert_eq!(wait_result.outcomes[0].0, id_a);
3686        assert!(matches!(
3687            wait_result.outcomes[0].1,
3688            OperationTerminalOutcome::Completed(_)
3689        ));
3690        assert_eq!(wait_result.outcomes[1].0, id_b);
3691        assert!(matches!(
3692            wait_result.outcomes[1].1,
3693            OperationTerminalOutcome::Failed { .. }
3694        ));
3695        // Obligation carries the awaited IDs
3696        assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3697        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3698    }
3699
3700    /// Exercises the trait `wait_all` path (via `dyn OpsLifecycleRegistry`)
3701    /// which must submit WaitAll through the DSL for cross-machine handoff.
3702    #[tokio::test]
3703    async fn wait_all_trait_path_submits_through_authority() {
3704        let registry = RuntimeOpsLifecycleRegistry::new();
3705        let spec = background_spec("trait-wait");
3706        let op_id = spec.id.clone();
3707        registry.register_operation(spec).unwrap();
3708        registry.provisioning_succeeded(&op_id).unwrap();
3709        registry
3710            .complete_operation(
3711                &op_id,
3712                OperationResult {
3713                    id: op_id.clone(),
3714                    content: "done".into(),
3715                    is_error: false,
3716                    duration_ms: 1,
3717                    tokens_used: 0,
3718                },
3719            )
3720            .unwrap();
3721
3722        // Call through trait object to exercise the trait impl, not the inherent method.
3723        let trait_ref: &dyn OpsLifecycleRegistry = &registry;
3724        let wait_result = trait_ref
3725            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3726            .await
3727            .unwrap();
3728        assert_eq!(wait_result.outcomes.len(), 1);
3729        assert!(matches!(
3730            wait_result.outcomes[0].1,
3731            OperationTerminalOutcome::Completed(_)
3732        ));
3733        // Obligation carries the validated ID
3734        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3735        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3736        let state = registry.read_state().unwrap();
3737        assert!(
3738            !state.wait_active(),
3739            "already-satisfied wait_all must be cleared by generated satisfaction authority"
3740        );
3741        assert!(state.wait_operation_ids().unwrap().is_empty());
3742    }
3743
3744    #[tokio::test]
3745    async fn wait_all_duplicate_rejection_is_generated() {
3746        let registry = RuntimeOpsLifecycleRegistry::new();
3747        let spec = background_spec("duplicate-wait");
3748        let op_id = spec.id.clone();
3749        registry.register_operation(spec).unwrap();
3750        registry.provisioning_succeeded(&op_id).unwrap();
3751
3752        let result = registry
3753            .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3754            .await;
3755
3756        assert!(matches!(
3757            result,
3758            Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3759        ));
3760        let state = registry.read_state().unwrap();
3761        assert!(
3762            !state.wait_active(),
3763            "duplicate wait rejection must not create a shell or machine barrier"
3764        );
3765        assert!(state.wait_operation_ids().unwrap().is_empty());
3766    }
3767
3768    #[tokio::test]
3769    async fn wait_all_active_rejection_is_generated() {
3770        let registry = RuntimeOpsLifecycleRegistry::new();
3771        let spec = background_spec("active-wait");
3772        let op_id = spec.id.clone();
3773        registry.register_operation(spec).unwrap();
3774        registry.provisioning_succeeded(&op_id).unwrap();
3775
3776        let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3777        let result = registry
3778            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3779            .await;
3780
3781        assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3782        drop(active_wait);
3783        let state = registry.read_state().unwrap();
3784        assert!(!state.wait_active());
3785        assert!(state.wait_operation_ids().unwrap().is_empty());
3786    }
3787
3788    #[tokio::test]
3789    async fn wait_all_unknown_operation_rejection_is_generated() {
3790        let registry = RuntimeOpsLifecycleRegistry::new();
3791        let op_id = OperationId::new();
3792
3793        let result = registry
3794            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3795            .await;
3796
3797        assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3798        let state = registry.read_state().unwrap();
3799        assert!(!state.wait_active());
3800        assert!(state.wait_operation_ids().unwrap().is_empty());
3801    }
3802
3803    #[tokio::test]
3804    async fn wait_all_resolves_from_authority_owned_wait_request() {
3805        let registry = RuntimeOpsLifecycleRegistry::new();
3806        let run_id = test_run_id();
3807
3808        let spec = background_spec("pending");
3809        let op_id = spec.id.clone();
3810        registry.register_operation(spec).unwrap();
3811        registry.provisioning_succeeded(&op_id).unwrap();
3812
3813        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3814        tokio::pin!(wait_fut);
3815        assert!(
3816            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3817                .await
3818                .is_err()
3819        );
3820
3821        let active_wait_request_id = {
3822            let state = registry.read_state().unwrap();
3823            let wait_request_id = match state.wait_request_id.clone() {
3824                Some(wait_request_id) => wait_request_id,
3825                None => panic!("wait request should be active"),
3826            };
3827            assert_eq!(
3828                state.wait_operation_ids().unwrap().as_slice(),
3829                std::slice::from_ref(&op_id)
3830            );
3831            wait_request_id
3832        };
3833
3834        registry
3835            .complete_operation(
3836                &op_id,
3837                OperationResult {
3838                    id: op_id.clone(),
3839                    content: "done".into(),
3840                    is_error: false,
3841                    duration_ms: 1,
3842                    tokens_used: 0,
3843                },
3844            )
3845            .unwrap();
3846
3847        let wait_result = wait_fut.await.unwrap();
3848        assert_eq!(
3849            wait_result.satisfied.wait_request_id,
3850            active_wait_request_id
3851        );
3852        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
3853        assert!(matches!(
3854            wait_result.outcomes.as_slice(),
3855            [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
3856        ));
3857        assert!(registry.read_state().unwrap().wait_request_id.is_none());
3858    }
3859
3860    #[tokio::test]
3861    async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
3862        let registry = RuntimeOpsLifecycleRegistry::new();
3863        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3864        registry.set_persistence_channel(
3865            tx,
3866            meerkat_core::RuntimeEpochId::new(),
3867            Arc::new(meerkat_core::EpochCursorState::new()),
3868        );
3869        let worker = std::thread::spawn(move || {
3870            let request = rx.blocking_recv().expect("persistence request");
3871            let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
3872                "injected persistence failure".to_string(),
3873            )));
3874        });
3875
3876        let spec = background_spec("feed-after-persist");
3877        let op_id = spec.id.clone();
3878        registry.register_operation(spec).unwrap();
3879        registry.provisioning_succeeded(&op_id).unwrap();
3880        let watch = registry.register_watcher(&op_id).unwrap();
3881
3882        let err = registry
3883            .complete_operation(
3884                &op_id,
3885                OperationResult {
3886                    id: op_id.clone(),
3887                    content: "done".into(),
3888                    is_error: false,
3889                    duration_ms: 1,
3890                    tokens_used: 0,
3891                },
3892            )
3893            .expect_err("persistence failure must fail the terminal transition");
3894        assert!(
3895            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
3896            "unexpected persistence error: {err:?}"
3897        );
3898        worker.join().expect("persistence worker");
3899
3900        let feed = registry.completion_feed_handle();
3901        let batch = feed.list_since(0);
3902        assert!(
3903            batch.entries.is_empty(),
3904            "completion feed must not publish entries before durable snapshot success"
3905        );
3906        assert_eq!(batch.watermark, 0);
3907        assert!(
3908            tokio::time::timeout(std::time::Duration::from_millis(10), watch)
3909                .await
3910                .is_err(),
3911            "watchers must not resolve before durable terminal snapshot success"
3912        );
3913        assert_eq!(
3914            registry.snapshot(&op_id).unwrap().unwrap().status,
3915            OperationStatus::Running,
3916            "failed persistence must roll the generated terminal transition back"
3917        );
3918
3919        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
3920        registry.set_persistence_channel(
3921            tx,
3922            meerkat_core::RuntimeEpochId::new(),
3923            Arc::new(meerkat_core::EpochCursorState::new()),
3924        );
3925        let worker = std::thread::spawn(move || {
3926            let request = rx.blocking_recv().expect("second persistence request");
3927            let _ = request.result_tx.send(Ok(()));
3928        });
3929        registry
3930            .complete_operation(
3931                &op_id,
3932                OperationResult {
3933                    id: op_id.clone(),
3934                    content: "done".into(),
3935                    is_error: false,
3936                    duration_ms: 1,
3937                    tokens_used: 0,
3938                },
3939            )
3940            .expect("rolled-back terminal transition should remain retryable");
3941        worker.join().expect("second persistence worker");
3942
3943        let batch = feed.list_since(0);
3944        assert_eq!(batch.entries.len(), 1);
3945        assert_eq!(batch.entries[0].operation_id, op_id);
3946    }
3947
3948    #[tokio::test]
3949    async fn dropping_wait_all_future_cancels_active_wait_request() {
3950        let registry = RuntimeOpsLifecycleRegistry::new();
3951        let run_id = test_run_id();
3952
3953        let spec = background_spec("cancelled-wait");
3954        let op_id = spec.id.clone();
3955        registry.register_operation(spec).unwrap();
3956        registry.provisioning_succeeded(&op_id).unwrap();
3957
3958        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3959        drop(wait_fut);
3960
3961        let state = registry.read_state().unwrap();
3962        assert!(state.wait_request_id.is_none());
3963        assert!(state.wait_operation_ids().unwrap().is_empty());
3964        assert!(!state.wait_active());
3965    }
3966
3967    /// id 101: an authority-invariant corruption surfaced during a terminal
3968    /// transition must NOT report the op terminal with a silently-hung barrier.
3969    /// The terminal call must return the typed `Internal` fault, AND the awaited
3970    /// `wait_all` future must resolve to `Err` (via the dropped-sender arm)
3971    /// instead of hanging forever.
3972    #[tokio::test]
3973    async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
3974        let registry = RuntimeOpsLifecycleRegistry::new();
3975        let run_id = test_run_id();
3976
3977        let spec = background_spec("corrupt-barrier");
3978        let op_id = spec.id.clone();
3979        registry.register_operation(spec).unwrap();
3980        registry.provisioning_succeeded(&op_id).unwrap();
3981
3982        // Activate a barrier whose waiter is still pending.
3983        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
3984        tokio::pin!(wait_fut);
3985        assert!(
3986            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
3987                .await
3988                .is_err(),
3989            "barrier waiter must still be pending before corruption"
3990        );
3991        assert!(registry.read_state().unwrap().wait_request_id.is_some());
3992
3993        // Corrupt the generated wait authority: an active wait_request_id with
3994        // no wait_run_id forces `try_satisfy_wait_all_authority` down its
3995        // non-GuardRejected `Internal` arm during the terminal transition.
3996        {
3997            let mut state = registry.write_state().unwrap();
3998            let mut machine_state = state.dsl.0.state().clone();
3999            machine_state.wait_run_id = None;
4000            state.dsl = DslAuthority(Box::new(
4001                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4002            ));
4003        }
4004
4005        // The terminal transition must FAIL with the typed authority fault, not
4006        // report the op complete.
4007        let err = registry
4008            .complete_operation(
4009                &op_id,
4010                OperationResult {
4011                    id: op_id.clone(),
4012                    content: "done".into(),
4013                    is_error: false,
4014                    duration_ms: 1,
4015                    tokens_used: 0,
4016                },
4017            )
4018            .expect_err("corrupt wait authority must fail the terminal transition");
4019        assert!(
4020            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
4021            "unexpected terminal error: {err:?}"
4022        );
4023
4024        // The waiter must resolve to Err (dropped sender), not hang.
4025        let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
4026            .await
4027            .expect("waiter must resolve, not hang, after authority corruption");
4028        match waiter_result {
4029            Err(OpsLifecycleError::Internal(message)) => assert!(
4030                message.contains("wait_all completion channel dropped"),
4031                "unexpected waiter error message: {message}"
4032            ),
4033            other => panic!("expected dropped-channel Internal error, got {other:?}"),
4034        }
4035    }
4036
4037    /// id 97: a poisoned registry lock must surface as a typed `Internal` fault
4038    /// from `completion_cursor`, NOT laundered into `Ok(None)` (which means "no
4039    /// generated cursor authority") or a bare `None`.
4040    #[test]
4041    fn completion_cursor_propagates_poison_not_none() {
4042        let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
4043
4044        // Poison the registry RwLock by panicking while holding the write guard.
4045        let poison_registry = std::sync::Arc::clone(&registry);
4046        let join = std::thread::spawn(move || {
4047            let _guard = poison_registry.write_state().unwrap();
4048            panic!("intentional panic to poison ops lifecycle registry lock");
4049        });
4050        assert!(
4051            join.join().is_err(),
4052            "poisoning thread must have panicked while holding the write guard"
4053        );
4054
4055        let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
4056        let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
4057        match result {
4058            Err(OpsLifecycleError::Internal(message)) => assert!(
4059                message.contains("ops lifecycle registry poisoned"),
4060                "unexpected cursor error message: {message}"
4061            ),
4062            other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
4063        }
4064    }
4065
4066    #[test]
4067    fn terminate_owner_only_targets_non_terminal_operations() {
4068        let registry = RuntimeOpsLifecycleRegistry::new();
4069
4070        let running_spec = background_spec("running");
4071        let running_id = running_spec.id.clone();
4072        registry.register_operation(running_spec).unwrap();
4073        registry.provisioning_succeeded(&running_id).unwrap();
4074
4075        let completed_spec = background_spec("completed");
4076        let completed_id = completed_spec.id.clone();
4077        registry.register_operation(completed_spec).unwrap();
4078        registry.provisioning_succeeded(&completed_id).unwrap();
4079        registry
4080            .complete_operation(
4081                &completed_id,
4082                OperationResult {
4083                    id: completed_id.clone(),
4084                    content: "done".into(),
4085                    is_error: false,
4086                    duration_ms: 1,
4087                    tokens_used: 0,
4088                },
4089            )
4090            .unwrap();
4091
4092        registry.terminate_owner("shutdown".into()).unwrap();
4093
4094        assert!(matches!(
4095            registry.snapshot(&running_id).unwrap().unwrap().status,
4096            OperationStatus::Terminated
4097        ));
4098        assert!(matches!(
4099            registry.snapshot(&completed_id).unwrap().unwrap().status,
4100            OperationStatus::Completed
4101        ));
4102    }
4103
4104    #[test]
4105    fn unregister_retirement_fences_detached_late_callbacks() {
4106        let registry = RuntimeOpsLifecycleRegistry::new();
4107        let running_spec = background_spec("detached-running");
4108        let running_id = running_spec.id.clone();
4109        registry.register_operation(running_spec).unwrap();
4110        registry.provisioning_succeeded(&running_id).unwrap();
4111
4112        registry
4113            .retire_owner_for_unregister("owner unregistered".into())
4114            .unwrap();
4115        assert!(matches!(
4116            registry.snapshot(&running_id).unwrap().unwrap().status,
4117            OperationStatus::Terminated
4118        ));
4119
4120        let late_result = OperationResult {
4121            id: running_id.clone(),
4122            content: "late detached completion".into(),
4123            is_error: false,
4124            duration_ms: 1,
4125            tokens_used: 0,
4126        };
4127        assert_eq!(
4128            registry.complete_operation(&running_id, late_result),
4129            Err(OpsLifecycleError::OwnerRetired)
4130        );
4131        assert_eq!(
4132            registry.register_operation(background_spec("late-new-op")),
4133            Err(OpsLifecycleError::OwnerRetired)
4134        );
4135        assert_eq!(
4136            registry.report_progress(
4137                &running_id,
4138                OperationProgressUpdate {
4139                    message: "late progress".into(),
4140                    percent: None,
4141                },
4142            ),
4143            Err(OpsLifecycleError::OwnerRetired)
4144        );
4145    }
4146
4147    #[test]
4148    fn collect_completed_drains_terminal_operations() {
4149        let registry = RuntimeOpsLifecycleRegistry::new();
4150
4151        let spec_a = background_spec("a");
4152        let id_a = spec_a.id.clone();
4153        registry.register_operation(spec_a).unwrap();
4154        registry.provisioning_succeeded(&id_a).unwrap();
4155        registry
4156            .complete_operation(
4157                &id_a,
4158                OperationResult {
4159                    id: id_a.clone(),
4160                    content: "done".into(),
4161                    is_error: false,
4162                    duration_ms: 1,
4163                    tokens_used: 0,
4164                },
4165            )
4166            .unwrap();
4167
4168        let spec_b = background_spec("b");
4169        let id_b = spec_b.id.clone();
4170        registry.register_operation(spec_b).unwrap();
4171
4172        let collected = registry.collect_completed().unwrap();
4173        assert_eq!(collected.len(), 1);
4174        assert_eq!(collected[0].0, id_a);
4175
4176        assert!(registry.snapshot(&id_a).unwrap().is_none());
4177        assert!(registry.snapshot(&id_b).unwrap().is_some());
4178
4179        let collected2 = registry.collect_completed().unwrap();
4180        assert!(collected2.is_empty());
4181    }
4182
4183    #[test]
4184    fn bounded_completed_retention_evicts_oldest() {
4185        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4186            max_completed: 3,
4187            max_concurrent: None,
4188        });
4189
4190        let mut ids = Vec::new();
4191        for i in 0..5 {
4192            let spec = background_spec(&format!("op-{i}"));
4193            let id = spec.id.clone();
4194            registry.register_operation(spec).unwrap();
4195            registry.provisioning_succeeded(&id).unwrap();
4196            registry
4197                .complete_operation(
4198                    &id,
4199                    OperationResult {
4200                        id: id.clone(),
4201                        content: format!("done-{i}"),
4202                        is_error: false,
4203                        duration_ms: 1,
4204                        tokens_used: 0,
4205                    },
4206                )
4207                .unwrap();
4208            ids.push(id);
4209        }
4210
4211        assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4212        assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4213        assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4214        assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4215        assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4216    }
4217
4218    #[test]
4219    fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4220        let registry = RuntimeOpsLifecycleRegistry::new();
4221
4222        let completed_spec = background_spec("completed");
4223        let completed_id = completed_spec.id.clone();
4224        registry.register_operation(completed_spec).unwrap();
4225        registry.provisioning_succeeded(&completed_id).unwrap();
4226        registry
4227            .complete_operation(
4228                &completed_id,
4229                OperationResult {
4230                    id: completed_id.clone(),
4231                    content: "done".into(),
4232                    is_error: false,
4233                    duration_ms: 1,
4234                    tokens_used: 0,
4235                },
4236            )
4237            .unwrap();
4238
4239        let running_spec = background_spec("running");
4240        let running_id = running_spec.id.clone();
4241        registry.register_operation(running_spec).unwrap();
4242        registry.provisioning_succeeded(&running_id).unwrap();
4243
4244        let cursor_state = meerkat_core::EpochCursorState::new();
4245        let snapshot = registry
4246            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4247            .unwrap();
4248        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4249
4250        assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4251        assert!(recovered.snapshot(&running_id).unwrap().is_none());
4252
4253        let collected = recovered.collect_completed().unwrap();
4254        assert_eq!(collected.len(), 1);
4255        assert_eq!(collected[0].0, completed_id);
4256    }
4257
4258    #[test]
4259    fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4260        let registry = RuntimeOpsLifecycleRegistry::new();
4261
4262        let mut spec = background_spec("capacity");
4263        spec.kind = OperationKind::BackgroundToolCapacitySlot;
4264        let operation_id = spec.id.clone();
4265        registry.register_operation(spec).unwrap();
4266        registry.provisioning_succeeded(&operation_id).unwrap();
4267        registry.mark_retired(&operation_id).unwrap();
4268
4269        assert!(registry.snapshot(&operation_id).unwrap().is_none());
4270
4271        let cursor_state = meerkat_core::EpochCursorState::new();
4272        let snapshot = registry
4273            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4274            .unwrap();
4275        assert!(
4276            !snapshot
4277                .authority_state
4278                .operations
4279                .contains_key(&operation_id)
4280        );
4281        assert!(!snapshot.operation_specs.contains_key(&operation_id));
4282        assert!(snapshot.completion_entries.is_empty());
4283
4284        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4285        assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4286    }
4287
4288    #[test]
4289    fn recovered_snapshot_uses_authority_operation_source() {
4290        let registry = RuntimeOpsLifecycleRegistry::new();
4291        let child_session_id = SessionId::new();
4292        let operation_source = OperationSource::session_child(child_session_id.clone());
4293        let spec = OperationSpec {
4294            id: OperationId::new(),
4295            kind: OperationKind::MobMemberChild,
4296            owner_session_id: SessionId::new(),
4297            display_name: "source-recovery".into(),
4298            source_label: "test".into(),
4299            operation_source: Some(operation_source.clone()),
4300            child_session_id: Some(child_session_id),
4301            expect_peer_channel: true,
4302        };
4303        let operation_id = spec.id.clone();
4304
4305        registry.register_operation(spec).unwrap();
4306        registry.provisioning_succeeded(&operation_id).unwrap();
4307        registry.mark_retired(&operation_id).unwrap();
4308
4309        let cursor_state = meerkat_core::EpochCursorState::new();
4310        let mut snapshot = registry
4311            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4312            .unwrap();
4313        assert_eq!(
4314            snapshot
4315                .authority_state
4316                .operations
4317                .get(&operation_id)
4318                .and_then(|state| state.operation_source.as_ref()),
4319            Some(&operation_source)
4320        );
4321
4322        snapshot
4323            .operation_specs
4324            .get_mut(&operation_id)
4325            .expect("persisted spec")
4326            .operation_source = None;
4327        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4328        assert_eq!(
4329            recovered
4330                .snapshot(&operation_id)
4331                .unwrap()
4332                .unwrap()
4333                .operation_source,
4334            Some(operation_source)
4335        );
4336    }
4337
4338    #[test]
4339    fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4340        let registry = RuntimeOpsLifecycleRegistry::new();
4341        let child_session_id = SessionId::new();
4342        let operation_source = OperationSource::session_child(child_session_id.clone());
4343        let spec = OperationSpec {
4344            id: OperationId::new(),
4345            kind: OperationKind::MobMemberChild,
4346            owner_session_id: SessionId::new(),
4347            display_name: "source-drift".into(),
4348            source_label: "test".into(),
4349            operation_source: Some(operation_source),
4350            child_session_id: Some(child_session_id),
4351            expect_peer_channel: true,
4352        };
4353        let operation_id = spec.id.clone();
4354
4355        registry.register_operation(spec).unwrap();
4356        registry.provisioning_succeeded(&operation_id).unwrap();
4357        registry.mark_retired(&operation_id).unwrap();
4358
4359        let cursor_state = meerkat_core::EpochCursorState::new();
4360        let mut snapshot = registry
4361            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4362            .unwrap();
4363        snapshot
4364            .operation_specs
4365            .get_mut(&operation_id)
4366            .expect("persisted spec")
4367            .operation_source = Some(OperationSource::session_child(SessionId::new()));
4368
4369        let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4370            .expect_err("source mirror drift must fail recovery");
4371        assert!(
4372            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4373            "unexpected recovery error: {err:?}"
4374        );
4375    }
4376
4377    #[test]
4378    fn persisted_authority_state_serializes_explicit_no_operation_source() {
4379        let registry = RuntimeOpsLifecycleRegistry::new();
4380
4381        let spec = background_spec("explicit-no-source");
4382        let operation_id = spec.id.clone();
4383        registry.register_operation(spec).unwrap();
4384        registry.provisioning_succeeded(&operation_id).unwrap();
4385
4386        let cursor_state = meerkat_core::EpochCursorState::new();
4387        let snapshot = registry
4388            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4389            .unwrap();
4390        let value = serde_json::to_value(&snapshot).unwrap();
4391        let operations = value
4392            .get("authority_state")
4393            .and_then(|state| state.get("operations"))
4394            .and_then(serde_json::Value::as_object)
4395            .expect("serialized authority operations");
4396        let persisted_state = operations
4397            .values()
4398            .next()
4399            .and_then(serde_json::Value::as_object)
4400            .expect("serialized operation state");
4401
4402        assert!(
4403            persisted_state
4404                .get("operation_source")
4405                .is_some_and(serde_json::Value::is_null),
4406            "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4407        );
4408
4409        let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4410        assert_eq!(
4411            recovered_snapshot
4412                .authority_state
4413                .operations
4414                .get(&operation_id)
4415                .expect("round-tripped operation")
4416                .operation_source,
4417            None
4418        );
4419    }
4420
4421    #[test]
4422    fn persisted_authority_state_rejects_missing_operation_source_fact() {
4423        let registry = RuntimeOpsLifecycleRegistry::new();
4424
4425        let spec = background_spec("missing-source-fact");
4426        registry.register_operation(spec).unwrap();
4427
4428        let cursor_state = meerkat_core::EpochCursorState::new();
4429        let snapshot = registry
4430            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4431            .unwrap();
4432        let mut value = serde_json::to_value(&snapshot).unwrap();
4433        let operations = value
4434            .get_mut("authority_state")
4435            .and_then(|state| state.get_mut("operations"))
4436            .and_then(serde_json::Value::as_object_mut)
4437            .expect("serialized authority operations");
4438        let operation_state = operations
4439            .values_mut()
4440            .next()
4441            .and_then(serde_json::Value::as_object_mut)
4442            .expect("serialized operation state");
4443        assert!(operation_state.remove("operation_source").is_some());
4444
4445        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4446            .expect_err("missing generated source fact must fail recovery snapshot decoding");
4447        assert!(
4448            err.to_string().contains("operation_source"),
4449            "unexpected decode error: {err}"
4450        );
4451    }
4452
4453    #[test]
4454    fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4455        let registry = RuntimeOpsLifecycleRegistry::new();
4456
4457        let spec = background_spec("missing-feed-authority");
4458        let operation_id = spec.id.clone();
4459        registry.register_operation(spec).unwrap();
4460        registry.provisioning_succeeded(&operation_id).unwrap();
4461        registry
4462            .complete_operation(
4463                &operation_id,
4464                OperationResult {
4465                    id: operation_id.clone(),
4466                    content: "done".into(),
4467                    is_error: false,
4468                    duration_ms: 1,
4469                    tokens_used: 0,
4470                },
4471            )
4472            .unwrap();
4473
4474        let cursor_state = meerkat_core::EpochCursorState::new();
4475        let snapshot = registry
4476            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4477            .unwrap();
4478        let mut value = serde_json::to_value(&snapshot).unwrap();
4479        let authority_state = value
4480            .get_mut("authority_state")
4481            .and_then(serde_json::Value::as_object_mut)
4482            .expect("serialized authority state");
4483        assert!(authority_state.remove("completion_feed_entries").is_some());
4484
4485        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4486            .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4487        assert!(
4488            err.to_string().contains("completion_feed_entries"),
4489            "unexpected decode error: {err}"
4490        );
4491    }
4492
4493    #[test]
4494    fn public_child_session_projection_uses_authority_operation_source() {
4495        let registry = RuntimeOpsLifecycleRegistry::new();
4496        let authority_child_session_id = SessionId::new();
4497        let stale_shell_child_session_id = SessionId::new();
4498        let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4499        let spec = OperationSpec {
4500            id: OperationId::new(),
4501            kind: OperationKind::MobMemberChild,
4502            owner_session_id: SessionId::new(),
4503            display_name: "child-projection".into(),
4504            source_label: "test".into(),
4505            operation_source: Some(operation_source),
4506            child_session_id: Some(stale_shell_child_session_id),
4507            expect_peer_channel: true,
4508        };
4509        let operation_id = spec.id.clone();
4510
4511        registry.register_operation(spec).unwrap();
4512
4513        assert_eq!(
4514            registry
4515                .snapshot(&operation_id)
4516                .unwrap()
4517                .unwrap()
4518                .child_session_id,
4519            Some(authority_child_session_id.clone())
4520        );
4521
4522        let cursor_state = meerkat_core::EpochCursorState::new();
4523        let snapshot = registry
4524            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4525            .unwrap();
4526        assert_eq!(
4527            snapshot
4528                .operation_specs
4529                .get(&operation_id)
4530                .expect("persisted spec")
4531                .child_session_id,
4532            Some(authority_child_session_id)
4533        );
4534    }
4535
4536    #[test]
4537    fn generated_terminal_payload_projection_fails_closed() {
4538        let registry = RuntimeOpsLifecycleRegistry::new();
4539
4540        let spec = background_spec("terminal-payload-drift");
4541        let operation_id = spec.id.clone();
4542        registry.register_operation(spec).unwrap();
4543        registry.provisioning_succeeded(&operation_id).unwrap();
4544        registry
4545            .complete_operation(
4546                &operation_id,
4547                OperationResult {
4548                    id: operation_id.clone(),
4549                    content: "done".into(),
4550                    is_error: false,
4551                    duration_ms: 1,
4552                    tokens_used: 0,
4553                },
4554            )
4555            .unwrap();
4556
4557        {
4558            let mut state = registry.write_state().unwrap();
4559            let mut machine_state = state.dsl.0.state().clone();
4560            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4561            machine_state
4562                .op_terminal_payload
4563                .insert(operation_id_key, OperationTerminalOutcome::Retired);
4564            state.dsl = DslAuthority(Box::new(
4565                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4566            ));
4567        }
4568
4569        let err = match registry.register_watcher(&operation_id) {
4570            Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4571            Err(err) => err,
4572        };
4573        assert!(
4574            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4575            "unexpected watcher error: {err:?}"
4576        );
4577        let err = registry
4578            .snapshot(&operation_id)
4579            .expect_err("invalid generated terminal payload must reject public snapshot");
4580        assert!(
4581            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
4582            "unexpected public snapshot error: {err:?}"
4583        );
4584
4585        let cursor_state = meerkat_core::EpochCursorState::new();
4586        let err = match registry
4587            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4588        {
4589            Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
4590            Err(err) => err,
4591        };
4592        assert!(
4593            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4594            "unexpected snapshot error: {err:?}"
4595        );
4596
4597        let err = match registry.collect_completed() {
4598            Ok(_) => panic!("invalid generated terminal payload must reject collection"),
4599            Err(err) => err,
4600        };
4601        assert!(
4602            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4603            "unexpected collection error: {err:?}"
4604        );
4605    }
4606
4607    #[test]
4608    fn generated_terminal_payload_missing_projection_fails_closed() {
4609        let registry = RuntimeOpsLifecycleRegistry::new();
4610
4611        let spec = background_spec("terminal-payload-missing");
4612        let operation_id = spec.id.clone();
4613        registry.register_operation(spec).unwrap();
4614        registry.provisioning_succeeded(&operation_id).unwrap();
4615        registry
4616            .fail_operation(&operation_id, "boom".into())
4617            .unwrap();
4618
4619        {
4620            let mut state = registry.write_state().unwrap();
4621            let mut machine_state = state.dsl.0.state().clone();
4622            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4623            machine_state.op_terminal_payload.remove(&operation_id_key);
4624            state.dsl = DslAuthority(Box::new(
4625                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4626            ));
4627        }
4628
4629        let err = match registry.register_watcher(&operation_id) {
4630            Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
4631            Err(err) => err,
4632        };
4633        assert!(
4634            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4635            "unexpected watcher error: {err:?}"
4636        );
4637        let err = registry
4638            .snapshot(&operation_id)
4639            .expect_err("missing generated terminal payload must reject public snapshot");
4640        assert!(
4641            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
4642            "unexpected public snapshot error: {err:?}"
4643        );
4644    }
4645
4646    #[test]
4647    fn generated_terminal_status_without_outcome_fails_closed() {
4648        let registry = RuntimeOpsLifecycleRegistry::new();
4649
4650        let spec = background_spec("terminal-outcome-missing");
4651        let operation_id = spec.id.clone();
4652        registry.register_operation(spec).unwrap();
4653        registry.provisioning_succeeded(&operation_id).unwrap();
4654        registry
4655            .fail_operation(&operation_id, "boom".into())
4656            .unwrap();
4657
4658        {
4659            let mut state = registry.write_state().unwrap();
4660            let mut machine_state = state.dsl.0.state().clone();
4661            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4662            machine_state.op_terminal_outcomes.remove(&operation_id_key);
4663            state.dsl = DslAuthority(Box::new(
4664                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4665            ));
4666        }
4667
4668        let err = registry
4669            .snapshot(&operation_id)
4670            .expect_err("terminal status without outcome must reject public snapshot");
4671        assert!(
4672            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4673            "unexpected public snapshot error: {err:?}"
4674        );
4675
4676        let err = match registry.collect_completed() {
4677            Ok(_) => panic!("terminal status without outcome must reject collection"),
4678            Err(err) => err,
4679        };
4680        assert!(
4681            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
4682            "unexpected collection error: {err:?}"
4683        );
4684    }
4685
4686    #[test]
4687    fn generated_operation_source_projection_fails_closed() {
4688        let registry = RuntimeOpsLifecycleRegistry::new();
4689        let child_session_id = SessionId::new();
4690        let operation_source = OperationSource::session_child(child_session_id.clone());
4691        let spec = OperationSpec {
4692            id: OperationId::new(),
4693            kind: OperationKind::MobMemberChild,
4694            owner_session_id: SessionId::new(),
4695            display_name: "source-authority-drift".into(),
4696            source_label: "test".into(),
4697            operation_source: Some(operation_source),
4698            child_session_id: Some(child_session_id),
4699            expect_peer_channel: true,
4700        };
4701        let operation_id = spec.id.clone();
4702
4703        registry.register_operation(spec).unwrap();
4704
4705        {
4706            let mut state = registry.write_state().unwrap();
4707            let mut machine_state = state.dsl.0.state().clone();
4708            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4709            machine_state
4710                .op_sources
4711                .get_mut(&operation_id_key)
4712                .expect("generated operation source")
4713                .session_id = None;
4714            state.dsl = DslAuthority(Box::new(
4715                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4716            ));
4717        }
4718
4719        let err = registry
4720            .snapshot(&operation_id)
4721            .expect_err("invalid generated operation source must reject public snapshot");
4722        assert!(
4723            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4724            "unexpected public snapshot error: {err:?}"
4725        );
4726        let err = registry
4727            .list_operations()
4728            .expect_err("invalid generated operation source must reject public operation list");
4729        assert!(
4730            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4731            "unexpected operation list error: {err:?}"
4732        );
4733
4734        let cursor_state = meerkat_core::EpochCursorState::new();
4735        let err = match registry
4736            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4737        {
4738            Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
4739            Err(err) => err,
4740        };
4741        assert!(
4742            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
4743            "unexpected snapshot error: {err:?}"
4744        );
4745    }
4746
4747    #[test]
4748    fn generated_operation_id_projection_fails_closed() {
4749        let registry = RuntimeOpsLifecycleRegistry::new();
4750
4751        {
4752            let mut state = registry.write_state().unwrap();
4753            let mut machine_state = state.dsl.0.state().clone();
4754            machine_state.op_statuses.insert(
4755                "not-json-operation-id".into(),
4756                mm_dsl::OperationStatus::Running,
4757            );
4758            state.dsl = DslAuthority(Box::new(
4759                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4760            ));
4761        }
4762
4763        let err = registry
4764            .list_operations()
4765            .expect_err("invalid generated operation id must reject public operation list");
4766        assert!(
4767            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4768            "unexpected operation list error: {err:?}"
4769        );
4770
4771        let cursor_state = meerkat_core::EpochCursorState::new();
4772        let err = match registry
4773            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4774        {
4775            Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
4776            Err(err) => err,
4777        };
4778        assert!(
4779            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
4780            "unexpected persistence snapshot error: {err:?}"
4781        );
4782    }
4783
4784    #[test]
4785    fn generated_missing_kind_projection_fails_closed() {
4786        let registry = RuntimeOpsLifecycleRegistry::new();
4787        let spec = background_spec("missing-kind");
4788        let operation_id = spec.id.clone();
4789        registry.register_operation(spec).unwrap();
4790
4791        {
4792            let mut state = registry.write_state().unwrap();
4793            let mut machine_state = state.dsl.0.state().clone();
4794            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4795            machine_state.op_kinds.remove(&operation_id_key);
4796            state.dsl = DslAuthority(Box::new(
4797                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4798            ));
4799        }
4800
4801        let err = registry
4802            .snapshot(&operation_id)
4803            .expect_err("missing generated kind must reject public snapshot");
4804        assert!(
4805            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4806            "unexpected public snapshot error: {err:?}"
4807        );
4808        let err = registry
4809            .list_operations()
4810            .expect_err("missing generated kind must reject public list");
4811        assert!(
4812            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4813            "unexpected public list error: {err:?}"
4814        );
4815
4816        let cursor_state = meerkat_core::EpochCursorState::new();
4817        let err = registry
4818            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4819            .expect_err("missing generated kind must reject persistence snapshot");
4820        assert!(
4821            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
4822            "unexpected persistence snapshot error: {err:?}"
4823        );
4824    }
4825
4826    #[test]
4827    fn generated_missing_status_projection_fails_closed() {
4828        let registry = RuntimeOpsLifecycleRegistry::new();
4829        let spec = background_spec("missing-status");
4830        let operation_id = spec.id.clone();
4831        registry.register_operation(spec).unwrap();
4832
4833        {
4834            let mut state = registry.write_state().unwrap();
4835            let mut machine_state = state.dsl.0.state().clone();
4836            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4837            machine_state.op_statuses.remove(&operation_id_key);
4838            state.dsl = DslAuthority(Box::new(
4839                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4840            ));
4841        }
4842
4843        let err = registry
4844            .snapshot(&operation_id)
4845            .expect_err("missing generated status must reject public snapshot");
4846        assert!(
4847            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4848            "unexpected public snapshot error: {err:?}"
4849        );
4850        let err = registry
4851            .list_operations()
4852            .expect_err("missing generated status must reject public list");
4853        assert!(
4854            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4855            "unexpected public list error: {err:?}"
4856        );
4857        let err = registry
4858            .classify_operation_public_result(&operation_id)
4859            .expect_err("missing generated status must reject public-result classification");
4860        assert!(
4861            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
4862            "unexpected public-result error: {err:?}"
4863        );
4864    }
4865
4866    #[test]
4867    fn generated_retiring_public_result_remains_running_until_terminal() {
4868        let registry = RuntimeOpsLifecycleRegistry::new();
4869        let spec = background_spec("retiring-public-result");
4870        let operation_id = spec.id.clone();
4871        registry.register_operation(spec).unwrap();
4872        registry.provisioning_succeeded(&operation_id).unwrap();
4873        registry.request_retire(&operation_id).unwrap();
4874
4875        let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
4876        assert_eq!(snapshot.status, OperationStatus::Retiring);
4877        assert!(snapshot.terminal_outcome.is_none());
4878        assert!(!snapshot.terminal);
4879        assert_eq!(
4880            snapshot.public_result_class,
4881            OperationPublicResultClass::Running
4882        );
4883        assert_eq!(
4884            registry
4885                .classify_operation_public_result(&operation_id)
4886                .unwrap(),
4887            OperationPublicResultClass::Running
4888        );
4889    }
4890
4891    #[test]
4892    fn generated_missing_peer_ready_projection_fails_closed() {
4893        let registry = RuntimeOpsLifecycleRegistry::new();
4894        let spec = background_spec("missing-peer-ready");
4895        let operation_id = spec.id.clone();
4896        registry.register_operation(spec).unwrap();
4897
4898        {
4899            let mut state = registry.write_state().unwrap();
4900            let mut machine_state = state.dsl.0.state().clone();
4901            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4902            machine_state.op_peer_ready.remove(&operation_id_key);
4903            state.dsl = DslAuthority(Box::new(
4904                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4905            ));
4906        }
4907
4908        let err = registry
4909            .snapshot(&operation_id)
4910            .expect_err("missing generated peer-ready fact must reject public snapshot");
4911        assert!(
4912            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4913            "unexpected public snapshot error: {err:?}"
4914        );
4915
4916        let cursor_state = meerkat_core::EpochCursorState::new();
4917        let err = registry
4918            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4919            .expect_err("missing generated peer-ready fact must reject persistence snapshot");
4920        assert!(
4921            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
4922            "unexpected persistence snapshot error: {err:?}"
4923        );
4924    }
4925
4926    #[test]
4927    fn generated_missing_progress_count_projection_fails_closed() {
4928        let registry = RuntimeOpsLifecycleRegistry::new();
4929        let spec = background_spec("missing-progress-count");
4930        let operation_id = spec.id.clone();
4931        registry.register_operation(spec).unwrap();
4932
4933        {
4934            let mut state = registry.write_state().unwrap();
4935            let mut machine_state = state.dsl.0.state().clone();
4936            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4937            machine_state.op_progress_counts.remove(&operation_id_key);
4938            state.dsl = DslAuthority(Box::new(
4939                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4940            ));
4941        }
4942
4943        let err = registry
4944            .snapshot(&operation_id)
4945            .expect_err("missing generated progress count must reject public snapshot");
4946        assert!(
4947            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4948            "unexpected public snapshot error: {err:?}"
4949        );
4950
4951        let cursor_state = meerkat_core::EpochCursorState::new();
4952        let err = registry
4953            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4954            .expect_err("missing generated progress count must reject persistence snapshot");
4955        assert!(
4956            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
4957            "unexpected persistence snapshot error: {err:?}"
4958        );
4959    }
4960
4961    #[test]
4962    fn generated_terminal_sequence_missing_persistence_fails_closed() {
4963        let registry = RuntimeOpsLifecycleRegistry::new();
4964        let spec = background_spec("terminal-sequence-missing");
4965        let operation_id = spec.id.clone();
4966        registry.register_operation(spec).unwrap();
4967        registry.provisioning_succeeded(&operation_id).unwrap();
4968        registry
4969            .complete_operation(
4970                &operation_id,
4971                OperationResult {
4972                    id: operation_id.clone(),
4973                    content: "done".into(),
4974                    is_error: false,
4975                    duration_ms: 1,
4976                    tokens_used: 0,
4977                },
4978            )
4979            .unwrap();
4980
4981        {
4982            let mut state = registry.write_state().unwrap();
4983            let mut machine_state = state.dsl.0.state().clone();
4984            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4985            machine_state.op_completion_seq.remove(&operation_id_key);
4986            state.dsl = DslAuthority(Box::new(
4987                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4988            ));
4989        }
4990
4991        let cursor_state = meerkat_core::EpochCursorState::new();
4992        let err = registry
4993            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4994            .expect_err("missing generated terminal sequence must reject persistence snapshot");
4995        assert!(
4996            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
4997            "unexpected persistence snapshot error: {err:?}"
4998        );
4999    }
5000
5001    #[test]
5002    fn generated_record_without_shell_projection_fails_closed() {
5003        let registry = RuntimeOpsLifecycleRegistry::new();
5004        let spec = background_spec("missing-shell-record");
5005        let operation_id = spec.id.clone();
5006        registry.register_operation(spec).unwrap();
5007
5008        {
5009            let mut state = registry.write_state().unwrap();
5010            state.records.remove(&operation_id);
5011        }
5012
5013        let err = registry
5014            .snapshot(&operation_id)
5015            .expect_err("generated operation without shell record must reject public snapshot");
5016        assert!(
5017            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5018            "unexpected public snapshot error: {err:?}"
5019        );
5020        let err = registry
5021            .list_operations()
5022            .expect_err("generated operation without shell record must reject public list");
5023        assert!(
5024            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5025            "unexpected public list error: {err:?}"
5026        );
5027    }
5028
5029    #[test]
5030    fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
5031        let registry = RuntimeOpsLifecycleRegistry::new();
5032        let child_session_id = SessionId::new();
5033        let operation_source = OperationSource::session_child(child_session_id.clone());
5034        let spec = OperationSpec {
5035            id: OperationId::new(),
5036            kind: OperationKind::MobMemberChild,
5037            owner_session_id: SessionId::new(),
5038            display_name: "child-drift".into(),
5039            source_label: "test".into(),
5040            operation_source: Some(operation_source),
5041            child_session_id: Some(child_session_id.clone()),
5042            expect_peer_channel: true,
5043        };
5044        let operation_id = spec.id.clone();
5045
5046        registry.register_operation(spec).unwrap();
5047        registry.provisioning_succeeded(&operation_id).unwrap();
5048        registry.mark_retired(&operation_id).unwrap();
5049
5050        let cursor_state = meerkat_core::EpochCursorState::new();
5051        let mut snapshot = registry
5052            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5053            .unwrap();
5054        snapshot
5055            .operation_specs
5056            .get_mut(&operation_id)
5057            .expect("persisted spec")
5058            .child_session_id = Some(SessionId::new());
5059
5060        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5061        assert_eq!(
5062            recovered
5063                .snapshot(&operation_id)
5064                .unwrap()
5065                .unwrap()
5066                .child_session_id,
5067            Some(child_session_id)
5068        );
5069    }
5070
5071    #[test]
5072    fn completion_wake_class_is_generated_by_operation_kind() {
5073        let registry = RuntimeOpsLifecycleRegistry::new();
5074        let operation_id = OperationId::new();
5075
5076        assert_eq!(
5077            registry
5078                .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
5079                .unwrap(),
5080            OperationCompletionWakeClass::Wake
5081        );
5082        assert_eq!(
5083            registry
5084                .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
5085                .unwrap(),
5086            OperationCompletionWakeClass::Ignore
5087        );
5088        assert_eq!(
5089            registry
5090                .classify_operation_completion_wake(
5091                    &operation_id,
5092                    OperationKind::BackgroundToolCapacitySlot,
5093                )
5094                .unwrap(),
5095            OperationCompletionWakeClass::Ignore
5096        );
5097    }
5098
5099    #[test]
5100    fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
5101        let registry = RuntimeOpsLifecycleRegistry::new();
5102
5103        let running_spec = background_spec("running");
5104        let running_id = running_spec.id.clone();
5105        registry.register_operation(running_spec.clone()).unwrap();
5106        registry.provisioning_succeeded(&running_id).unwrap();
5107
5108        let cursor_state = meerkat_core::EpochCursorState::new();
5109        let mut snapshot = registry
5110            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5111            .unwrap();
5112        snapshot.completion_entries.push(CompletionEntry {
5113            seq: 1,
5114            operation_id: running_id.clone(),
5115            kind: running_spec.kind,
5116            display_name: running_spec.display_name,
5117            terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5118                id: running_id,
5119                content: "phantom".into(),
5120                is_error: false,
5121                duration_ms: 1,
5122                tokens_used: 0,
5123            }),
5124            completed_at_ms: None,
5125        });
5126
5127        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5128            Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5129            Err(err) => err,
5130        };
5131        assert!(
5132            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5133            "unexpected recovery error: {err:?}"
5134        );
5135    }
5136
5137    #[test]
5138    fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5139        let registry = RuntimeOpsLifecycleRegistry::new();
5140        let phantom_id = OperationId::new();
5141        {
5142            let state = registry.read_state().unwrap();
5143            state.feed_buffer.push(CompletionEntry {
5144                seq: 1,
5145                operation_id: phantom_id.clone(),
5146                kind: OperationKind::BackgroundToolOp,
5147                display_name: "phantom".into(),
5148                terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5149                    id: phantom_id,
5150                    content: "phantom".into(),
5151                    is_error: false,
5152                    duration_ms: 1,
5153                    tokens_used: 0,
5154                }),
5155                completed_at_ms: None,
5156            });
5157        }
5158
5159        let cursor_state = meerkat_core::EpochCursorState::new();
5160        let err = match registry
5161            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5162        {
5163            Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5164            Err(err) => err,
5165        };
5166        assert!(
5167            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5168            "unexpected capture error: {err:?}"
5169        );
5170    }
5171
5172    #[test]
5173    fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5174        let registry = RuntimeOpsLifecycleRegistry::new();
5175
5176        let spec = background_spec("terminal");
5177        let operation_id = spec.id.clone();
5178        registry.register_operation(spec).unwrap();
5179        registry.provisioning_succeeded(&operation_id).unwrap();
5180        registry
5181            .complete_operation(
5182                &operation_id,
5183                OperationResult {
5184                    id: operation_id.clone(),
5185                    content: "done".into(),
5186                    is_error: false,
5187                    duration_ms: 1,
5188                    tokens_used: 0,
5189                },
5190            )
5191            .unwrap();
5192
5193        let cursor_state = meerkat_core::EpochCursorState::new();
5194        let mut snapshot = registry
5195            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5196            .unwrap();
5197        let phantom_id = OperationId::new();
5198        let phantom_result = OperationResult {
5199            id: phantom_id.clone(),
5200            content: "phantom".into(),
5201            is_error: false,
5202            duration_ms: 1,
5203            tokens_used: 0,
5204        };
5205        let phantom_entry = CompletionFeedCanonicalState {
5206            seq: snapshot.authority_state.next_completion_seq + 1,
5207            kind: OperationKind::BackgroundToolOp,
5208            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5209        };
5210        snapshot
5211            .authority_state
5212            .completion_feed_entries
5213            .insert(phantom_id.clone(), phantom_entry);
5214        snapshot.completion_entries.push(CompletionEntry {
5215            seq: snapshot.authority_state.next_completion_seq + 1,
5216            operation_id: phantom_id.clone(),
5217            kind: OperationKind::BackgroundToolOp,
5218            display_name: "phantom".into(),
5219            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5220            completed_at_ms: None,
5221        });
5222
5223        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5224            Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5225            Err(err) => err,
5226        };
5227        assert!(
5228            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5229            "unexpected recovery error: {err:?}"
5230        );
5231    }
5232
5233    #[test]
5234    fn recovered_completed_order_uses_generated_completion_sequences() {
5235        let registry = RuntimeOpsLifecycleRegistry::new();
5236
5237        let spec_a = background_spec("a");
5238        let id_a = spec_a.id.clone();
5239        registry.register_operation(spec_a).unwrap();
5240        registry.provisioning_succeeded(&id_a).unwrap();
5241        registry
5242            .complete_operation(
5243                &id_a,
5244                OperationResult {
5245                    id: id_a.clone(),
5246                    content: "a".into(),
5247                    is_error: false,
5248                    duration_ms: 1,
5249                    tokens_used: 0,
5250                },
5251            )
5252            .unwrap();
5253
5254        let spec_b = background_spec("b");
5255        let id_b = spec_b.id.clone();
5256        registry.register_operation(spec_b).unwrap();
5257        registry.provisioning_succeeded(&id_b).unwrap();
5258        registry
5259            .complete_operation(
5260                &id_b,
5261                OperationResult {
5262                    id: id_b.clone(),
5263                    content: "b".into(),
5264                    is_error: false,
5265                    duration_ms: 1,
5266                    tokens_used: 0,
5267                },
5268            )
5269            .unwrap();
5270
5271        let cursor_state = meerkat_core::EpochCursorState::new();
5272        let mut snapshot = registry
5273            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5274            .unwrap();
5275        snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5276
5277        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5278        let collected = recovered.collect_completed().unwrap();
5279
5280        assert_eq!(collected[0].0, id_a);
5281        assert_eq!(collected[1].0, id_b);
5282    }
5283
5284    #[test]
5285    fn max_concurrent_enforcement() {
5286        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5287            max_completed: DEFAULT_MAX_COMPLETED,
5288            max_concurrent: Some(2),
5289        });
5290
5291        let spec_a = background_spec("a");
5292        let id_a = spec_a.id.clone();
5293        registry.register_operation(spec_a).unwrap();
5294
5295        let spec_b = background_spec("b");
5296        registry.register_operation(spec_b).unwrap();
5297
5298        let spec_c = background_spec("c");
5299        let result = registry.register_operation(spec_c);
5300        assert!(matches!(
5301            result,
5302            Err(OpsLifecycleError::MaxConcurrentExceeded {
5303                limit: 2,
5304                active: 2,
5305            })
5306        ));
5307
5308        registry.provisioning_succeeded(&id_a).unwrap();
5309        registry
5310            .complete_operation(
5311                &id_a,
5312                OperationResult {
5313                    id: id_a.clone(),
5314                    content: "done".into(),
5315                    is_error: false,
5316                    duration_ms: 1,
5317                    tokens_used: 0,
5318                },
5319            )
5320            .unwrap();
5321
5322        let spec_d = background_spec("d");
5323        assert!(registry.register_operation(spec_d).is_ok());
5324    }
5325
5326    #[test]
5327    fn snapshot_includes_timestamps() {
5328        let registry = RuntimeOpsLifecycleRegistry::new();
5329        let spec = background_spec("timed");
5330        let op_id = spec.id.clone();
5331        registry.register_operation(spec).unwrap();
5332
5333        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5334        assert!(snap1.created_at_ms > 0);
5335        assert!(snap1.started_at_ms.is_none());
5336        assert!(snap1.completed_at_ms.is_none());
5337        assert!(snap1.elapsed_ms.is_none());
5338
5339        registry.provisioning_succeeded(&op_id).unwrap();
5340        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5341        assert!(snap2.started_at_ms.is_some());
5342        assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5343
5344        registry
5345            .complete_operation(
5346                &op_id,
5347                OperationResult {
5348                    id: op_id.clone(),
5349                    content: "done".into(),
5350                    is_error: false,
5351                    duration_ms: 1,
5352                    tokens_used: 0,
5353                },
5354            )
5355            .unwrap();
5356        let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5357        assert!(snap3.completed_at_ms.is_some());
5358        assert!(snap3.elapsed_ms.is_some());
5359        assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5360    }
5361
5362    #[test]
5363    fn snapshot_includes_peer_handle() {
5364        let registry = RuntimeOpsLifecycleRegistry::new();
5365        let child_session_id = SessionId::new();
5366        let spec = OperationSpec {
5367            id: OperationId::new(),
5368            kind: OperationKind::MobMemberChild,
5369            owner_session_id: SessionId::new(),
5370            display_name: "peer-test".into(),
5371            source_label: "test".into(),
5372            operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5373            child_session_id: Some(child_session_id),
5374            expect_peer_channel: true,
5375        };
5376        let op_id = spec.id.clone();
5377        registry.register_operation(spec).unwrap();
5378        registry.provisioning_succeeded(&op_id).unwrap();
5379
5380        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5381        assert!(snap1.peer_handle.is_none());
5382
5383        let handle = OperationPeerHandle {
5384            peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5385            trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5386                "member-x",
5387                PeerId::new(),
5388                "inproc://x",
5389            )
5390            .unwrap(),
5391        };
5392        registry.peer_ready(&op_id, handle).unwrap();
5393
5394        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5395        assert_eq!(
5396            snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5397            "member-x"
5398        );
5399    }
5400}