Skip to main content

meerkat_runtime/
ops_lifecycle.rs

1//! In-memory runtime implementation of the shared async-operation lifecycle seam.
2//!
3//! Per-operation canonical lifecycle state lives in the MeerkatMachine DSL
4//! authority (`op_statuses`, `op_terminal_outcomes`, `op_kinds`,
5//! `op_sources`, `op_peer_ready`, `op_progress_counts`, `active_op_count`,
6//! `wait_active`, `wait_operation_ids`). This shell layer owns pure mechanics: watcher
7//! channels, timestamps, peer handles, snapshot assembly, FIFO eviction
8//! bookkeeping, the completion feed buffer, and typed delivery of generated
9//! admission/rejection feedback.
10//!
11//! Per-transition legality ("is `CompleteOp` legal on a `Provisioning` op?")
12//! is NOT owned by the shell — it lives in the DSL's `from_status_valid`
13//! guards on each op-lifecycle transition. The shell's only job on a
14//! `GuardRejected` rejection is to ask the generated rejection resolver for
15//! the public result class.
16
17use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
18use std::future::Future;
19use std::sync::atomic::{AtomicU64, Ordering};
20use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
21use std::task::{Context, Poll};
22
23use meerkat_core::completion_feed::{
24    CompletionBatch, CompletionEntry, CompletionFeed, CompletionSeq,
25};
26
27#[cfg(target_arch = "wasm32")]
28use crate::tokio;
29use meerkat_core::lifecycle::{RunId, WaitRequestId};
30use meerkat_core::ops_lifecycle::{
31    CompletionCursorConsumer, DEFAULT_MAX_COMPLETED, OperationCompletionWakeClass,
32    OperationCompletionWatch, OperationId, OperationKind, OperationLifecycleAction,
33    OperationLifecycleSnapshot, OperationPeerHandle, OperationProgressUpdate,
34    OperationPublicResultClass, OperationResult, OperationSource, OperationSpec, OperationStatus,
35    OperationTerminalOutcome, OpsLifecycleError, OpsLifecycleRegistry, WaitAllResult,
36    WaitAllSatisfied,
37};
38use meerkat_core::time_compat::{Instant, SystemTime, UNIX_EPOCH};
39use meerkat_core::types::SessionId;
40
41use crate::meerkat_machine::dsl as mm_dsl;
42
43// ---------------------------------------------------------------------------
44// Serde-only persisted canonical state shells
45// ---------------------------------------------------------------------------
46//
47// These structures preserve the persisted fact shape of `PersistedOpsSnapshot`.
48// They are pure serde shells — no methods beyond read-only field accessors,
49// no authority behavior. Optional generated facts that can be explicitly
50// absent still serialize as present `null` so recovery can distinguish
51// generated "none" from a missing persisted fact.
52
53fn deserialize_required_operation_source<'de, D>(
54    deserializer: D,
55) -> Result<Option<OperationSource>, D::Error>
56where
57    D: serde::Deserializer<'de>,
58{
59    <Option<OperationSource> as serde::Deserialize>::deserialize(deserializer)
60}
61
62/// Canonical per-operation state as captured in persisted snapshots.
63#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
64pub struct OperationCanonicalState {
65    status: OperationStatus,
66    kind: OperationKind,
67    #[serde(deserialize_with = "deserialize_required_operation_source")]
68    operation_source: Option<OperationSource>,
69    peer_ready: bool,
70    progress_count: u32,
71    watcher_count: u32,
72    terminal_outcome: Option<OperationTerminalOutcome>,
73    #[serde(default, skip_serializing_if = "Option::is_none")]
74    completion_sequence: Option<CompletionSeq>,
75    terminal_buffered: bool,
76}
77
78/// Generated-owned public completion feed fact captured from DSL authority.
79#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
80pub struct CompletionFeedCanonicalState {
81    seq: CompletionSeq,
82    kind: OperationKind,
83    terminal_outcome: OperationTerminalOutcome,
84}
85
86/// Canonical registry-level state as captured in persisted snapshots.
87#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
88pub struct RegistryCanonicalState {
89    operations: HashMap<OperationId, OperationCanonicalState>,
90    completion_feed_entries: HashMap<OperationId, CompletionFeedCanonicalState>,
91    completed_order: VecDeque<OperationId>,
92    max_completed: usize,
93    max_concurrent: Option<usize>,
94    active_count: usize,
95    wait_request_id: Option<WaitRequestId>,
96    wait_operation_ids: Vec<OperationId>,
97    next_completion_seq: CompletionSeq,
98}
99
100impl RegistryCanonicalState {
101    /// Maximum completed operations retained at capture time.
102    pub fn max_completed(&self) -> usize {
103        self.max_completed
104    }
105
106    /// Maximum concurrent non-terminal operations at capture time.
107    pub fn max_concurrent(&self) -> Option<usize> {
108        self.max_concurrent
109    }
110
111    /// Number of operations captured in the snapshot.
112    pub fn operation_count(&self) -> usize {
113        self.operations.len()
114    }
115
116    /// Number of generated-owned public completion feed entries captured.
117    pub fn completion_feed_count(&self) -> usize {
118        self.completion_feed_entries.len()
119    }
120}
121
122// ---------------------------------------------------------------------------
123// Serializable snapshot for persistence
124// ---------------------------------------------------------------------------
125
126/// Serializable snapshot of the ops lifecycle registry state.
127///
128/// Captured on terminal transitions for durable persistence. Contains
129/// canonical state, operation specs, persisted completion feed entries, and
130/// consumer cursor values. Wire format preserved verbatim from legacy
131/// runtime versions for backward compatibility.
132#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
133pub struct PersistedOpsSnapshot {
134    /// Epoch identity at capture time.
135    pub epoch_id: meerkat_core::RuntimeEpochId,
136    /// Canonical machine-owned state at capture time.
137    pub authority_state: RegistryCanonicalState,
138    /// Per-operation specs for shell record reconstruction.
139    pub operation_specs: HashMap<OperationId, meerkat_core::ops_lifecycle::OperationSpec>,
140    /// Persisted completion feed projection metadata. Canonical feed truth is
141    /// captured in `authority_state.completion_feed_entries`.
142    pub completion_entries: Vec<CompletionEntry>,
143    /// Exact bounded public-feed retention observed with `completion_entries`.
144    ///
145    /// Released 0.8.10 snapshots predate this carrier. Recovery of that
146    /// released shape preserves the retained public suffix but does not invent
147    /// an eviction gap from the sparse global completion sequence.
148    #[serde(default, skip_serializing_if = "Option::is_none")]
149    completion_feed_retention: Option<PersistedCompletionFeedRetention>,
150    /// Consumer cursor snapshot at capture time.
151    pub cursors: meerkat_core::EpochCursorSnapshot,
152}
153
154/// Store-owned facts about public completion rows actually evicted from the
155/// bounded feed projection.
156#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
157struct PersistedCompletionFeedRetention {
158    dropped_through: CompletionSeq,
159    dropped_count: u64,
160}
161
162#[derive(Debug)]
163pub struct OpsLifecyclePersistenceRequest {
164    snapshot: PersistedOpsSnapshot,
165    result_tx: std::sync::mpsc::SyncSender<Result<(), OpsLifecycleError>>,
166}
167
168impl OpsLifecyclePersistenceRequest {
169    pub fn snapshot(&self) -> &PersistedOpsSnapshot {
170        &self.snapshot
171    }
172
173    pub fn complete(self, result: Result<(), OpsLifecycleError>) {
174        let _ = self.result_tx.send(result);
175    }
176}
177
178// ---------------------------------------------------------------------------
179// Concrete completion feed buffer
180// ---------------------------------------------------------------------------
181
182/// Shared inner state of the completion feed buffer.
183///
184/// Protected by the registry's `RwLock<ShellState>` for writes, and by its
185/// own `RwLock` for reads by external consumers (agent boundary, idle wake).
186#[derive(Debug)]
187struct FeedBufferInner {
188    entries: VecDeque<CompletionEntry>,
189    watermark: CompletionSeq,
190    dropped_through: CompletionSeq,
191    dropped_count: u64,
192    max_retained: usize,
193}
194
195/// Shared completion feed buffer owned by the runtime registry.
196///
197/// The registry writes entries under its own write lock. External consumers
198/// read through the [`RuntimeCompletionFeed`] handle.
199#[derive(Debug)]
200struct FeedBuffer {
201    inner: RwLock<FeedBufferInner>,
202    /// Atomic mirror of watermark for lock-free `watermark()` reads.
203    watermark_atomic: AtomicU64,
204    /// Notifies all waiters when new entries are appended.
205    notify: tokio::sync::Notify,
206}
207
208impl FeedBuffer {
209    fn new(max_retained: usize) -> Self {
210        Self {
211            inner: RwLock::new(FeedBufferInner {
212                entries: VecDeque::new(),
213                watermark: 0,
214                dropped_through: 0,
215                dropped_count: 0,
216                max_retained,
217            }),
218            watermark_atomic: AtomicU64::new(0),
219            notify: tokio::sync::Notify::new(),
220        }
221    }
222
223    fn push(&self, entry: CompletionEntry) {
224        let mut inner = self
225            .inner
226            .write()
227            .unwrap_or_else(std::sync::PoisonError::into_inner);
228        let seq = entry.seq;
229        inner.entries.push_back(entry);
230        inner.watermark = seq;
231
232        // Evict oldest if over capacity.
233        while inner.entries.len() > inner.max_retained {
234            if let Some(dropped) = inner.entries.pop_front() {
235                inner.dropped_through = inner.dropped_through.max(dropped.seq);
236                inner.dropped_count = inner.dropped_count.saturating_add(1);
237            }
238        }
239
240        drop(inner);
241
242        self.watermark_atomic.store(seq, Ordering::Release);
243        self.notify.notify_waiters();
244    }
245
246    fn recover_evicted_prefix(
247        &self,
248        retention: PersistedCompletionFeedRetention,
249    ) -> Result<(), OpsLifecycleError> {
250        let PersistedCompletionFeedRetention {
251            dropped_through,
252            dropped_count,
253        } = retention;
254        if (dropped_through == 0) != (dropped_count == 0) || dropped_count > dropped_through {
255            return Err(OpsLifecycleError::Internal(
256                "persisted completion-feed retention carrier is inconsistent".to_string(),
257            ));
258        }
259        if dropped_count == 0 {
260            return Ok(());
261        }
262        let mut inner = self
263            .inner
264            .write()
265            .unwrap_or_else(std::sync::PoisonError::into_inner);
266        inner.dropped_through = dropped_through;
267        inner.dropped_count = dropped_count;
268        Ok(())
269    }
270}
271
272/// Read-only handle to the runtime completion feed.
273///
274/// Implements [`CompletionFeed`] for external consumers. Obtained via
275/// [`RuntimeOpsLifecycleRegistry::completion_feed()`].
276#[derive(Debug, Clone)]
277pub struct RuntimeCompletionFeed {
278    buffer: Arc<FeedBuffer>,
279    /// Generated completion-feed authority used only to reconstruct a suffix
280    /// that rolled out of the bounded public projection. Weak avoids a
281    /// registry/feed cycle; if the owner is gone the typed gap remains.
282    authority: std::sync::Weak<RwLock<ShellState>>,
283}
284
285impl CompletionFeed for RuntimeCompletionFeed {
286    fn watermark(&self) -> CompletionSeq {
287        self.buffer.watermark_atomic.load(Ordering::Acquire)
288    }
289
290    fn list_since(&self, after_seq: CompletionSeq) -> CompletionBatch {
291        let inner = self
292            .buffer
293            .inner
294            .read()
295            .unwrap_or_else(std::sync::PoisonError::into_inner);
296        let entries: Vec<CompletionEntry> = inner
297            .entries
298            .iter()
299            .filter(|e| e.seq > after_seq)
300            .cloned()
301            .collect();
302        let watermark = inner.watermark;
303        let dropped_count = inner.dropped_count;
304        let batch = CompletionBatch {
305            entries,
306            watermark,
307            dropped_through: inner.dropped_through,
308        };
309        drop(inner);
310
311        if batch.gap_after(after_seq).is_none() {
312            return batch;
313        }
314
315        let Some(authority) = self.authority.upgrade() else {
316            return batch;
317        };
318        let state = authority
319            .read()
320            .unwrap_or_else(std::sync::PoisonError::into_inner);
321        let Ok(authority_entries) = state.completion_feed_authority_entries() else {
322            // Keep the typed gap visible. A consumer must fail closed rather
323            // than accepting a suffix that generated authority could not
324            // reconstruct.
325            return batch;
326        };
327        let reconstructed_dropped_count = authority_entries
328            .values()
329            .filter(|entry| entry.seq <= batch.dropped_through)
330            .count() as u64;
331        if reconstructed_dropped_count != dropped_count {
332            // The public projection says more feed entries were evicted than
333            // generated authority can reproduce. Keep the gap typed rather
334            // than treating a partial reconstruction as complete custody.
335            return batch;
336        }
337        let buffered_projection_by_id: HashMap<OperationId, CompletionEntry> = batch
338            .entries
339            .iter()
340            .cloned()
341            .map(|entry| (entry.operation_id.clone(), entry))
342            .collect();
343        let mut entries = authority_entries
344            .into_iter()
345            .filter(|(_, entry)| entry.seq > after_seq && entry.seq <= batch.watermark)
346            .map(|(operation_id, entry)| {
347                if let Some(buffered) = buffered_projection_by_id.get(&operation_id) {
348                    return buffered.clone();
349                }
350                let display_name = state
351                    .records
352                    .get(&operation_id)
353                    .map(|record| record.spec.display_name.clone())
354                    .unwrap_or_default();
355                let completed_at_ms = state.records.get(&operation_id).and_then(|record| {
356                    record
357                        .completed_at
358                        .map(|completed_at| record.epoch_millis_for_instant(completed_at))
359                });
360                CompletionEntry {
361                    seq: entry.seq,
362                    operation_id,
363                    kind: entry.kind,
364                    display_name,
365                    terminal_outcome: entry.terminal_outcome,
366                    completed_at_ms,
367                }
368            })
369            .collect::<Vec<_>>();
370        entries.sort_by_key(|entry| entry.seq);
371        CompletionBatch {
372            entries,
373            watermark: batch.watermark,
374            // Generated authority reconstructed the complete semantic suffix,
375            // so bounded projection eviction is no longer a consumer gap.
376            dropped_through: 0,
377        }
378    }
379
380    fn wait_for_advance(
381        &self,
382        after_seq: CompletionSeq,
383    ) -> std::pin::Pin<Box<dyn Future<Output = CompletionSeq> + Send + '_>> {
384        Box::pin(async move {
385            loop {
386                // Register the waiter BEFORE reading the watermark.
387                // notify_waiters() in push() only wakes already-registered
388                // listeners — if we read first and push() lands between the
389                // read and notified().await, the wake is lost.
390                let notified = self.buffer.notify.notified();
391                let current = self.buffer.watermark_atomic.load(Ordering::Acquire);
392                if current > after_seq {
393                    return current;
394                }
395                notified.await;
396            }
397        })
398    }
399}
400
401// ---------------------------------------------------------------------------
402// Shell-only per-operation record (not part of canonical machine state)
403// ---------------------------------------------------------------------------
404
405#[derive(Debug)]
406struct OperationCompletionNotifier {
407    tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>,
408}
409
410impl OperationCompletionNotifier {
411    fn new(tx: tokio::sync::oneshot::Sender<OperationTerminalOutcome>) -> Self {
412        Self { tx }
413    }
414
415    fn notify_after_generated_terminal(self, outcome: &OperationTerminalOutcome) {
416        let _ = self.tx.send(outcome.clone());
417    }
418}
419
420fn operation_completion_watch_from_receiver(
421    rx: tokio::sync::oneshot::Receiver<OperationTerminalOutcome>,
422) -> OperationCompletionWatch {
423    Box::pin(async move {
424        rx.await
425            .map_err(|_| meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
426    })
427}
428
429fn resolved_operation_completion_watch(
430    outcome: OperationTerminalOutcome,
431) -> OperationCompletionWatch {
432    Box::pin(async move { Ok(outcome) })
433}
434
435/// Shell-owned data for a single operation. Canonical lifecycle state lives in
436/// the DSL authority; this struct holds I/O concerns that the DSL has no
437/// knowledge of.
438#[derive(Debug)]
439struct ShellRecord {
440    spec: OperationSpec,
441    peer_handle: Option<OperationPeerHandle>,
442    /// Private waiter plumbing. Notifiers are drained only from
443    /// `finalize_terminal()` after the generated authority has accepted and
444    /// stored a terminal outcome.
445    watchers: Vec<OperationCompletionNotifier>,
446    // Monotonic timestamps for elapsed computation
447    created_at: Instant,
448    started_at: Option<Instant>,
449    completed_at: Option<Instant>,
450    // Wall-clock anchor captured at creation for epoch millis
451    created_at_wall: SystemTime,
452}
453
454#[derive(Debug)]
455struct PendingWaitState {
456    wait_request_id: WaitRequestId,
457    sender: tokio::sync::oneshot::Sender<WaitAllSatisfied>,
458}
459
460enum WaitAllAuthorityPlan {
461    AlreadySatisfied(WaitAllSatisfied),
462    ActivateBarrier,
463}
464
465#[derive(Debug, Clone, Copy, PartialEq, Eq)]
466enum RecoveredOperationRecordDisposition {
467    Retain,
468    Discard,
469}
470
471impl ShellRecord {
472    fn new(spec: OperationSpec) -> Self {
473        Self {
474            spec,
475            peer_handle: None,
476            watchers: Vec::new(),
477            created_at: Instant::now(),
478            started_at: None,
479            completed_at: None,
480            created_at_wall: SystemTime::now(),
481        }
482    }
483
484    fn epoch_millis(wall_anchor: &SystemTime) -> u64 {
485        wall_anchor
486            .duration_since(UNIX_EPOCH)
487            .map(|d| d.as_millis() as u64)
488            .unwrap_or(0)
489    }
490
491    fn epoch_millis_for_instant(&self, instant: Instant) -> u64 {
492        // Compute wall time for a given instant using the wall-clock anchor:
493        // wall_time = created_at_wall + (instant - created_at)
494        let offset = instant.saturating_duration_since(self.created_at);
495        let wall = self.created_at_wall + offset;
496        Self::epoch_millis(&wall)
497    }
498
499    /// Notify all watchers with the given terminal outcome and drain the list.
500    fn notify_watchers(&mut self, outcome: &OperationTerminalOutcome) {
501        for watcher in std::mem::take(&mut self.watchers) {
502            watcher.notify_after_generated_terminal(outcome);
503        }
504    }
505
506    /// Mark the completion timestamp.
507    fn mark_completed(&mut self) {
508        self.completed_at = Some(Instant::now());
509    }
510}
511
512// ---------------------------------------------------------------------------
513// Combined shell state: DSL authority + shell records
514// ---------------------------------------------------------------------------
515
516#[derive(Debug)]
517struct ShellState {
518    /// DSL authority — sole source of truth for per-op canonical state.
519    dsl: DslAuthority,
520    /// Shell-owned per-operation records (specs, watchers, timestamps, peer handles).
521    records: HashMap<OperationId, ShellRecord>,
522    /// Pending wait-all coordination (oneshot channel).
523    pending_wait: Option<PendingWaitState>,
524    /// FIFO ordering of completed operation IDs for bounded eviction.
525    completed_order: VecDeque<OperationId>,
526    /// Maximum completed operations to retain.
527    max_completed: usize,
528    /// Maximum concurrent non-terminal operations (None = unlimited).
529    max_concurrent: Option<usize>,
530    /// Oneshot correlation id for the currently-pending `wait_all` future.
531    ///
532    /// Barrier membership (`wait_operation_ids`) and activation (`wait_active`)
533    /// are DSL-owned. This field is pure transport mechanics — the identity
534    /// the oneshot sender is tagged with so `Drop` can correlate cancellation.
535    wait_request_id: Option<WaitRequestId>,
536    /// Shared feed buffer for completion events.
537    feed_buffer: Arc<FeedBuffer>,
538    /// Persistence channel for durable snapshot writes (set via `set_persistence_channel`).
539    persist_tx: Option<crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>>,
540    /// Terminal unregister permanently fences any later channel replacement.
541    persistence_sealed: bool,
542    /// Epoch ID for persistence snapshots.
543    persist_epoch_id: Option<meerkat_core::RuntimeEpochId>,
544    /// Shared cursor state for persistence snapshots.
545    persist_cursor_state: Option<Arc<meerkat_core::EpochCursorState>>,
546    /// Mechanical epoch fence closed by canonical owner teardown.
547    ///
548    /// Generated authority terminalizes each known operation. This bit closes
549    /// the shell admission door so detached callbacks that retained an Arc to
550    /// the registry cannot create or mutate operations after unregister.
551    owner_retired: bool,
552}
553
554/// Wrapper around the DSL authority that provides `Debug` output.
555///
556/// The generated `MeerkatMachineAuthority` does not derive `Debug`, but
557/// `ShellState` requires it. This wrapper delegates to the inner state's
558/// `Debug` impl.
559struct DslAuthority(Box<mm_dsl::MeerkatMachineAuthority>);
560
561impl std::fmt::Debug for DslAuthority {
562    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
563        f.debug_struct("DslAuthority")
564            .field("state", self.0.state())
565            .finish()
566    }
567}
568
569/// Create a DSL authority initialized through generated authority. Per-op
570/// transitions guard only on `op_statuses.contains_key(operation_id)`, so the
571/// phase stays in `Idle` permanently (they all `to Idle`).
572fn new_ops_dsl_authority() -> DslAuthority {
573    DslAuthority(Box::new(
574        crate::meerkat_machine::dsl_authority::new_initialized_authority(
575            "ops lifecycle DSL Initialize must be accepted",
576        ),
577    ))
578}
579
580impl ShellState {
581    fn new(max_completed: usize, max_concurrent: Option<usize>) -> Self {
582        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState creating dsl");
583        let dsl = new_ops_dsl_authority();
584        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created dsl");
585        // Zero retention cannot preserve even the completion that just
586        // advanced the cursor watermark. Normalize it to the smallest sound
587        // bounded feed instead of publishing an immediately unauthorized row.
588        let max_completed = max_completed.max(1);
589        // The projection and generated completion authority must share one
590        // retention boundary. A larger shell buffer can retain entries after
591        // `EvictCompletedOp` removes their generated classifier, turning a
592        // legitimate lagging cursor into an untyped stale-authority exit.
593        let feed_capacity = max_completed.max(1);
594        tracing::info!(
595            feed_capacity,
596            "RuntimeOpsLifecycleRegistry::ShellState creating feed buffer"
597        );
598        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
599        tracing::info!("RuntimeOpsLifecycleRegistry::ShellState created feed buffer");
600        Self {
601            dsl,
602            records: HashMap::new(),
603            pending_wait: None,
604            completed_order: VecDeque::new(),
605            max_completed,
606            max_concurrent,
607            wait_request_id: None,
608            // Entries are evicted by the same completed-operation bound as
609            // generated authority. FeedBuffer records the exact lost prefix;
610            // a lagging consumer therefore receives a typed gap rather than a
611            // projection row whose classifier has already disappeared.
612            feed_buffer,
613            persist_tx: None,
614            persistence_sealed: false,
615            persist_epoch_id: None,
616            persist_cursor_state: None,
617            owner_retired: false,
618        }
619    }
620
621    fn ensure_owner_active(&self) -> Result<(), OpsLifecycleError> {
622        if self.owner_retired {
623            Err(OpsLifecycleError::OwnerRetired)
624        } else {
625            Ok(())
626        }
627    }
628
629    /// Apply a DSL input, mapping transition errors into
630    /// [`OpsLifecycleError::Internal`]. Callers that need to distinguish
631    /// guard rejections (legal-transition violations) from internal desync
632    /// should use [`Self::dsl_apply_raw`] and classify the error themselves;
633    /// this helper is for DSL inputs whose preconditions the caller has
634    /// already fully validated (e.g., `RequestWaitAll`, `SatisfyWaitAll`).
635    fn dsl_apply(
636        &mut self,
637        input: mm_dsl::MeerkatMachineInput,
638        context: &str,
639    ) -> Result<(), OpsLifecycleError> {
640        self.dsl_apply_raw(input).map_err(|err| {
641            OpsLifecycleError::Internal(format!("DSL rejected ops transition ({context}): {err:?}"))
642        })
643    }
644
645    /// Apply a DSL input, returning the raw kernel-level rejection so callers
646    /// can distinguish `GuardRejected` (a legitimate legality violation, e.g.,
647    /// `complete_operation` on a `Provisioning` op) from
648    /// `NoMatchingTransition` (a shell/DSL desync). Op-lifecycle entry points
649    /// feed guard rejections into generated rejection feedback before surfacing
650    /// public result classes.
651    fn dsl_apply_raw(
652        &mut self,
653        input: mm_dsl::MeerkatMachineInput,
654    ) -> Result<(), mm_dsl::MeerkatMachineTransitionError> {
655        mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map(|_transition| ())
656    }
657
658    fn dsl_apply_with_effects(
659        &mut self,
660        input: mm_dsl::MeerkatMachineInput,
661        context: &str,
662    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
663        let transition =
664            mm_dsl::MeerkatMachineMutator::apply(&mut *self.dsl.0, input).map_err(|err| {
665                OpsLifecycleError::Internal(format!(
666                    "DSL rejected ops transition ({context}): {err:?}"
667                ))
668            })?;
669        Ok(transition.into_effects())
670    }
671
672    /// Fail-closed read of a typed terminal payload entry: the payload IS the
673    /// domain [`OperationTerminalOutcome`] (K8b fold — no JSON codec), but the
674    /// shell still refuses to surface a payload whose variant disagrees with
675    /// the recorded discriminant.
676    fn checked_terminal_payload(
677        kind: mm_dsl::OperationTerminalOutcomeKind,
678        payload: &OperationTerminalOutcome,
679        authority: &str,
680        operation_id: &str,
681    ) -> Result<OperationTerminalOutcome, OpsLifecycleError> {
682        if mm_dsl::OperationTerminalOutcomeKind::from(payload) != kind {
683            return Err(OpsLifecycleError::Internal(format!(
684                "{authority} payload variant for {operation_id} does not match terminal outcome discriminant"
685            )));
686        }
687        Ok(payload.clone())
688    }
689
690    /// Read the DSL operation status for `id`, or `None` if not registered.
691    fn status(&self, id: &OperationId) -> Option<OperationStatus> {
692        let id_key = mm_dsl::OperationId::from_domain(id).0;
693        self.dsl
694            .0
695            .state()
696            .op_statuses
697            .get(&id_key)
698            .copied()
699            .map(OperationStatus::from)
700    }
701
702    fn require_status(&self, id: &OperationId) -> Result<OperationStatus, OpsLifecycleError> {
703        self.status(id).ok_or_else(|| {
704            OpsLifecycleError::Internal(format!(
705                "generated op lifecycle authority missing status for {id}"
706            ))
707        })
708    }
709
710    /// Read the DSL operation kind for `id`, or `None` if not registered.
711    fn kind(&self, id: &OperationId) -> Option<OperationKind> {
712        let id_key = mm_dsl::OperationId::from_domain(id).0;
713        self.dsl
714            .0
715            .state()
716            .op_kinds
717            .get(&id_key)
718            .copied()
719            .map(OperationKind::from)
720    }
721
722    fn require_kind(&self, id: &OperationId) -> Result<OperationKind, OpsLifecycleError> {
723        self.kind(id).ok_or_else(|| {
724            OpsLifecycleError::Internal(format!(
725                "generated op lifecycle authority missing kind for {id}"
726            ))
727        })
728    }
729
730    fn operation_source(
731        &self,
732        id: &OperationId,
733    ) -> Result<Option<OperationSource>, OpsLifecycleError> {
734        let id_key = mm_dsl::OperationId::from_domain(id).0;
735        self.dsl
736            .0
737            .state()
738            .op_sources
739            .get(&id_key)
740            .map(|source| {
741                source.to_domain().map_err(|error| {
742                    OpsLifecycleError::Internal(format!(
743                        "generated operation source authority has invalid source for {id}: {error}"
744                    ))
745                })
746            })
747            .transpose()
748    }
749
750    fn child_session_id_from_operation_source(
751        operation_source: Option<&OperationSource>,
752    ) -> Option<SessionId> {
753        match operation_source {
754            Some(OperationSource::SessionChild { session_id }) => Some(session_id.clone()),
755            Some(OperationSource::BackendPeer { .. } | OperationSource::DetachedJob { .. })
756            | None => None,
757        }
758    }
759
760    fn align_spec_child_session_id_to_source(
761        spec: &mut OperationSpec,
762        operation_source: Option<&OperationSource>,
763    ) {
764        spec.child_session_id = Self::child_session_id_from_operation_source(operation_source);
765    }
766
767    /// Read the peer-ready flag for `id`.
768    fn peer_ready(&self, id: &OperationId) -> Option<bool> {
769        let id_key = mm_dsl::OperationId::from_domain(id).0;
770        self.dsl.0.state().op_peer_ready.get(&id_key).copied()
771    }
772
773    fn require_peer_ready(&self, id: &OperationId) -> Result<bool, OpsLifecycleError> {
774        self.peer_ready(id).ok_or_else(|| {
775            OpsLifecycleError::Internal(format!(
776                "generated op peer wiring authority missing peer-ready fact for {id}"
777            ))
778        })
779    }
780
781    /// Read the progress counter for `id`.
782    fn progress_count(&self, id: &OperationId) -> Option<u32> {
783        let id_key = mm_dsl::OperationId::from_domain(id).0;
784        self.dsl
785            .0
786            .state()
787            .op_progress_counts
788            .get(&id_key)
789            .map(|v| (*v).min(u32::MAX as u64) as u32)
790    }
791
792    fn require_progress_count(&self, id: &OperationId) -> Result<u32, OpsLifecycleError> {
793        self.progress_count(id).ok_or_else(|| {
794            OpsLifecycleError::Internal(format!(
795                "generated op progress authority missing progress count for {id}"
796            ))
797        })
798    }
799
800    /// Read the terminal outcome for `id` by pairing the DSL's typed
801    /// discriminant with the companion payload JSON. Returns `None` when the
802    /// op has no recorded terminal discriminant.
803    fn terminal_outcome(
804        &self,
805        id: &OperationId,
806    ) -> Result<Option<OperationTerminalOutcome>, OpsLifecycleError> {
807        let id_key = mm_dsl::OperationId::from_domain(id).0;
808        let state = self.dsl.0.state();
809        let status = self.status(id);
810        let terminal = match status {
811            Some(status) => Self::operation_status_is_terminal(id, status)?,
812            None => false,
813        };
814        let kind = state.op_terminal_outcomes.get(&id_key).copied();
815        let Some(kind) = kind else {
816            if terminal {
817                return Err(OpsLifecycleError::Internal(format!(
818                    "generated op terminal authority missing terminal outcome for {id}"
819                )));
820            }
821            return Ok(None);
822        };
823        if !terminal {
824            return Err(OpsLifecycleError::Internal(format!(
825                "generated op terminal authority has terminal outcome for non-terminal {id}"
826            )));
827        }
828        let payload = state.op_terminal_payload.get(&id_key).ok_or_else(|| {
829            OpsLifecycleError::Internal(format!(
830                "generated op terminal authority missing terminal payload for {id}"
831            ))
832        })?;
833        Self::checked_terminal_payload(kind, payload, "generated op terminal authority", &id_key)
834            .map(Some)
835    }
836
837    /// Whether the operation is currently tracked in DSL state.
838    fn contains(&self, id: &OperationId) -> bool {
839        let id_key = mm_dsl::OperationId::from_domain(id).0;
840        self.dsl.0.state().op_statuses.contains_key(&id_key)
841    }
842
843    /// Number of non-terminal operations (derived from DSL state).
844    fn active_count(&self) -> usize {
845        self.dsl.0.state().active_op_count as usize
846    }
847
848    /// Number of operations currently tracked (including terminal).
849    fn operation_count(&self) -> usize {
850        self.dsl.0.state().op_statuses.len()
851    }
852
853    /// Iterate over all tracked operation IDs (DSL keys converted to domain).
854    fn operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
855        let mut ids = BTreeSet::new();
856        let state = self.dsl.0.state();
857        Self::collect_operation_id_keys(&mut ids, "op_statuses", state.op_statuses.keys())?;
858        Self::collect_operation_id_keys(&mut ids, "op_kinds", state.op_kinds.keys())?;
859        Self::collect_operation_id_keys(&mut ids, "op_sources", state.op_sources.keys())?;
860        Self::collect_operation_id_keys(&mut ids, "op_peer_ready", state.op_peer_ready.keys())?;
861        Self::collect_operation_id_keys(
862            &mut ids,
863            "op_progress_counts",
864            state.op_progress_counts.keys(),
865        )?;
866        Self::collect_operation_id_keys(
867            &mut ids,
868            "op_terminal_outcomes",
869            state.op_terminal_outcomes.keys(),
870        )?;
871        Self::collect_operation_id_keys(
872            &mut ids,
873            "op_terminal_payload",
874            state.op_terminal_payload.keys(),
875        )?;
876        Self::collect_operation_id_keys(
877            &mut ids,
878            "op_completion_seq",
879            state.op_completion_seq.keys(),
880        )?;
881        ids.extend(self.records.keys().cloned());
882        Ok(ids.into_iter().collect())
883    }
884
885    fn collect_operation_id_keys<'a, I>(
886        ids: &mut BTreeSet<OperationId>,
887        field: &str,
888        keys: I,
889    ) -> Result<(), OpsLifecycleError>
890    where
891        I: IntoIterator<Item = &'a String>,
892    {
893        for key in keys {
894            let id = serde_json::from_str::<OperationId>(key).map_err(|error| {
895                OpsLifecycleError::Internal(format!(
896                    "generated operation identity authority used invalid operation id key in {field}: {key}: {error}"
897                ))
898            })?;
899            ids.insert(id);
900        }
901        Ok(())
902    }
903
904    fn has_generated_operation_record_fact(&self, id: &OperationId) -> bool {
905        let id_key = mm_dsl::OperationId::from_domain(id).0;
906        let state = self.dsl.0.state();
907        state.op_statuses.contains_key(&id_key)
908            || state.op_kinds.contains_key(&id_key)
909            || state.op_sources.contains_key(&id_key)
910            || state.op_peer_ready.contains_key(&id_key)
911            || state.op_progress_counts.contains_key(&id_key)
912            || state.op_terminal_outcomes.contains_key(&id_key)
913            || state.op_terminal_payload.contains_key(&id_key)
914            || state.op_completion_seq.contains_key(&id_key)
915    }
916
917    /// Read the DSL-minted completion sequence for a terminal operation.
918    fn completion_sequence(&self, id: &OperationId) -> Option<CompletionSeq> {
919        let id_key = mm_dsl::OperationId::from_domain(id).0;
920        self.dsl.0.state().op_completion_seq.get(&id_key).copied()
921    }
922
923    fn completion_feed_authority_entries(
924        &self,
925    ) -> Result<HashMap<OperationId, CompletionFeedCanonicalState>, OpsLifecycleError> {
926        let state = self.dsl.0.state();
927        let sequence_keys: BTreeSet<String> =
928            state.completion_feed_sequences.keys().cloned().collect();
929        let companion_domains: [(&str, BTreeSet<String>); 3] = [
930            (
931                "completion_feed_kinds",
932                state.completion_feed_kinds.keys().cloned().collect(),
933            ),
934            (
935                "completion_feed_terminal_outcomes",
936                state
937                    .completion_feed_terminal_outcomes
938                    .keys()
939                    .cloned()
940                    .collect(),
941            ),
942            (
943                "completion_feed_terminal_payload",
944                state
945                    .completion_feed_terminal_payload
946                    .keys()
947                    .cloned()
948                    .collect(),
949            ),
950        ];
951        for (field, keys) in companion_domains {
952            if keys != sequence_keys {
953                return Err(OpsLifecycleError::Internal(format!(
954                    "generated completion feed authority has mismatched {field} domain"
955                )));
956            }
957        }
958
959        let mut entries = HashMap::new();
960        for (id_key, seq) in &state.completion_feed_sequences {
961            if !state.completion_sequence_claims.contains(seq) {
962                return Err(OpsLifecycleError::Internal(format!(
963                    "generated completion feed authority sequence {seq} for {id_key} is not claimed"
964                )));
965            }
966            let operation_id = serde_json::from_str::<OperationId>(id_key).map_err(|error| {
967                OpsLifecycleError::Internal(format!(
968                    "generated completion feed authority used invalid operation id key {id_key}: {error}"
969                ))
970            })?;
971            let kind = state
972                .completion_feed_kinds
973                .get(id_key)
974                .copied()
975                .map(OperationKind::from)
976                .ok_or_else(|| {
977                    OpsLifecycleError::Internal(format!(
978                        "generated completion feed authority missing kind for {id_key}"
979                    ))
980                })?;
981            let outcome_kind = state
982                .completion_feed_terminal_outcomes
983                .get(id_key)
984                .copied()
985                .ok_or_else(|| {
986                    OpsLifecycleError::Internal(format!(
987                        "generated completion feed authority missing terminal outcome for {id_key}"
988                    ))
989                })?;
990            let payload = state
991                .completion_feed_terminal_payload
992                .get(id_key)
993                .ok_or_else(|| {
994                    OpsLifecycleError::Internal(format!(
995                        "generated completion feed authority missing terminal payload for {id_key}"
996                    ))
997                })?;
998            let terminal_outcome = Self::checked_terminal_payload(
999                outcome_kind,
1000                payload,
1001                "generated completion feed authority",
1002                id_key,
1003            )?;
1004            entries.insert(
1005                operation_id,
1006                CompletionFeedCanonicalState {
1007                    seq: *seq,
1008                    kind,
1009                    terminal_outcome,
1010                },
1011            );
1012        }
1013        Ok(entries)
1014    }
1015
1016    fn completion_cursor(&self, consumer: CompletionCursorConsumer) -> CompletionSeq {
1017        let state = self.dsl.0.state();
1018        match consumer {
1019            CompletionCursorConsumer::AgentApplied => state.completion_agent_applied_cursor,
1020            CompletionCursorConsumer::RuntimeObserved => state.completion_runtime_observed_cursor,
1021            CompletionCursorConsumer::RuntimeInjected => state.completion_runtime_injected_cursor,
1022        }
1023    }
1024
1025    fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
1026        meerkat_core::EpochCursorSnapshot {
1027            agent_applied_cursor: self.completion_cursor(CompletionCursorConsumer::AgentApplied),
1028            runtime_observed_seq: self.completion_cursor(CompletionCursorConsumer::RuntimeObserved),
1029            runtime_last_injected_seq: self
1030                .completion_cursor(CompletionCursorConsumer::RuntimeInjected),
1031        }
1032    }
1033
1034    /// Build a snapshot from DSL state + shell record.
1035    fn snapshot(
1036        &self,
1037        id: &OperationId,
1038    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
1039        let Some(shell) = self.records.get(id) else {
1040            if self.has_generated_operation_record_fact(id) {
1041                return Err(OpsLifecycleError::Internal(format!(
1042                    "generated op lifecycle authority has operation facts without shell projection record for {id}"
1043                )));
1044            }
1045            return Ok(None);
1046        };
1047        let kind = self.require_kind(id)?;
1048        let status = self.require_status(id)?;
1049        let terminal = Self::operation_status_is_terminal(id, status)?;
1050        let public_result_class = Self::operation_public_result_class(id, status)?;
1051        let peer_ready = self.require_peer_ready(id)?;
1052        let progress_count = self.require_progress_count(id)?;
1053        let operation_source = self.operation_source(id)?;
1054        let terminal_outcome = self.terminal_outcome(id)?;
1055
1056        let created_at_ms = ShellRecord::epoch_millis(&shell.created_at_wall);
1057        let started_at_ms = shell.started_at.map(|i| shell.epoch_millis_for_instant(i));
1058        let completed_at_ms = shell
1059            .completed_at
1060            .map(|i| shell.epoch_millis_for_instant(i));
1061        let elapsed_ms = shell.completed_at.map(|completed| {
1062            completed
1063                .saturating_duration_since(shell.created_at)
1064                .as_millis() as u64
1065        });
1066
1067        Ok(Some(OperationLifecycleSnapshot {
1068            id: shell.spec.id.clone(),
1069            kind,
1070            owner_session_id: shell.spec.owner_session_id.clone(),
1071            display_name: shell.spec.display_name.clone(),
1072            source_label: shell.spec.source_label.clone(),
1073            child_session_id: Self::child_session_id_from_operation_source(
1074                operation_source.as_ref(),
1075            ),
1076            operation_source,
1077            expect_peer_channel: shell.spec.expect_peer_channel,
1078            status,
1079            terminal,
1080            public_result_class,
1081            peer_ready,
1082            progress_count,
1083            watcher_count: shell.watchers.len() as u32,
1084            terminal_outcome,
1085            peer_handle: shell.peer_handle.clone(),
1086            created_at_ms,
1087            started_at_ms,
1088            completed_at_ms,
1089            elapsed_ms,
1090        }))
1091    }
1092
1093    /// Emit shell-side mechanics for a terminal transition: notify watchers,
1094    /// push generated-authorized CompletionEntry rows, retain in FIFO, evict
1095    /// as needed. Called AFTER the DSL transition has already persisted the
1096    /// terminal status + outcome.
1097    fn finalize_terminal(
1098        &mut self,
1099        id: &OperationId,
1100    ) -> Result<Option<CompletionEntry>, OpsLifecycleError> {
1101        let outcome = self.terminal_outcome(id)?.ok_or_else(|| {
1102            OpsLifecycleError::Internal(format!(
1103                "generated op terminal transition did not mint terminal outcome for {id}"
1104            ))
1105        })?;
1106        let kind = self.require_kind(id)?;
1107
1108        // Notify watchers and mark completion timestamp.
1109        if let Some(shell) = self.records.get_mut(id) {
1110            shell.notify_watchers(&outcome);
1111            shell.mark_completed();
1112        }
1113
1114        if Self::operation_durability_class(id, kind)? == mm_dsl::OperationDurabilityClass::Discard
1115        {
1116            self.dsl_apply(
1117                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
1118                    operation_id: mm_dsl::OperationId::from_domain(id).0,
1119                },
1120                "CollectCompletedOp",
1121            )?;
1122            self.records.remove(id);
1123            self.completed_order.retain(|queued| queued != id);
1124            return Ok(None);
1125        }
1126
1127        let mut completion_entry = None;
1128        if Self::operation_completion_feed_class(id, kind)?
1129            == mm_dsl::OperationCompletionFeedClass::Emit
1130        {
1131            let feed_authority = self
1132                .completion_feed_authority_entries()?
1133                .remove(id)
1134                .ok_or_else(|| {
1135                    OpsLifecycleError::Internal(format!(
1136                        "generated op terminal transition did not mint completion feed authority for {id}"
1137                    ))
1138                })?;
1139            if feed_authority.kind != kind || feed_authority.terminal_outcome != outcome {
1140                return Err(OpsLifecycleError::Internal(format!(
1141                    "generated completion feed authority drifted from terminal op authority for {id}"
1142                )));
1143            }
1144            let seq = self.completion_sequence(id).ok_or_else(|| {
1145                OpsLifecycleError::Internal(format!(
1146                    "generated op terminal transition did not mint completion sequence for {id}"
1147                ))
1148            })?;
1149            if feed_authority.seq != seq {
1150                return Err(OpsLifecycleError::Internal(format!(
1151                    "generated completion feed authority sequence drifted for {id}"
1152                )));
1153            }
1154            let display_name = self
1155                .records
1156                .get(id)
1157                .map(|r| r.spec.display_name.clone())
1158                .unwrap_or_default();
1159            let completed_at_ms = self
1160                .records
1161                .get(id)
1162                .and_then(|r| r.completed_at.map(|i| r.epoch_millis_for_instant(i)));
1163            completion_entry = Some(CompletionEntry {
1164                seq: feed_authority.seq,
1165                operation_id: id.clone(),
1166                kind: feed_authority.kind,
1167                display_name,
1168                terminal_outcome: feed_authority.terminal_outcome,
1169                completed_at_ms,
1170            });
1171        }
1172
1173        // FIFO retention + eviction.
1174        self.completed_order.push_back(id.clone());
1175        while self.completed_order.len() > self.max_completed {
1176            if let Some(evicted) = self.completed_order.pop_front() {
1177                self.dsl_apply(
1178                    mm_dsl::MeerkatMachineInput::EvictCompletedOp {
1179                        operation_id: mm_dsl::OperationId::from_domain(&evicted).0,
1180                    },
1181                    "EvictCompletedOp",
1182                )?;
1183                self.records.remove(&evicted);
1184            }
1185        }
1186
1187        // Satisfy a pending wait request if all its ops are now terminal. On
1188        // authority-invariant corruption this propagates the typed fault (and
1189        // drops the barrier oneshot so the waiter resolves to Err) instead of
1190        // reporting the op terminal with a silently-hung barrier.
1191        self.maybe_satisfy_wait()?;
1192        Ok(completion_entry)
1193    }
1194
1195    fn publish_completion_entry(&self, entry: Option<CompletionEntry>) {
1196        if let Some(entry) = entry {
1197            self.feed_buffer.push(entry);
1198        }
1199    }
1200
1201    /// Read barrier membership from DSL state (sole owner).
1202    fn wait_operation_ids(&self) -> Result<Vec<OperationId>, OpsLifecycleError> {
1203        self.dsl
1204            .0
1205            .state()
1206            .wait_operation_ids
1207            .iter()
1208            .map(|key| {
1209                serde_json::from_str::<OperationId>(key).map_err(|error| {
1210                    OpsLifecycleError::Internal(format!(
1211                        "generated wait operation identity authority used invalid operation id key {key}: {error}"
1212                    ))
1213                })
1214            })
1215            .collect()
1216    }
1217
1218    /// Whether the DSL has a barrier wait active.
1219    #[cfg(test)]
1220    fn wait_active(&self) -> bool {
1221        self.dsl.0.state().wait_active
1222    }
1223
1224    fn wait_all_satisfied_from_effects(
1225        effects: &[mm_dsl::MeerkatMachineEffect],
1226    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1227        let mut satisfied = None;
1228        for effect in effects {
1229            let mm_dsl::MeerkatMachineEffect::WaitAllSatisfied {
1230                wait_request_id,
1231                run_id,
1232                operation_ids,
1233            } = effect
1234            else {
1235                continue;
1236            };
1237            if satisfied.is_some() {
1238                return Err(OpsLifecycleError::Internal(
1239                    "generated wait_all authority emitted multiple satisfaction effects".into(),
1240                ));
1241            }
1242            let wait_uuid = uuid::Uuid::parse_str(&wait_request_id.0).map_err(|err| {
1243                OpsLifecycleError::Internal(format!(
1244                    "generated wait_all authority emitted invalid wait request id '{}': {err}",
1245                    wait_request_id.0
1246                ))
1247            })?;
1248            let mut ids = Vec::with_capacity(operation_ids.len());
1249            for operation_id in operation_ids {
1250                ids.push(
1251                    serde_json::from_str::<OperationId>(&operation_id.0).map_err(|err| {
1252                        OpsLifecycleError::Internal(format!(
1253                            "generated wait_all authority emitted invalid operation id '{}': {err}",
1254                            operation_id.0
1255                        ))
1256                    })?,
1257                );
1258            }
1259            satisfied = Some(WaitAllSatisfied {
1260                wait_request_id: WaitRequestId::from_uuid(wait_uuid),
1261                run_id: RunId::from_uuid(uuid::Uuid::parse_str(&run_id.0).map_err(|err| {
1262                    OpsLifecycleError::Internal(format!(
1263                        "generated wait_all authority emitted invalid run id '{}': {err}",
1264                        run_id.0
1265                    ))
1266                })?),
1267                operation_ids: ids,
1268            });
1269        }
1270        Ok(satisfied)
1271    }
1272
1273    fn parse_wait_all_operation_id(
1274        raw: &str,
1275        context: &str,
1276    ) -> Result<OperationId, OpsLifecycleError> {
1277        serde_json::from_str::<OperationId>(raw).map_err(|err| {
1278            OpsLifecycleError::Internal(format!(
1279                "generated wait_all authority emitted invalid {context} operation id '{raw}': {err}"
1280            ))
1281        })
1282    }
1283
1284    fn duplicate_wait_operation_id(operation_ids: &[OperationId]) -> Option<OperationId> {
1285        let mut seen = HashSet::new();
1286        operation_ids
1287            .iter()
1288            .find(|operation_id| !seen.insert((*operation_id).clone()))
1289            .cloned()
1290    }
1291
1292    fn wait_all_admission_error_from_effects(
1293        wait_request_id: &WaitRequestId,
1294        effects: &[mm_dsl::MeerkatMachineEffect],
1295    ) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
1296        let mut admission = None;
1297        for effect in effects {
1298            let mm_dsl::MeerkatMachineEffect::WaitAllAdmissionResolved {
1299                wait_request_id: resolved_wait_request_id,
1300                result,
1301                reject_reason,
1302                rejected_operation_id,
1303            } = effect
1304            else {
1305                continue;
1306            };
1307            if admission.is_some() {
1308                return Err(OpsLifecycleError::Internal(
1309                    "generated wait_all authority emitted multiple admission results".into(),
1310                ));
1311            }
1312            let resolved_uuid =
1313                uuid::Uuid::parse_str(&resolved_wait_request_id.0).map_err(|err| {
1314                    OpsLifecycleError::Internal(format!(
1315                        "generated wait_all authority emitted invalid wait request id '{}': {err}",
1316                        resolved_wait_request_id.0
1317                    ))
1318                })?;
1319            let resolved_wait_request_id = WaitRequestId::from_uuid(resolved_uuid);
1320            if &resolved_wait_request_id != wait_request_id {
1321                return Err(OpsLifecycleError::Internal(format!(
1322                    "generated wait_all authority resolved wait request {resolved_wait_request_id} while shell requested {wait_request_id}"
1323                )));
1324            }
1325            admission = Some(match result {
1326                mm_dsl::WaitAllAdmissionResultKind::Accept => {
1327                    if reject_reason.is_some() || rejected_operation_id.is_some() {
1328                        return Err(OpsLifecycleError::Internal(
1329                            "generated wait_all authority accepted with rejection payload".into(),
1330                        ));
1331                    }
1332                    None
1333                }
1334                mm_dsl::WaitAllAdmissionResultKind::Reject => {
1335                    let reason = reject_reason.ok_or_else(|| {
1336                        OpsLifecycleError::Internal(
1337                            "generated wait_all authority rejected without reason".into(),
1338                        )
1339                    })?;
1340                    let error = match reason {
1341                        mm_dsl::WaitAllRejectReasonKind::DuplicateOperation => {
1342                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1343                                OpsLifecycleError::Internal(
1344                                    "generated wait_all authority rejected duplicate without operation id"
1345                                        .into(),
1346                                )
1347                            })?;
1348                            OpsLifecycleError::DuplicateWaitOperation(
1349                                Self::parse_wait_all_operation_id(raw, "duplicate")?,
1350                            )
1351                        }
1352                        mm_dsl::WaitAllRejectReasonKind::WaitAlreadyActive => {
1353                            if rejected_operation_id.is_some() {
1354                                return Err(OpsLifecycleError::Internal(
1355                                    "generated wait_all authority rejected active wait with operation id"
1356                                        .into(),
1357                                ));
1358                            }
1359                            OpsLifecycleError::WaitAlreadyActive
1360                        }
1361                        mm_dsl::WaitAllRejectReasonKind::OperationNotFound => {
1362                            let raw = rejected_operation_id.as_deref().ok_or_else(|| {
1363                                OpsLifecycleError::Internal(
1364                                    "generated wait_all authority rejected missing operation without operation id"
1365                                        .into(),
1366                                )
1367                            })?;
1368                            OpsLifecycleError::NotFound(Self::parse_wait_all_operation_id(
1369                                raw, "missing",
1370                            )?)
1371                        }
1372                    };
1373                    Some(error)
1374                }
1375            });
1376        }
1377        admission.ok_or_else(|| {
1378            OpsLifecycleError::Internal(
1379                "generated wait_all authority emitted no admission result".into(),
1380            )
1381        })
1382    }
1383
1384    fn resolve_wait_all_admission(
1385        &mut self,
1386        wait_request_id: &WaitRequestId,
1387        operation_ids: &[OperationId],
1388        dsl_ids: &BTreeSet<String>,
1389        dsl_id_tokens: &BTreeSet<mm_dsl::OperationId>,
1390        operation_token_by_id: &BTreeMap<String, mm_dsl::OperationId>,
1391        operation_id_by_token: &BTreeMap<mm_dsl::OperationId, String>,
1392    ) -> Result<(), OpsLifecycleError> {
1393        let duplicate = Self::duplicate_wait_operation_id(operation_ids)
1394            .map(|operation_id| mm_dsl::OperationId::from_domain(&operation_id).0);
1395        let not_found = operation_ids
1396            .iter()
1397            .find(|operation_id| !self.contains(operation_id))
1398            .map(|operation_id| mm_dsl::OperationId::from_domain(operation_id).0);
1399        let dsl_id_sequence: Vec<String> = operation_ids
1400            .iter()
1401            .map(|id| mm_dsl::OperationId::from_domain(id).0)
1402            .collect();
1403        let effects = self.dsl_apply_with_effects(
1404            mm_dsl::MeerkatMachineInput::ResolveWaitAllAdmission {
1405                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1406                operation_id_sequence: dsl_id_sequence,
1407                operation_ids: dsl_ids.clone(),
1408                operation_id_tokens: dsl_id_tokens.clone(),
1409                operation_token_by_id: operation_token_by_id.clone(),
1410                operation_id_by_token: operation_id_by_token.clone(),
1411                duplicate_operation_id: duplicate,
1412                not_found_operation_id: not_found,
1413            },
1414            "ResolveWaitAllAdmission",
1415        )?;
1416        if let Some(error) = Self::wait_all_admission_error_from_effects(wait_request_id, &effects)?
1417        {
1418            return Err(error);
1419        }
1420        Ok(())
1421    }
1422
1423    fn try_satisfy_wait_all_authority(
1424        &mut self,
1425    ) -> Result<Option<WaitAllSatisfied>, OpsLifecycleError> {
1426        let Some(dsl_wait_request_id) = self.dsl.0.state().wait_request_id.clone() else {
1427            return Ok(None);
1428        };
1429        let Some(dsl_run_id) = self.dsl.0.state().wait_run_id.clone() else {
1430            return Err(OpsLifecycleError::Internal(
1431                "generated wait_all authority has active wait without run id".into(),
1432            ));
1433        };
1434        let dsl_operation_id_tokens = self.dsl.0.state().wait_operation_id_tokens.clone();
1435        let transition = match mm_dsl::MeerkatMachineMutator::apply(
1436            &mut *self.dsl.0,
1437            mm_dsl::MeerkatMachineInput::SatisfyWaitAll {
1438                wait_request_id: dsl_wait_request_id,
1439                run_id: dsl_run_id,
1440                operation_id_tokens: dsl_operation_id_tokens,
1441            },
1442        ) {
1443            Ok(transition) => transition,
1444            Err(mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. }) => return Ok(None),
1445            Err(err) => {
1446                return Err(OpsLifecycleError::Internal(format!(
1447                    "DSL rejected ops transition (SatisfyWaitAll): {err:?}"
1448                )));
1449            }
1450        };
1451        Self::wait_all_satisfied_from_effects(transition.effects())?
1452            .ok_or_else(|| {
1453                OpsLifecycleError::Internal(
1454                    "generated wait_all authority accepted satisfaction without effect".into(),
1455                )
1456            })
1457            .map(Some)
1458    }
1459
1460    fn begin_wait_all_authority(
1461        &mut self,
1462        run_id: &RunId,
1463        wait_request_id: &WaitRequestId,
1464        operation_ids: &[OperationId],
1465    ) -> Result<WaitAllAuthorityPlan, OpsLifecycleError> {
1466        let mut dsl_ids = BTreeSet::new();
1467        let mut dsl_id_tokens = BTreeSet::new();
1468        let mut operation_token_by_id = BTreeMap::new();
1469        let mut operation_id_by_token = BTreeMap::new();
1470        for id in operation_ids {
1471            let token = mm_dsl::OperationId::from_domain(id);
1472            let raw_id = token.0.clone();
1473            dsl_ids.insert(raw_id.clone());
1474            dsl_id_tokens.insert(token.clone());
1475            operation_token_by_id.insert(raw_id.clone(), token.clone());
1476            operation_id_by_token.insert(token, raw_id);
1477        }
1478        self.resolve_wait_all_admission(
1479            wait_request_id,
1480            operation_ids,
1481            &dsl_ids,
1482            &dsl_id_tokens,
1483            &operation_token_by_id,
1484            &operation_id_by_token,
1485        )?;
1486        self.dsl_apply(
1487            mm_dsl::MeerkatMachineInput::RequestWaitAll {
1488                run_id: mm_dsl::RunId::from_domain(run_id),
1489                wait_request_id: mm_dsl::WaitRequestId::from_domain(wait_request_id),
1490                operation_id_sequence: operation_ids
1491                    .iter()
1492                    .map(|id| mm_dsl::OperationId::from_domain(id).0)
1493                    .collect(),
1494                operation_ids: dsl_ids,
1495                operation_id_tokens: dsl_id_tokens,
1496                operation_token_by_id,
1497                operation_id_by_token,
1498            },
1499            "RequestWaitAll",
1500        )?;
1501        if let Some(satisfied) = self.try_satisfy_wait_all_authority()? {
1502            return Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied));
1503        }
1504        Ok(WaitAllAuthorityPlan::ActivateBarrier)
1505    }
1506
1507    fn owner_termination_targets(
1508        &self,
1509    ) -> Result<Vec<(OperationId, OperationStatus)>, OpsLifecycleError> {
1510        let mut targets = Vec::new();
1511        for id in self.operation_ids()? {
1512            let status = self.require_status(&id)?;
1513            if !Self::operation_status_is_terminal(&id, status)? {
1514                targets.push((id, status));
1515            }
1516        }
1517        Ok(targets)
1518    }
1519
1520    fn operation_status_is_terminal(
1521        operation_id: &OperationId,
1522        status: OperationStatus,
1523    ) -> Result<bool, OpsLifecycleError> {
1524        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1525        let effects = Self::apply_stateless_classifier(
1526            mm_dsl::MeerkatMachineInput::ClassifyOperationTerminality {
1527                operation_id: operation_id_key.clone(),
1528                status: mm_dsl::OperationStatus::from(status),
1529            },
1530            "ClassifyOperationTerminality",
1531        )?;
1532        let mut terminal = None;
1533        for effect in effects {
1534            match effect {
1535                mm_dsl::MeerkatMachineEffect::OperationTerminal { operation_id }
1536                    if operation_id == operation_id_key =>
1537                {
1538                    terminal = Some(true);
1539                }
1540                mm_dsl::MeerkatMachineEffect::OperationNonTerminal { operation_id }
1541                    if operation_id == operation_id_key =>
1542                {
1543                    terminal = Some(false);
1544                }
1545                other => {
1546                    return Err(OpsLifecycleError::Internal(format!(
1547                        "unexpected generated operation terminality effect: {other:?}"
1548                    )));
1549                }
1550            }
1551        }
1552        terminal.ok_or_else(|| {
1553            OpsLifecycleError::Internal(format!(
1554                "generated operation terminality authority emitted no effect for {operation_id}"
1555            ))
1556        })
1557    }
1558
1559    fn operation_public_result_class(
1560        operation_id: &OperationId,
1561        status: OperationStatus,
1562    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
1563        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1564        let effects = Self::apply_stateless_classifier(
1565            mm_dsl::MeerkatMachineInput::ClassifyOperationPublicResult {
1566                operation_id: operation_id_key.clone(),
1567                status: mm_dsl::OperationStatus::from(status),
1568            },
1569            "ClassifyOperationPublicResult",
1570        )?;
1571        let mut result = None;
1572        for effect in effects {
1573            match effect {
1574                mm_dsl::MeerkatMachineEffect::OperationPublicResultClassified {
1575                    operation_id,
1576                    result: classified,
1577                } if operation_id == operation_id_key => {
1578                    result = Some(OperationPublicResultClass::from(classified));
1579                }
1580                other => {
1581                    return Err(OpsLifecycleError::Internal(format!(
1582                        "unexpected generated operation public-result effect: {other:?}"
1583                    )));
1584                }
1585            }
1586        }
1587        result.ok_or_else(|| {
1588            OpsLifecycleError::Internal(format!(
1589                "generated operation public-result authority emitted no effect for {operation_id}"
1590            ))
1591        })
1592    }
1593
1594    fn operation_transition_rejection_is_idempotent(
1595        operation_id: &OperationId,
1596        action: OperationLifecycleAction,
1597        status: OperationStatus,
1598    ) -> Result<bool, OpsLifecycleError> {
1599        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1600        let action = mm_dsl::OpLifecycleActionKind::from(action);
1601        let status = mm_dsl::OperationStatus::from(status);
1602        let effects = Self::apply_stateless_classifier(
1603            mm_dsl::MeerkatMachineInput::ClassifyOperationTransitionIdempotence {
1604                operation_id: operation_id_key.clone(),
1605                action,
1606                status,
1607            },
1608            "ClassifyOperationTransitionIdempotence",
1609        )?;
1610        let mut idempotent = None;
1611        for effect in effects {
1612            match effect {
1613                mm_dsl::MeerkatMachineEffect::OperationTransitionIdempotentSuccess {
1614                    operation_id,
1615                    action: effect_action,
1616                    status: effect_status,
1617                } if operation_id == operation_id_key
1618                    && effect_action == action
1619                    && effect_status == status =>
1620                {
1621                    idempotent = Some(true);
1622                }
1623                mm_dsl::MeerkatMachineEffect::OperationTransitionNotIdempotent {
1624                    operation_id,
1625                    action: effect_action,
1626                    status: effect_status,
1627                } if operation_id == operation_id_key
1628                    && effect_action == action
1629                    && effect_status == status =>
1630                {
1631                    idempotent = Some(false);
1632                }
1633                other => {
1634                    return Err(OpsLifecycleError::Internal(format!(
1635                        "unexpected generated operation transition-idempotence effect: {other:?}"
1636                    )));
1637                }
1638            }
1639        }
1640        idempotent.ok_or_else(|| {
1641            OpsLifecycleError::Internal(format!(
1642                "generated operation transition-idempotence authority emitted no effect for {operation_id}"
1643            ))
1644        })
1645    }
1646
1647    fn operation_completion_feed_class(
1648        operation_id: &OperationId,
1649        kind: OperationKind,
1650    ) -> Result<mm_dsl::OperationCompletionFeedClass, OpsLifecycleError> {
1651        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1652        let kind = mm_dsl::OperationKind::from(kind);
1653        let effects = Self::apply_stateless_classifier(
1654            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionFeed {
1655                operation_id: operation_id_key.clone(),
1656                kind,
1657            },
1658            "ClassifyOperationCompletionFeed",
1659        )?;
1660        let mut class = None;
1661        for effect in effects {
1662            match effect {
1663                mm_dsl::MeerkatMachineEffect::OperationCompletionFeedClassified {
1664                    operation_id,
1665                    result,
1666                } if operation_id == operation_id_key => {
1667                    class = Some(result);
1668                }
1669                other => {
1670                    return Err(OpsLifecycleError::Internal(format!(
1671                        "unexpected generated operation completion-feed effect: {other:?}"
1672                    )));
1673                }
1674            }
1675        }
1676        class.ok_or_else(|| {
1677            OpsLifecycleError::Internal(format!(
1678                "generated operation completion-feed authority emitted no effect for {operation_id}"
1679            ))
1680        })
1681    }
1682
1683    fn operation_completion_wake_class(
1684        operation_id: &OperationId,
1685        kind: OperationKind,
1686    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
1687        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1688        let kind = mm_dsl::OperationKind::from(kind);
1689        let effects = Self::apply_stateless_classifier(
1690            mm_dsl::MeerkatMachineInput::ClassifyOperationCompletionWake {
1691                operation_id: operation_id_key.clone(),
1692                kind,
1693            },
1694            "ClassifyOperationCompletionWake",
1695        )?;
1696        let mut class = None;
1697        for effect in effects {
1698            match effect {
1699                mm_dsl::MeerkatMachineEffect::OperationCompletionWakeClassified {
1700                    operation_id,
1701                    result,
1702                } if operation_id == operation_id_key => {
1703                    class = Some(OperationCompletionWakeClass::from(result));
1704                }
1705                other => {
1706                    return Err(OpsLifecycleError::Internal(format!(
1707                        "unexpected generated operation completion-wake effect: {other:?}"
1708                    )));
1709                }
1710            }
1711        }
1712        class.ok_or_else(|| {
1713            OpsLifecycleError::Internal(format!(
1714                "generated operation completion-wake authority emitted no effect for {operation_id}"
1715            ))
1716        })
1717    }
1718
1719    fn operation_durability_class(
1720        operation_id: &OperationId,
1721        kind: OperationKind,
1722    ) -> Result<mm_dsl::OperationDurabilityClass, OpsLifecycleError> {
1723        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1724        let kind = mm_dsl::OperationKind::from(kind);
1725        let effects = Self::apply_stateless_classifier(
1726            mm_dsl::MeerkatMachineInput::ClassifyOperationDurability {
1727                operation_id: operation_id_key.clone(),
1728                kind,
1729            },
1730            "ClassifyOperationDurability",
1731        )?;
1732        let mut class = None;
1733        for effect in effects {
1734            match effect {
1735                mm_dsl::MeerkatMachineEffect::OperationDurabilityClassified {
1736                    operation_id,
1737                    result,
1738                } if operation_id == operation_id_key => {
1739                    class = Some(result);
1740                }
1741                other => {
1742                    return Err(OpsLifecycleError::Internal(format!(
1743                        "unexpected generated operation durability effect: {other:?}"
1744                    )));
1745                }
1746            }
1747        }
1748        class.ok_or_else(|| {
1749            OpsLifecycleError::Internal(format!(
1750                "generated operation durability authority emitted no effect for {operation_id}"
1751            ))
1752        })
1753    }
1754
1755    fn recovered_operation_record_disposition(
1756        operation_id: &OperationId,
1757        status: OperationStatus,
1758        kind: OperationKind,
1759        terminal_outcome_present: bool,
1760        terminal_payload_present: bool,
1761        completion_sequence_present: bool,
1762    ) -> Result<RecoveredOperationRecordDisposition, OpsLifecycleError> {
1763        let operation_id_key = mm_dsl::OperationId::from_domain(operation_id).0;
1764        let effects = Self::apply_stateless_classifier(
1765            mm_dsl::MeerkatMachineInput::ClassifyRecoveredOperationRecord {
1766                operation_id: operation_id_key.clone(),
1767                status: mm_dsl::OperationStatus::from(status),
1768                kind: mm_dsl::OperationKind::from(kind),
1769                terminal_outcome_present,
1770                terminal_payload_present,
1771                completion_sequence_present,
1772            },
1773            "ClassifyRecoveredOperationRecord",
1774        )?;
1775        let mut disposition = None;
1776        for effect in effects {
1777            match effect {
1778                mm_dsl::MeerkatMachineEffect::RetainTerminalRecord { operation_id }
1779                    if operation_id == operation_id_key =>
1780                {
1781                    disposition = Some(RecoveredOperationRecordDisposition::Retain);
1782                }
1783                mm_dsl::MeerkatMachineEffect::DiscardRecoveredOperationRecord { operation_id }
1784                    if operation_id == operation_id_key =>
1785                {
1786                    disposition = Some(RecoveredOperationRecordDisposition::Discard);
1787                }
1788                other => {
1789                    return Err(OpsLifecycleError::Internal(format!(
1790                        "unexpected generated recovered-operation classification effect: {other:?}"
1791                    )));
1792                }
1793            }
1794        }
1795        disposition.ok_or_else(|| {
1796            OpsLifecycleError::Internal(format!(
1797                "generated recovered-operation classifier emitted no effect for {operation_id}"
1798            ))
1799        })
1800    }
1801
1802    fn apply_stateless_classifier(
1803        input: mm_dsl::MeerkatMachineInput,
1804        label: &'static str,
1805    ) -> Result<Vec<mm_dsl::MeerkatMachineEffect>, OpsLifecycleError> {
1806        let mut authority = crate::meerkat_machine::dsl_authority::new_initialized_authority(
1807            "ops stateless classifier Initialize must be accepted",
1808        );
1809        let transition =
1810            mm_dsl::MeerkatMachineMutator::apply(&mut authority, input).map_err(|err| {
1811                OpsLifecycleError::Internal(format!(
1812                    "DSL rejected ops transition ({label}): {err:?}"
1813                ))
1814            })?;
1815        Ok(transition.into_effects())
1816    }
1817
1818    /// Check whether a pending barrier wait is now satisfied and resolve it.
1819    ///
1820    /// Barrier membership and the "all members terminal" decision both live
1821    /// in the DSL: `wait_operation_ids` carries the set, `wait_active`
1822    /// signals a pending barrier, and `SatisfyWaitAll`'s
1823    /// `all_members_terminal` guard owns the fixed-point test. The shell
1824    /// echoes the DSL-owned request id and typed operation tokens into
1825    /// `SatisfyWaitAll` so the transition can clear the barrier before
1826    /// rendering the handoff effect. It still fires idempotently on every
1827    /// terminal transition and lets the DSL guard reject early firings as a
1828    /// no-op. On acceptance (transition returns `Ok`), the shell selects the
1829    /// correlated oneshot and delivers the `WaitAllSatisfied` obligation
1830    /// token.
1831    ///
1832    /// `wait_request_id` is the shell-owned oneshot correlation id that
1833    /// selects which sender to notify; when the DSL barrier satisfies
1834    /// without a live correlation (post-recovery, or duplicate resolution),
1835    /// the oneshot simply remains pending. That benign no-correlation case is
1836    /// the only path that leaves the oneshot pending: on authority-invariant
1837    /// corruption the pending sender is dropped so the waiter's
1838    /// `WaitAllFuture` resolves to `Err` and the typed
1839    /// [`OpsLifecycleError`] propagates to the terminal transition caller
1840    /// instead of reporting the op complete with a silently-hung barrier.
1841    fn maybe_satisfy_wait(&mut self) -> Result<(), OpsLifecycleError> {
1842        let satisfied = match self.try_satisfy_wait_all_authority() {
1843            Ok(Some(satisfied)) => satisfied,
1844            Ok(None) => return Ok(()),
1845            Err(err) => {
1846                // Authority-invariant corruption: do NOT report the op complete
1847                // with a silently-pending barrier oneshot. Drop the sender so
1848                // the waiter's `WaitAllFuture` resolves to `Err` via its
1849                // `Poll::Ready(Err(_))` arm (the same mechanism
1850                // `cancel_wait_all_internal` uses), then propagate the typed
1851                // fault. The invariant is already broken, so we do not attempt
1852                // a `CancelWaitAll` rollback on the corrupt machine.
1853                if let Some(pending) = self.pending_wait.take() {
1854                    drop(pending.sender);
1855                }
1856                self.wait_request_id = None;
1857                return Err(err);
1858            }
1859        };
1860        let shell_wait_id = self.wait_request_id.take();
1861        if shell_wait_id
1862            .as_ref()
1863            .is_some_and(|id| id != &satisfied.wait_request_id)
1864        {
1865            tracing::error!(
1866                shell_wait_request_id = ?shell_wait_id,
1867                authority_wait_request_id = %satisfied.wait_request_id,
1868                "generated wait_all authority satisfied a different wait request"
1869            );
1870        }
1871        if let Some(pending) = self.pending_wait.take() {
1872            if pending.wait_request_id == satisfied.wait_request_id {
1873                let _ = pending.sender.send(satisfied);
1874            } else if let Some(shell_wait_id) = shell_wait_id {
1875                tracing::error!(
1876                    shell_wait_request_id = %shell_wait_id,
1877                    pending_wait_request_id = %pending.wait_request_id,
1878                    authority_wait_request_id = %satisfied.wait_request_id,
1879                    "generated wait_all authority satisfied without a matching pending waiter"
1880                );
1881            }
1882        }
1883        Ok(())
1884    }
1885
1886    /// Persist a terminal snapshot if a persistence channel is wired.
1887    ///
1888    /// Called after terminal transitions. Captures authority + entries + cursors under the write
1889    /// lock (caller already holds it), submits a persistence request, and waits for the worker's
1890    /// durable-store result. Returning success only means the snapshot write itself succeeded.
1891    fn maybe_persist(&self) -> Result<(), OpsLifecycleError> {
1892        let (tx, epoch_id, cursor_state) = match (
1893            &self.persist_tx,
1894            &self.persist_epoch_id,
1895            &self.persist_cursor_state,
1896        ) {
1897            (Some(tx), Some(epoch_id), Some(cs)) => (tx, epoch_id, cs),
1898            _ => return Ok(()),
1899        };
1900
1901        let snapshot = self.capture_snapshot(epoch_id.clone(), cursor_state)?;
1902        let (result_tx, result_rx) = std::sync::mpsc::sync_channel(1);
1903        let request = OpsLifecyclePersistenceRequest {
1904            snapshot,
1905            result_tx,
1906        };
1907
1908        tx.send(request).map_err(|_| {
1909            OpsLifecycleError::Internal(
1910                "ops lifecycle persistence channel closed before terminal snapshot could be queued"
1911                    .into(),
1912            )
1913        })?;
1914        result_rx.recv().map_err(|_| {
1915            OpsLifecycleError::Internal(
1916                "ops lifecycle persistence worker dropped terminal snapshot before confirming durability"
1917                    .into(),
1918            )
1919        })?
1920    }
1921
1922    /// Capture the full persisted snapshot for the current state.
1923    fn capture_snapshot(
1924        &self,
1925        epoch_id: meerkat_core::RuntimeEpochId,
1926        _cursor_state: &meerkat_core::EpochCursorState,
1927    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
1928        let mut operations: HashMap<OperationId, OperationCanonicalState> = HashMap::new();
1929        for op_id in self.operation_ids()? {
1930            let status = self.require_status(&op_id)?;
1931            let kind = self.require_kind(&op_id)?;
1932            if Self::operation_durability_class(&op_id, kind)?
1933                != mm_dsl::OperationDurabilityClass::Retain
1934            {
1935                continue;
1936            }
1937            let peer_ready = self.require_peer_ready(&op_id)?;
1938            let progress_count = self.require_progress_count(&op_id)?;
1939            let operation_source = self.operation_source(&op_id)?;
1940            let terminal_outcome = self.terminal_outcome(&op_id)?;
1941            let completion_sequence = self.completion_sequence(&op_id);
1942            if terminal_outcome.is_some() && completion_sequence.is_none() {
1943                return Err(OpsLifecycleError::Internal(format!(
1944                    "generated op terminal authority missing completion sequence for retained terminal {op_id}"
1945                )));
1946            }
1947            if terminal_outcome.is_none() && completion_sequence.is_some() {
1948                return Err(OpsLifecycleError::Internal(format!(
1949                    "generated op terminal authority has completion sequence for non-terminal {op_id}"
1950                )));
1951            }
1952            let terminal_buffered = terminal_outcome.is_some();
1953            let watcher_count = self
1954                .records
1955                .get(&op_id)
1956                .map(|r| r.watchers.len() as u32)
1957                .unwrap_or(0);
1958            operations.insert(
1959                op_id,
1960                OperationCanonicalState {
1961                    status,
1962                    kind,
1963                    operation_source,
1964                    peer_ready,
1965                    progress_count,
1966                    watcher_count,
1967                    terminal_outcome,
1968                    completion_sequence,
1969                    terminal_buffered,
1970                },
1971            );
1972        }
1973        let operation_specs: HashMap<OperationId, OperationSpec> = self
1974            .records
1975            .iter()
1976            .filter(|(id, _)| operations.contains_key(*id))
1977            .map(|(id, record)| {
1978                let mut spec = record.spec.clone();
1979                let operation_source = operations
1980                    .get(id)
1981                    .and_then(|state| state.operation_source.as_ref());
1982                Self::align_spec_child_session_id_to_source(&mut spec, operation_source);
1983                (id.clone(), spec)
1984            })
1985            .collect();
1986        let completed_order: VecDeque<OperationId> = self
1987            .completed_order
1988            .iter()
1989            .filter(|id| operations.contains_key(*id))
1990            .cloned()
1991            .collect();
1992        let active_count = operations
1993            .iter()
1994            .filter(|(id, state)| {
1995                matches!(
1996                    Self::operation_status_is_terminal(id, state.status),
1997                    Ok(false)
1998                )
1999            })
2000            .count();
2001        let authority_completion_entries = self.completion_feed_authority_entries()?;
2002        let (published_completion_entries_by_id, completion_feed_retention): (
2003            HashMap<OperationId, CompletionEntry>,
2004            PersistedCompletionFeedRetention,
2005        ) = {
2006            let inner = self
2007                .feed_buffer
2008                .inner
2009                .read()
2010                .unwrap_or_else(std::sync::PoisonError::into_inner);
2011            let mut entries = HashMap::new();
2012            for entry in &inner.entries {
2013                if !authority_completion_entries.contains_key(&entry.operation_id) {
2014                    return Err(OpsLifecycleError::Internal(format!(
2015                        "public completion feed projection for {} has no generated authority",
2016                        entry.operation_id
2017                    )));
2018                }
2019                if entries
2020                    .insert(entry.operation_id.clone(), entry.clone())
2021                    .is_some()
2022                {
2023                    return Err(OpsLifecycleError::Internal(format!(
2024                        "public completion feed projection for {} appeared more than once",
2025                        entry.operation_id
2026                    )));
2027                }
2028            }
2029            (
2030                entries,
2031                PersistedCompletionFeedRetention {
2032                    dropped_through: inner.dropped_through,
2033                    dropped_count: inner.dropped_count,
2034                },
2035            )
2036        };
2037        let mut completion_entries: Vec<CompletionEntry> = authority_completion_entries
2038            .iter()
2039            .map(|(operation_id, authority_entry)| {
2040                if let Some(projection) = published_completion_entries_by_id.get(operation_id) {
2041                    if projection.seq != authority_entry.seq
2042                        || projection.kind != authority_entry.kind
2043                        || projection.terminal_outcome != authority_entry.terminal_outcome
2044                    {
2045                        return Err(OpsLifecycleError::Internal(format!(
2046                            "public completion feed projection for {operation_id} drifted from generated authority"
2047                        )));
2048                    }
2049                    return Ok(projection.clone());
2050                }
2051
2052                let display_name = self
2053                    .records
2054                    .get(operation_id)
2055                    .map(|record| record.spec.display_name.clone())
2056                    .unwrap_or_default();
2057                let completed_at_ms = self.records.get(operation_id).and_then(|record| {
2058                    record
2059                        .completed_at
2060                        .map(|completed_at| record.epoch_millis_for_instant(completed_at))
2061                });
2062
2063                Ok(CompletionEntry {
2064                    seq: authority_entry.seq,
2065                    operation_id: operation_id.clone(),
2066                    kind: authority_entry.kind,
2067                    display_name,
2068                    terminal_outcome: authority_entry.terminal_outcome.clone(),
2069                    completed_at_ms,
2070                })
2071            })
2072            .collect::<Result<_, _>>()?;
2073        completion_entries.sort_by_key(|entry| entry.seq);
2074
2075        let authority_state = RegistryCanonicalState {
2076            operations,
2077            completion_feed_entries: authority_completion_entries,
2078            completed_order,
2079            max_completed: self.max_completed,
2080            max_concurrent: self.max_concurrent,
2081            active_count,
2082            wait_request_id: self.wait_request_id.clone(),
2083            wait_operation_ids: self.wait_operation_ids()?,
2084            next_completion_seq: self.dsl.0.state().next_completion_seq,
2085        };
2086
2087        Ok(PersistedOpsSnapshot {
2088            epoch_id,
2089            authority_state,
2090            operation_specs,
2091            completion_entries,
2092            completion_feed_retention: Some(completion_feed_retention),
2093            cursors: self.completion_cursor_snapshot(),
2094        })
2095    }
2096
2097    fn shell_record_mut(
2098        &mut self,
2099        id: &OperationId,
2100    ) -> Result<&mut ShellRecord, OpsLifecycleError> {
2101        self.records
2102            .get_mut(id)
2103            .ok_or_else(|| OpsLifecycleError::NotFound(id.clone()))
2104    }
2105
2106    fn collect_wait_outcomes(
2107        &self,
2108        operation_ids: &[OperationId],
2109    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
2110        operation_ids
2111            .iter()
2112            .map(|operation_id| {
2113                let outcome = self.terminal_outcome(operation_id)?.ok_or_else(|| {
2114                    OpsLifecycleError::Internal(format!(
2115                        "wait_all completed without terminal outcome for {operation_id}"
2116                    ))
2117                })?;
2118                Ok((operation_id.clone(), outcome))
2119            })
2120            .collect()
2121    }
2122}
2123
2124impl Default for ShellState {
2125    fn default() -> Self {
2126        Self::new(DEFAULT_MAX_COMPLETED, None)
2127    }
2128}
2129
2130// ---------------------------------------------------------------------------
2131// Public configuration & registry
2132// ---------------------------------------------------------------------------
2133
2134/// Configuration for [`RuntimeOpsLifecycleRegistry`].
2135#[derive(Debug, Clone)]
2136pub struct OpsLifecycleConfig {
2137    /// Maximum number of completed operations to retain (default: 256).
2138    /// Values below one are normalized to one.
2139    pub max_completed: usize,
2140    /// Maximum concurrent non-terminal operations (None = unlimited).
2141    pub max_concurrent: Option<usize>,
2142}
2143
2144impl Default for OpsLifecycleConfig {
2145    fn default() -> Self {
2146        Self {
2147            max_completed: DEFAULT_MAX_COMPLETED,
2148            max_concurrent: None,
2149        }
2150    }
2151}
2152
2153/// Per-runtime shared registry for async operation lifecycle truth.
2154///
2155/// Per-operation canonical lifecycle state is owned by the DSL authority
2156/// embedded in the shell. This struct manages I/O concerns: watcher
2157/// channels, timestamps, peer handles, snapshot assembly, FIFO eviction,
2158/// and the completion feed buffer.
2159#[derive(Debug)]
2160pub struct RuntimeOpsLifecycleRegistry {
2161    state: Arc<RwLock<ShellState>>,
2162}
2163
2164#[derive(Debug, Clone)]
2165pub(crate) struct RuntimeOpsDiagnosticSnapshot {
2166    pub operation_count: usize,
2167    pub active_count: usize,
2168    pub wait_request_id: Option<WaitRequestId>,
2169    pub pending_wait_present: bool,
2170    pub pending_wait_request_id: Option<WaitRequestId>,
2171    pub wait_operation_ids: Vec<OperationId>,
2172    pub operations: Vec<OperationLifecycleSnapshot>,
2173}
2174
2175impl Default for RuntimeOpsLifecycleRegistry {
2176    fn default() -> Self {
2177        Self {
2178            state: Arc::new(RwLock::new(ShellState::default())),
2179        }
2180    }
2181}
2182
2183impl RuntimeOpsLifecycleRegistry {
2184    pub fn new() -> Self {
2185        let dsl = new_ops_dsl_authority();
2186        let feed_capacity = DEFAULT_MAX_COMPLETED.max(1);
2187        let feed_buffer = Arc::new(FeedBuffer::new(feed_capacity));
2188        Self {
2189            state: Arc::new(RwLock::new(ShellState {
2190                dsl,
2191                records: HashMap::new(),
2192                pending_wait: None,
2193                completed_order: VecDeque::new(),
2194                max_completed: DEFAULT_MAX_COMPLETED,
2195                max_concurrent: None,
2196                wait_request_id: None,
2197                feed_buffer,
2198                persist_tx: None,
2199                persistence_sealed: false,
2200                persist_epoch_id: None,
2201                persist_cursor_state: None,
2202                owner_retired: false,
2203            })),
2204        }
2205    }
2206
2207    pub fn with_config(config: OpsLifecycleConfig) -> Self {
2208        Self {
2209            state: Arc::new(RwLock::new(ShellState::new(
2210                config.max_completed,
2211                config.max_concurrent,
2212            ))),
2213        }
2214    }
2215
2216    fn recover_completion_feed_entry(
2217        shell: &mut ShellState,
2218        operation_id: &OperationId,
2219        entry: &CompletionFeedCanonicalState,
2220    ) -> Result<(), OpsLifecycleError> {
2221        let expected_operation_id = mm_dsl::OperationId::from_domain(operation_id).0;
2222        let terminal_outcome_kind =
2223            mm_dsl::OperationTerminalOutcomeKind::from(&entry.terminal_outcome);
2224        let effects = shell.dsl_apply_with_effects(
2225            mm_dsl::MeerkatMachineInput::RecoverCompletionFeedEntry {
2226                operation_id: expected_operation_id.clone(),
2227                kind: mm_dsl::OperationKind::from(entry.kind),
2228                terminal_outcome: terminal_outcome_kind,
2229                terminal_payload: entry.terminal_outcome.clone(),
2230                completion_sequence: entry.seq,
2231            },
2232            "RecoverCompletionFeedEntry",
2233        )?;
2234        let recovered = effects.iter().find_map(|effect| match effect {
2235            mm_dsl::MeerkatMachineEffect::CompletionFeedEntryRecovered {
2236                operation_id,
2237                seq,
2238                kind,
2239                terminal_outcome,
2240                terminal_payload,
2241            } => Some((
2242                operation_id,
2243                *seq,
2244                OperationKind::from(*kind),
2245                *terminal_outcome,
2246                terminal_payload,
2247            )),
2248            _ => None,
2249        });
2250        let Some((operation_id, seq, kind, terminal_outcome, terminal_payload)) = recovered else {
2251            return Err(OpsLifecycleError::Internal(
2252                "generated completion-feed recovery emitted no recovered entry".into(),
2253            ));
2254        };
2255        if operation_id != &expected_operation_id
2256            || seq != entry.seq
2257            || kind != entry.kind
2258            || terminal_outcome != terminal_outcome_kind
2259            || terminal_payload != &entry.terminal_outcome
2260        {
2261            return Err(OpsLifecycleError::Internal(format!(
2262                "generated completion-feed recovery drifted for {operation_id}"
2263            )));
2264        }
2265        Ok(())
2266    }
2267
2268    /// Wire a persistence channel for durable snapshot writes.
2269    ///
2270    /// After this call, terminal transitions (complete/fail/cancel/abort)
2271    /// capture a snapshot and queue it to the channel. A dedicated
2272    /// persistence task should drain the channel and write to the store.
2273    pub fn set_persistence_channel(
2274        &self,
2275        tx: crate::tokio::sync::mpsc::UnboundedSender<OpsLifecyclePersistenceRequest>,
2276        epoch_id: meerkat_core::RuntimeEpochId,
2277        cursor_state: Arc<meerkat_core::EpochCursorState>,
2278    ) {
2279        if let Ok(mut state) = self.state.write() {
2280            if state.owner_retired || state.persistence_sealed {
2281                tracing::warn!(
2282                    "ignored attempt to rewire a terminally sealed ops persistence channel"
2283                );
2284                return;
2285            }
2286            state.persist_tx = Some(tx);
2287            state.persist_epoch_id = Some(epoch_id);
2288            state.persist_cursor_state = Some(cursor_state);
2289        }
2290    }
2291
2292    /// Terminalize every live operation, persist those generated transitions,
2293    /// then close both lifecycle admission and the persistence producer.
2294    ///
2295    /// The write lock excludes every callback that retained this registry.
2296    /// Each terminal transition waits for its durable worker acknowledgement;
2297    /// taking the sender afterward therefore leaves no queued write behind.
2298    /// The unregister coordinator must join the owned worker before publishing
2299    /// final lifecycle deletion in the RuntimeStore.
2300    pub(crate) fn retire_owner_for_unregister(
2301        &self,
2302        reason: String,
2303    ) -> Result<(), OpsLifecycleError> {
2304        let closed_sender = {
2305            let mut state = self.write_state()?;
2306            if state.owner_retired {
2307                return Ok(());
2308            }
2309            terminate_owner_locked(&mut state, &reason)?;
2310            state.owner_retired = true;
2311            state.persistence_sealed = true;
2312            state.persist_epoch_id = None;
2313            state.persist_cursor_state = None;
2314            state.persist_tx.take()
2315        };
2316        drop(closed_sender);
2317        Ok(())
2318    }
2319
2320    /// Recover from a persisted snapshot.
2321    ///
2322    /// Rebuilds DSL state, retaining terminal operations and running
2323    /// `DetachedJobWait` bindings while stripping every other non-terminal
2324    /// operation. Fresh shell records come from specs; completion entries are
2325    /// seeded only after generated recovery authority accepts them.
2326    pub fn from_recovered(snapshot: PersistedOpsSnapshot) -> Result<Self, OpsLifecycleError> {
2327        let PersistedOpsSnapshot {
2328            epoch_id: _,
2329            authority_state,
2330            operation_specs,
2331            completion_entries,
2332            completion_feed_retention,
2333            cursors,
2334        } = snapshot;
2335        let max_completed = authority_state.max_completed;
2336        let max_concurrent = authority_state.max_concurrent;
2337        let next_completion_seq = authority_state.next_completion_seq;
2338        let authority_completion_entries = authority_state.completion_feed_entries;
2339        let authority_operations = authority_state.operations;
2340        let mut shell = ShellState::new(max_completed, max_concurrent);
2341
2342        // Replay every persisted op through generated recovery authority.
2343        // The transition accepts terminal records with outcome/sequence
2344        // witnesses plus running DetachedJobWait records with their typed
2345        // source. Other non-terminal rows remain volatile.
2346        let mut retained_ids: HashSet<OperationId> = HashSet::new();
2347        for (op_id, op_state) in authority_operations {
2348            let terminal_outcome = op_state
2349                .terminal_outcome
2350                .as_ref()
2351                .map(mm_dsl::OperationTerminalOutcomeKind::from);
2352            let terminal_payload = op_state.terminal_outcome.clone();
2353            let disposition = ShellState::recovered_operation_record_disposition(
2354                &op_id,
2355                op_state.status,
2356                op_state.kind,
2357                terminal_outcome.is_some(),
2358                terminal_payload.is_some(),
2359                op_state.completion_sequence.is_some(),
2360            )?;
2361            if disposition == RecoveredOperationRecordDisposition::Discard {
2362                continue;
2363            }
2364            if let Some(spec_source) = operation_specs
2365                .get(&op_id)
2366                .and_then(|spec| spec.operation_source.as_ref())
2367                && op_state.operation_source.as_ref() != Some(spec_source)
2368            {
2369                return Err(OpsLifecycleError::Internal(format!(
2370                    "persisted operation source mirror for {op_id} drifted from generated authority"
2371                )));
2372            }
2373            let recovery = mm_dsl::MeerkatMachineInput::RecoverOpRecord {
2374                operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
2375                status: mm_dsl::OperationStatus::from(op_state.status),
2376                kind: mm_dsl::OperationKind::from(op_state.kind),
2377                source: op_state
2378                    .operation_source
2379                    .as_ref()
2380                    .map(mm_dsl::OperationSource::from_domain),
2381                peer_ready: op_state.peer_ready,
2382                progress_count: u64::from(op_state.progress_count),
2383                terminal_outcome,
2384                terminal_payload,
2385                completion_sequence: op_state.completion_sequence,
2386            };
2387            shell.dsl_apply(recovery, "RecoverOpRecord")?;
2388            let recovered_seq = shell.completion_sequence(&op_id);
2389            if op_state.completion_sequence != recovered_seq {
2390                return Err(OpsLifecycleError::Internal(format!(
2391                    "generated op recovery completion sequence mismatch for {op_id}"
2392                )));
2393            }
2394            retained_ids.insert(op_id);
2395        }
2396        shell.dsl_apply(
2397            mm_dsl::MeerkatMachineInput::RecoverOpsCompletionCursor {
2398                next_completion_seq,
2399            },
2400            "RecoverOpsCompletionCursor",
2401        )?;
2402        shell.dsl_apply(
2403            mm_dsl::MeerkatMachineInput::RecoverCompletionConsumerCursors {
2404                agent_applied_cursor: cursors.agent_applied_cursor,
2405                runtime_observed_cursor: cursors.runtime_observed_seq,
2406                runtime_injected_cursor: cursors.runtime_last_injected_seq,
2407            },
2408            "RecoverCompletionConsumerCursors",
2409        )?;
2410
2411        // Rebuild completed_order from generated completion-sequence truth,
2412        // never from the persisted shell ordering mirror.
2413        let mut recovered_completed: Vec<(CompletionSeq, OperationId)> = retained_ids
2414            .iter()
2415            .filter_map(|id| shell.completion_sequence(id).map(|seq| (seq, id.clone())))
2416            .collect();
2417        recovered_completed.sort_by_key(|(seq, _)| *seq);
2418        shell.completed_order = recovered_completed.into_iter().map(|(_, id)| id).collect();
2419
2420        // Recover generated-owned feed authority for entries whose operation
2421        // record is no longer retained. Retained records already wrote their
2422        // feed authority through RecoverOpRecord above.
2423        for (operation_id, entry) in &authority_completion_entries {
2424            if !retained_ids.contains(operation_id) {
2425                Self::recover_completion_feed_entry(&mut shell, operation_id, entry)?;
2426            }
2427        }
2428
2429        let canonical_feed_entries = shell.completion_feed_authority_entries()?;
2430        for (operation_id, entry) in &authority_completion_entries {
2431            let Some(recovered_entry) = canonical_feed_entries.get(operation_id) else {
2432                return Err(OpsLifecycleError::Internal(format!(
2433                    "persisted completion feed authority for {operation_id} was not recovered"
2434                )));
2435            };
2436            if recovered_entry != entry {
2437                return Err(OpsLifecycleError::Internal(format!(
2438                    "persisted completion feed authority drifted from generated recovery for {operation_id}"
2439                )));
2440            }
2441        }
2442
2443        // Projection rows may carry display metadata only after generated
2444        // feed authority has decided the operation id, sequence, kind, and
2445        // terminal outcome. Any semantic drift fails closed.
2446        let mut projection_entries_by_id: HashMap<OperationId, CompletionEntry> = HashMap::new();
2447        for entry in completion_entries {
2448            let Some(authority_entry) = canonical_feed_entries.get(&entry.operation_id) else {
2449                return Err(OpsLifecycleError::Internal(format!(
2450                    "persisted completion feed projection for {} has no generated feed authority",
2451                    entry.operation_id
2452                )));
2453            };
2454            if authority_entry.seq != entry.seq
2455                || authority_entry.kind != entry.kind
2456                || authority_entry.terminal_outcome != entry.terminal_outcome
2457            {
2458                return Err(OpsLifecycleError::Internal(format!(
2459                    "persisted completion feed projection for {} drifted from generated feed authority",
2460                    entry.operation_id
2461                )));
2462            }
2463            projection_entries_by_id.insert(entry.operation_id.clone(), entry);
2464        }
2465
2466        let mut recovered_entries: Vec<(OperationId, CompletionFeedCanonicalState)> =
2467            canonical_feed_entries.into_iter().collect();
2468        recovered_entries.sort_by_key(|(_, entry)| entry.seq);
2469        if let Some(retention) = completion_feed_retention {
2470            if retention.dropped_count > 0 && recovered_entries.is_empty() {
2471                return Err(OpsLifecycleError::Internal(
2472                    "persisted completion-feed retention has no retained public suffix".to_string(),
2473                ));
2474            }
2475            if recovered_entries
2476                .first()
2477                .is_some_and(|(_, entry)| entry.seq <= retention.dropped_through)
2478            {
2479                return Err(OpsLifecycleError::Internal(
2480                    "persisted completion-feed retention overlaps the retained public suffix"
2481                        .to_string(),
2482                ));
2483            }
2484            shell.feed_buffer.recover_evicted_prefix(retention)?;
2485        }
2486        for (operation_id, entry) in recovered_entries {
2487            let projection = projection_entries_by_id.get(&operation_id);
2488            let display_name = operation_specs
2489                .get(&operation_id)
2490                .map(|spec| spec.display_name.clone())
2491                .or_else(|| projection.map(|entry| entry.display_name.clone()))
2492                .unwrap_or_default();
2493            let completed_at_ms = projection.and_then(|entry| entry.completed_at_ms);
2494            shell.feed_buffer.push(CompletionEntry {
2495                seq: entry.seq,
2496                operation_id,
2497                kind: entry.kind,
2498                display_name,
2499                terminal_outcome: entry.terminal_outcome,
2500                completed_at_ms,
2501            });
2502        }
2503
2504        // Rebuild shell records from specs (fresh timestamps, no watchers)
2505        // — only for operations still retained in the DSL state.
2506        for (op_id, spec) in operation_specs {
2507            if retained_ids.contains(&op_id) {
2508                let mut spec = spec;
2509                let operation_source = shell.operation_source(&op_id)?;
2510                ShellState::align_spec_child_session_id_to_source(
2511                    &mut spec,
2512                    operation_source.as_ref(),
2513                );
2514                shell.records.insert(
2515                    op_id,
2516                    ShellRecord {
2517                        spec,
2518                        peer_handle: None,
2519                        watchers: Vec::new(),
2520                        created_at: Instant::now(),
2521                        started_at: None,
2522                        completed_at: None,
2523                        created_at_wall: SystemTime::now(),
2524                    },
2525                );
2526            }
2527        }
2528
2529        Ok(Self {
2530            state: Arc::new(RwLock::new(shell)),
2531        })
2532    }
2533
2534    /// Capture a serializable snapshot of the current state for persistence.
2535    ///
2536    /// Includes authority state, operation specs, completion entries, and
2537    /// generated completion-consumer cursor values.
2538    pub fn capture_persistence_snapshot(
2539        &self,
2540        epoch_id: meerkat_core::RuntimeEpochId,
2541        cursor_state: &meerkat_core::EpochCursorState,
2542    ) -> Result<PersistedOpsSnapshot, OpsLifecycleError> {
2543        let state = self
2544            .state
2545            .read()
2546            .unwrap_or_else(std::sync::PoisonError::into_inner);
2547        state.capture_snapshot(epoch_id, cursor_state)
2548    }
2549
2550    /// Snapshot generated completion-consumer cursor state.
2551    pub fn completion_cursor_snapshot(&self) -> meerkat_core::EpochCursorSnapshot {
2552        let state = self
2553            .state
2554            .read()
2555            .unwrap_or_else(std::sync::PoisonError::into_inner);
2556        state.completion_cursor_snapshot()
2557    }
2558
2559    /// Return a read handle to the completion feed.
2560    pub fn completion_feed_handle(&self) -> Arc<dyn CompletionFeed> {
2561        let state = self
2562            .state
2563            .read()
2564            .unwrap_or_else(std::sync::PoisonError::into_inner);
2565        Arc::new(RuntimeCompletionFeed {
2566            buffer: Arc::clone(&state.feed_buffer),
2567            authority: Arc::downgrade(&self.state),
2568        })
2569    }
2570
2571    /// Capture a stable diagnostic snapshot of the canonical ops lifecycle state.
2572    pub(crate) fn diagnostic_snapshot(
2573        &self,
2574    ) -> Result<RuntimeOpsDiagnosticSnapshot, OpsLifecycleError> {
2575        let state = self
2576            .state
2577            .read()
2578            .unwrap_or_else(std::sync::PoisonError::into_inner);
2579        let mut operations = state
2580            .operation_ids()?
2581            .into_iter()
2582            .map(|id| state.snapshot(&id))
2583            .collect::<Result<Vec<_>, _>>()?
2584            .into_iter()
2585            .flatten()
2586            .collect::<Vec<_>>();
2587        operations.sort_by(|left, right| left.display_name.cmp(&right.display_name));
2588        Ok(RuntimeOpsDiagnosticSnapshot {
2589            operation_count: state.operation_count(),
2590            active_count: state.active_count(),
2591            wait_request_id: state.wait_request_id.clone(),
2592            pending_wait_present: state.pending_wait.is_some(),
2593            pending_wait_request_id: state
2594                .pending_wait
2595                .as_ref()
2596                .map(|pending_wait| pending_wait.wait_request_id.clone()),
2597            wait_operation_ids: state.wait_operation_ids()?,
2598            operations,
2599        })
2600    }
2601
2602    fn read_state(&self) -> Result<RwLockReadGuard<'_, ShellState>, OpsLifecycleError> {
2603        self.state
2604            .read()
2605            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2606    }
2607
2608    fn write_state(&self) -> Result<RwLockWriteGuard<'_, ShellState>, OpsLifecycleError> {
2609        self.state
2610            .write()
2611            .map_err(|_| OpsLifecycleError::Internal("ops lifecycle registry poisoned".into()))
2612    }
2613
2614    fn cancel_wait_all_internal(
2615        &self,
2616        wait_request_id: &WaitRequestId,
2617    ) -> Result<(), OpsLifecycleError> {
2618        let mut state = self.write_state()?;
2619        match state.wait_request_id.as_ref() {
2620            Some(active) if active == wait_request_id => {
2621                // Clear the DSL barrier via the dedicated `CancelWaitAll`
2622                // transition. Unlike `SatisfyWaitAll`, it does not require
2623                // every member to be terminal (the request was dropped, not
2624                // resolved) and does not emit the `WaitAllSatisfied`.
2625                state.dsl_apply(
2626                    mm_dsl::MeerkatMachineInput::CancelWaitAll,
2627                    "CancelWaitAll(cancel)",
2628                )?;
2629                state.wait_request_id = None;
2630                if state
2631                    .pending_wait
2632                    .as_ref()
2633                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2634                {
2635                    state.pending_wait = None;
2636                }
2637                Ok(())
2638            }
2639            _ => {
2640                if state
2641                    .pending_wait
2642                    .as_ref()
2643                    .is_some_and(|pending| pending.wait_request_id == *wait_request_id)
2644                {
2645                    state.pending_wait = None;
2646                }
2647                Ok(())
2648            }
2649        }
2650    }
2651}
2652
2653enum WaitAllFutureState {
2654    Ready(Option<Result<WaitAllResult, OpsLifecycleError>>),
2655    Waiting(tokio::sync::oneshot::Receiver<WaitAllSatisfied>),
2656    Done,
2657}
2658
2659struct WaitAllFuture<'a> {
2660    registry: &'a RuntimeOpsLifecycleRegistry,
2661    wait_request_id: WaitRequestId,
2662    state: WaitAllFutureState,
2663}
2664
2665impl Future for WaitAllFuture<'_> {
2666    type Output = Result<WaitAllResult, OpsLifecycleError>;
2667
2668    fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
2669        match &mut self.state {
2670            WaitAllFutureState::Ready(result) => {
2671                let ready = result.take().unwrap_or_else(|| {
2672                    Err(OpsLifecycleError::Internal(
2673                        "wait_all future polled after completion".into(),
2674                    ))
2675                });
2676                self.state = WaitAllFutureState::Done;
2677                Poll::Ready(ready)
2678            }
2679            WaitAllFutureState::Waiting(receiver) => match std::pin::Pin::new(receiver).poll(cx) {
2680                Poll::Pending => Poll::Pending,
2681                Poll::Ready(Ok(satisfied)) => {
2682                    let outcomes = match self.registry.read_state() {
2683                        Ok(state) => state.collect_wait_outcomes(&satisfied.operation_ids),
2684                        Err(err) => Err(err),
2685                    };
2686                    self.state = WaitAllFutureState::Done;
2687                    Poll::Ready(outcomes.map(|outcomes| WaitAllResult {
2688                        outcomes,
2689                        satisfied,
2690                    }))
2691                }
2692                Poll::Ready(Err(_)) => {
2693                    self.state = WaitAllFutureState::Done;
2694                    Poll::Ready(Err(OpsLifecycleError::Internal(
2695                        "wait_all completion channel dropped".into(),
2696                    )))
2697                }
2698            },
2699            WaitAllFutureState::Done => Poll::Ready(Err(OpsLifecycleError::Internal(
2700                "wait_all future polled after completion".into(),
2701            ))),
2702        }
2703    }
2704}
2705
2706impl Drop for WaitAllFuture<'_> {
2707    fn drop(&mut self) {
2708        if matches!(self.state, WaitAllFutureState::Waiting(_))
2709            && let Err(err) = self
2710                .registry
2711                .cancel_wait_all_internal(&self.wait_request_id)
2712        {
2713            tracing::error!(
2714                wait_request_id = %self.wait_request_id,
2715                error = %err,
2716                "generated wait_all authority rejected cancellation during drop"
2717            );
2718        }
2719    }
2720}
2721
2722// ---------------------------------------------------------------------------
2723// Generated op lifecycle result-class feedback
2724// ---------------------------------------------------------------------------
2725
2726fn op_lifecycle_action_label(action: mm_dsl::OpLifecycleActionKind) -> &'static str {
2727    match action {
2728        mm_dsl::OpLifecycleActionKind::Start => "provisioning_succeeded",
2729        mm_dsl::OpLifecycleActionKind::Fail => "fail_operation",
2730        mm_dsl::OpLifecycleActionKind::PeerReady => "peer_ready",
2731        mm_dsl::OpLifecycleActionKind::ProgressReported => "report_progress",
2732        mm_dsl::OpLifecycleActionKind::Complete => "complete_operation",
2733        mm_dsl::OpLifecycleActionKind::Abort => "abort_provisioning",
2734        mm_dsl::OpLifecycleActionKind::Cancel => "cancel_operation",
2735        mm_dsl::OpLifecycleActionKind::RetireRequested => "request_retire",
2736        mm_dsl::OpLifecycleActionKind::RetireCompleted => "mark_retired",
2737        mm_dsl::OpLifecycleActionKind::Terminate => "terminate_owner",
2738    }
2739}
2740
2741fn op_lifecycle_rejection_error_from_effects(
2742    id: &OperationId,
2743    requested_action: mm_dsl::OpLifecycleActionKind,
2744    effects: &[mm_dsl::MeerkatMachineEffect],
2745) -> Result<OpsLifecycleError, OpsLifecycleError> {
2746    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2747    let mut rejection = None;
2748    for effect in effects {
2749        let mm_dsl::MeerkatMachineEffect::OpLifecycleTransitionRejected {
2750            operation_id,
2751            action,
2752            reason,
2753            status,
2754        } = effect
2755        else {
2756            continue;
2757        };
2758        if rejection.is_some() {
2759            return Err(OpsLifecycleError::Internal(
2760                "generated op lifecycle authority emitted multiple rejection results".into(),
2761            ));
2762        }
2763        if operation_id != &expected_id || *action != requested_action {
2764            return Err(OpsLifecycleError::Internal(format!(
2765                "generated op lifecycle authority resolved {operation_id}/{action:?} while shell requested {expected_id}/{requested_action:?}"
2766            )));
2767        }
2768        let error = match reason {
2769            mm_dsl::OpLifecycleRejectReasonKind::OperationNotFound => {
2770                if status.is_some() {
2771                    return Err(OpsLifecycleError::Internal(
2772                        "generated op lifecycle authority emitted not-found with status".into(),
2773                    ));
2774                }
2775                OpsLifecycleError::NotFound(id.clone())
2776            }
2777            mm_dsl::OpLifecycleRejectReasonKind::InvalidTransition => {
2778                let status = status.ok_or_else(|| {
2779                    OpsLifecycleError::Internal(
2780                        "generated op lifecycle authority emitted invalid-transition without status"
2781                            .into(),
2782                    )
2783                })?;
2784                OpsLifecycleError::InvalidTransition {
2785                    id: id.clone(),
2786                    status: OperationStatus::from(status),
2787                    action: op_lifecycle_action_label(requested_action),
2788                }
2789            }
2790            mm_dsl::OpLifecycleRejectReasonKind::PeerNotExpected => {
2791                if status.is_none() {
2792                    return Err(OpsLifecycleError::Internal(
2793                        "generated op lifecycle authority emitted peer-not-expected without status"
2794                            .into(),
2795                    ));
2796                }
2797                OpsLifecycleError::PeerNotExpected(id.clone())
2798            }
2799            mm_dsl::OpLifecycleRejectReasonKind::AlreadyPeerReady => {
2800                if status.is_none() {
2801                    return Err(OpsLifecycleError::Internal(
2802                        "generated op lifecycle authority emitted already-peer-ready without status"
2803                            .into(),
2804                    ));
2805                }
2806                OpsLifecycleError::AlreadyPeerReady(id.clone())
2807            }
2808        };
2809        rejection = Some(error);
2810    }
2811    rejection.ok_or_else(|| {
2812        OpsLifecycleError::Internal(
2813            "generated op lifecycle authority emitted no rejection result".into(),
2814        )
2815    })
2816}
2817
2818fn classify_generated_op_rejection(
2819    state: &mut ShellState,
2820    err: mm_dsl::MeerkatMachineTransitionError,
2821    id: &OperationId,
2822    action: mm_dsl::OpLifecycleActionKind,
2823) -> OpsLifecycleError {
2824    match err {
2825        mm_dsl::MeerkatMachineTransitionError::GuardRejected { .. } => {
2826            match state.dsl_apply_with_effects(
2827                mm_dsl::MeerkatMachineInput::ResolveOpLifecycleTransitionRejection {
2828                    operation_id: mm_dsl::OperationId::from_domain(id).0,
2829                    action,
2830                },
2831                "ResolveOpLifecycleTransitionRejection",
2832            ) {
2833                Ok(effects) => op_lifecycle_rejection_error_from_effects(id, action, &effects)
2834                    .unwrap_or_else(|err| err),
2835                Err(err) => err,
2836            }
2837        }
2838        other => OpsLifecycleError::Internal(format!(
2839            "DSL rejected ops transition ({}): {other:?}",
2840            op_lifecycle_action_label(action)
2841        )),
2842    }
2843}
2844
2845fn apply_op_transition(
2846    state: &mut ShellState,
2847    id: &OperationId,
2848    input: mm_dsl::MeerkatMachineInput,
2849    action: mm_dsl::OpLifecycleActionKind,
2850) -> Result<(), OpsLifecycleError> {
2851    state.ensure_owner_active()?;
2852    state
2853        .dsl_apply_raw(input)
2854        .map_err(|err| classify_generated_op_rejection(state, err, id, action))
2855}
2856
2857fn apply_terminal_op_transition_and_persist(
2858    state: &mut ShellState,
2859    id: &OperationId,
2860    input: mm_dsl::MeerkatMachineInput,
2861    action: mm_dsl::OpLifecycleActionKind,
2862) -> Result<(), OpsLifecycleError> {
2863    let previous_snapshot = state.dsl.0.snapshot();
2864    apply_op_transition(state, id, input, action)?;
2865    if let Err(err) = state.maybe_persist() {
2866        state.dsl.0.restore_snapshot(previous_snapshot);
2867        return Err(err);
2868    }
2869    Ok(())
2870}
2871
2872fn terminate_owner_locked(state: &mut ShellState, reason: &str) -> Result<(), OpsLifecycleError> {
2873    state.ensure_owner_active()?;
2874    let to_terminate = state.owner_termination_targets()?;
2875
2876    for (op_id, _status) in &to_terminate {
2877        let terminal_outcome = OperationTerminalOutcome::Terminated {
2878            reason: reason.to_owned(),
2879        };
2880        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
2881
2882        apply_terminal_op_transition_and_persist(
2883            state,
2884            op_id,
2885            mm_dsl::MeerkatMachineInput::TerminateOp {
2886                operation_id: mm_dsl::OperationId::from_domain(op_id).0,
2887                outcome: outcome_kind,
2888                payload: terminal_outcome,
2889            },
2890            mm_dsl::OpLifecycleActionKind::Terminate,
2891        )?;
2892
2893        if let Some(entry) = state.finalize_terminal(op_id)? {
2894            state.publish_completion_entry(Some(entry));
2895        }
2896    }
2897    Ok(())
2898}
2899
2900fn op_registration_error_from_effects(
2901    id: &OperationId,
2902    effects: &[mm_dsl::MeerkatMachineEffect],
2903) -> Result<Option<OpsLifecycleError>, OpsLifecycleError> {
2904    let expected_id = mm_dsl::OperationId::from_domain(id).0;
2905    let mut admission = None;
2906    for effect in effects {
2907        let mm_dsl::MeerkatMachineEffect::OpRegistrationAdmissionResolved {
2908            operation_id,
2909            result,
2910            reject_reason,
2911            max_concurrent_limit,
2912            active_op_count,
2913        } = effect
2914        else {
2915            continue;
2916        };
2917        if admission.is_some() {
2918            return Err(OpsLifecycleError::Internal(
2919                "generated op registration authority emitted multiple admission results".into(),
2920            ));
2921        }
2922        if operation_id != &expected_id {
2923            return Err(OpsLifecycleError::Internal(format!(
2924                "generated op registration authority resolved {operation_id} while shell requested {expected_id}"
2925            )));
2926        }
2927        admission = Some(match result {
2928            mm_dsl::OpRegistrationAdmissionResultKind::Accept => {
2929                if reject_reason.is_some() {
2930                    return Err(OpsLifecycleError::Internal(
2931                        "generated op registration authority accepted with rejection reason".into(),
2932                    ));
2933                }
2934                None
2935            }
2936            mm_dsl::OpRegistrationAdmissionResultKind::Reject => {
2937                let reason = reject_reason.ok_or_else(|| {
2938                    OpsLifecycleError::Internal(
2939                        "generated op registration authority rejected without reason".into(),
2940                    )
2941                })?;
2942                let error = match reason {
2943                    mm_dsl::OpRegistrationRejectReasonKind::AlreadyRegistered => {
2944                        OpsLifecycleError::AlreadyRegistered(id.clone())
2945                    }
2946                    mm_dsl::OpRegistrationRejectReasonKind::MaxConcurrentExceeded => {
2947                        let limit = max_concurrent_limit.ok_or_else(|| {
2948                            OpsLifecycleError::Internal(
2949                                "generated op registration authority rejected capacity without limit"
2950                                    .into(),
2951                            )
2952                        })?;
2953                        OpsLifecycleError::MaxConcurrentExceeded {
2954                            limit: limit as usize,
2955                            active: *active_op_count as usize,
2956                        }
2957                    }
2958                };
2959                Some(error)
2960            }
2961        });
2962    }
2963    admission.ok_or_else(|| {
2964        OpsLifecycleError::Internal(
2965            "generated op registration authority emitted no admission result".into(),
2966        )
2967    })
2968}
2969
2970impl OpsLifecycleRegistry for RuntimeOpsLifecycleRegistry {
2971    fn register_operation(&self, spec: OperationSpec) -> Result<(), OpsLifecycleError> {
2972        self.register_operation_with_admission_limit(spec, None)
2973    }
2974
2975    fn register_operation_with_admission_limit(
2976        &self,
2977        mut spec: OperationSpec,
2978        max_concurrent: Option<usize>,
2979    ) -> Result<(), OpsLifecycleError> {
2980        let mut state = self.write_state()?;
2981        state.ensure_owner_active()?;
2982        let operation_id = spec.id.clone();
2983        let kind = spec.kind;
2984        let max_concurrent = max_concurrent
2985            .or(state.max_concurrent)
2986            .map(|limit| limit as u64);
2987
2988        let effects = state.dsl_apply_with_effects(
2989            mm_dsl::MeerkatMachineInput::RegisterOp {
2990                operation_id: mm_dsl::OperationId::from_domain(&operation_id).0,
2991                kind: mm_dsl::OperationKind::from_domain(&kind),
2992                source: spec
2993                    .operation_source
2994                    .as_ref()
2995                    .map(mm_dsl::OperationSource::from_domain),
2996                max_concurrent,
2997            },
2998            "RegisterOp",
2999        )?;
3000        if let Some(error) = op_registration_error_from_effects(&operation_id, &effects)? {
3001            return Err(error);
3002        }
3003
3004        let authority_operation_source = state.operation_source(&operation_id)?;
3005        ShellState::align_spec_child_session_id_to_source(
3006            &mut spec,
3007            authority_operation_source.as_ref(),
3008        );
3009
3010        // Insert shell record.
3011        state.records.insert(operation_id, ShellRecord::new(spec));
3012        Ok(())
3013    }
3014
3015    fn provisioning_succeeded(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3016        let mut state = self.write_state()?;
3017
3018        apply_op_transition(
3019            &mut state,
3020            id,
3021            mm_dsl::MeerkatMachineInput::StartOp {
3022                operation_id: mm_dsl::OperationId::from_domain(id).0,
3023            },
3024            mm_dsl::OpLifecycleActionKind::Start,
3025        )?;
3026
3027        // Shell concern: record the started timestamp.
3028        if let Some(shell) = state.records.get_mut(id) {
3029            shell.started_at = Some(Instant::now());
3030        }
3031        Ok(())
3032    }
3033
3034    fn provisioning_failed(
3035        &self,
3036        id: &OperationId,
3037        error: String,
3038    ) -> Result<(), OpsLifecycleError> {
3039        let mut state = self.write_state()?;
3040
3041        let terminal_outcome = OperationTerminalOutcome::Failed { error };
3042        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3043
3044        apply_terminal_op_transition_and_persist(
3045            &mut state,
3046            id,
3047            mm_dsl::MeerkatMachineInput::FailOp {
3048                operation_id: mm_dsl::OperationId::from_domain(id).0,
3049                outcome: outcome_kind,
3050                payload: terminal_outcome,
3051            },
3052            mm_dsl::OpLifecycleActionKind::Fail,
3053        )?;
3054
3055        let completion_entry = state.finalize_terminal(id)?;
3056        state.publish_completion_entry(completion_entry);
3057        Ok(())
3058    }
3059
3060    fn peer_ready(
3061        &self,
3062        id: &OperationId,
3063        peer: OperationPeerHandle,
3064    ) -> Result<(), OpsLifecycleError> {
3065        let mut state = self.write_state()?;
3066
3067        apply_op_transition(
3068            &mut state,
3069            id,
3070            mm_dsl::MeerkatMachineInput::PeerReadyOp {
3071                operation_id: mm_dsl::OperationId::from_domain(id).0,
3072            },
3073            mm_dsl::OpLifecycleActionKind::PeerReady,
3074        )?;
3075
3076        // Shell concern: store the peer handle.
3077        if let Some(shell) = state.records.get_mut(id) {
3078            shell.peer_handle = Some(peer);
3079        }
3080        Ok(())
3081    }
3082
3083    fn register_watcher(
3084        &self,
3085        id: &OperationId,
3086    ) -> Result<OperationCompletionWatch, OpsLifecycleError> {
3087        let mut state = self.write_state()?;
3088
3089        if !state.contains(id) {
3090            return Err(OpsLifecycleError::NotFound(id.clone()));
3091        }
3092
3093        // If already terminal, return an already-resolved watch.
3094        if let Some(outcome) = state.terminal_outcome(id)? {
3095            return Ok(resolved_operation_completion_watch(outcome));
3096        }
3097
3098        // Shell concern: create the channel and store the sender.
3099        let shell = state.shell_record_mut(id)?;
3100        let (tx, rx) = tokio::sync::oneshot::channel();
3101        let watch = operation_completion_watch_from_receiver(rx);
3102        shell.watchers.push(OperationCompletionNotifier::new(tx));
3103        Ok(watch)
3104    }
3105
3106    fn report_progress(
3107        &self,
3108        id: &OperationId,
3109        _update: OperationProgressUpdate,
3110    ) -> Result<(), OpsLifecycleError> {
3111        let mut state = self.write_state()?;
3112
3113        apply_op_transition(
3114            &mut state,
3115            id,
3116            mm_dsl::MeerkatMachineInput::ProgressReportedOp {
3117                operation_id: mm_dsl::OperationId::from_domain(id).0,
3118            },
3119            mm_dsl::OpLifecycleActionKind::ProgressReported,
3120        )?;
3121        Ok(())
3122    }
3123
3124    fn complete_operation(
3125        &self,
3126        id: &OperationId,
3127        result: OperationResult,
3128    ) -> Result<(), OpsLifecycleError> {
3129        let mut state = self.write_state()?;
3130
3131        let terminal_outcome = OperationTerminalOutcome::Completed(result);
3132        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3133
3134        apply_terminal_op_transition_and_persist(
3135            &mut state,
3136            id,
3137            mm_dsl::MeerkatMachineInput::CompleteOp {
3138                operation_id: mm_dsl::OperationId::from_domain(id).0,
3139                outcome: outcome_kind,
3140                payload: terminal_outcome,
3141            },
3142            mm_dsl::OpLifecycleActionKind::Complete,
3143        )?;
3144
3145        let completion_entry = state.finalize_terminal(id)?;
3146        state.publish_completion_entry(completion_entry);
3147        Ok(())
3148    }
3149
3150    fn fail_operation(&self, id: &OperationId, error: String) -> Result<(), OpsLifecycleError> {
3151        let mut state = self.write_state()?;
3152
3153        let terminal_outcome = OperationTerminalOutcome::Failed { error };
3154        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3155
3156        apply_terminal_op_transition_and_persist(
3157            &mut state,
3158            id,
3159            mm_dsl::MeerkatMachineInput::FailOp {
3160                operation_id: mm_dsl::OperationId::from_domain(id).0,
3161                outcome: outcome_kind,
3162                payload: terminal_outcome,
3163            },
3164            mm_dsl::OpLifecycleActionKind::Fail,
3165        )?;
3166
3167        let completion_entry = state.finalize_terminal(id)?;
3168        state.publish_completion_entry(completion_entry);
3169        Ok(())
3170    }
3171
3172    fn abort_provisioning(
3173        &self,
3174        id: &OperationId,
3175        reason: Option<String>,
3176    ) -> Result<(), OpsLifecycleError> {
3177        let mut state = self.write_state()?;
3178
3179        let terminal_outcome = OperationTerminalOutcome::Aborted { reason };
3180        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3181
3182        apply_terminal_op_transition_and_persist(
3183            &mut state,
3184            id,
3185            mm_dsl::MeerkatMachineInput::AbortOp {
3186                operation_id: mm_dsl::OperationId::from_domain(id).0,
3187                outcome: outcome_kind,
3188                payload: terminal_outcome,
3189            },
3190            mm_dsl::OpLifecycleActionKind::Abort,
3191        )?;
3192
3193        let completion_entry = state.finalize_terminal(id)?;
3194        state.publish_completion_entry(completion_entry);
3195        Ok(())
3196    }
3197
3198    fn rollback_unreturned_operation(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3199        let mut state = self.write_state()?;
3200        state.ensure_owner_active()?;
3201        let expected_id = mm_dsl::OperationId::from_domain(id).0;
3202        let effects = state.dsl_apply_with_effects(
3203            mm_dsl::MeerkatMachineInput::RollbackUnreturnedOp {
3204                operation_id: expected_id.clone(),
3205            },
3206            "RollbackUnreturnedOp",
3207        )?;
3208        let mut acknowledgements = effects.iter().filter(|effect| {
3209            matches!(
3210                effect,
3211                mm_dsl::MeerkatMachineEffect::UnreturnedOpRolledBack { operation_id }
3212                    if operation_id == &expected_id
3213            )
3214        });
3215        if acknowledgements.next().is_none() || acknowledgements.next().is_some() {
3216            return Err(OpsLifecycleError::Internal(format!(
3217                "generated rollback authority did not emit exactly one matching acknowledgement for {id}"
3218            )));
3219        }
3220        state.records.remove(id);
3221        state.completed_order.retain(|queued| queued != id);
3222        Ok(())
3223    }
3224
3225    fn cancel_operation(
3226        &self,
3227        id: &OperationId,
3228        reason: Option<String>,
3229    ) -> Result<(), OpsLifecycleError> {
3230        let mut state = self.write_state()?;
3231
3232        let terminal_outcome = OperationTerminalOutcome::Cancelled { reason };
3233        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3234
3235        apply_terminal_op_transition_and_persist(
3236            &mut state,
3237            id,
3238            mm_dsl::MeerkatMachineInput::CancelOp {
3239                operation_id: mm_dsl::OperationId::from_domain(id).0,
3240                outcome: outcome_kind,
3241                payload: terminal_outcome,
3242            },
3243            mm_dsl::OpLifecycleActionKind::Cancel,
3244        )?;
3245
3246        let completion_entry = state.finalize_terminal(id)?;
3247        state.publish_completion_entry(completion_entry);
3248        Ok(())
3249    }
3250
3251    fn request_retire(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3252        let mut state = self.write_state()?;
3253
3254        apply_op_transition(
3255            &mut state,
3256            id,
3257            mm_dsl::MeerkatMachineInput::RetireRequestedOp {
3258                operation_id: mm_dsl::OperationId::from_domain(id).0,
3259            },
3260            mm_dsl::OpLifecycleActionKind::RetireRequested,
3261        )?;
3262        Ok(())
3263    }
3264
3265    fn mark_retired(&self, id: &OperationId) -> Result<(), OpsLifecycleError> {
3266        let mut state = self.write_state()?;
3267
3268        let terminal_outcome = OperationTerminalOutcome::Retired;
3269        let outcome_kind = mm_dsl::OperationTerminalOutcomeKind::from(&terminal_outcome);
3270
3271        apply_terminal_op_transition_and_persist(
3272            &mut state,
3273            id,
3274            mm_dsl::MeerkatMachineInput::RetireCompletedOp {
3275                operation_id: mm_dsl::OperationId::from_domain(id).0,
3276                outcome: outcome_kind,
3277                payload: terminal_outcome,
3278            },
3279            mm_dsl::OpLifecycleActionKind::RetireCompleted,
3280        )?;
3281
3282        let completion_entry = state.finalize_terminal(id)?;
3283        state.publish_completion_entry(completion_entry);
3284        Ok(())
3285    }
3286
3287    fn snapshot(
3288        &self,
3289        id: &OperationId,
3290    ) -> Result<Option<OperationLifecycleSnapshot>, OpsLifecycleError> {
3291        let state = self.read_state()?;
3292        state.snapshot(id)
3293    }
3294
3295    fn list_operations(&self) -> Result<Vec<OperationLifecycleSnapshot>, OpsLifecycleError> {
3296        let state = self.read_state()?;
3297        let mut snapshots = Vec::new();
3298        for id in state.operation_ids()? {
3299            let snapshot = state.snapshot(&id)?.ok_or_else(|| {
3300                OpsLifecycleError::Internal(format!(
3301                    "operation {id} was present in generated lifecycle authority but produced no public snapshot"
3302                ))
3303            })?;
3304            snapshots.push(snapshot);
3305        }
3306        snapshots.sort_by(|left, right| left.display_name.cmp(&right.display_name));
3307        Ok(snapshots)
3308    }
3309
3310    fn classify_operation_terminality(
3311        &self,
3312        id: &OperationId,
3313        status: OperationStatus,
3314    ) -> Result<bool, OpsLifecycleError> {
3315        ShellState::operation_status_is_terminal(id, status)
3316    }
3317
3318    fn classify_operation_public_result(
3319        &self,
3320        id: &OperationId,
3321    ) -> Result<OperationPublicResultClass, OpsLifecycleError> {
3322        let state = self.read_state()?;
3323        let status = match state.status(id) {
3324            Some(status) => status,
3325            None if state.records.contains_key(id)
3326                || state.has_generated_operation_record_fact(id) =>
3327            {
3328                return Err(OpsLifecycleError::Internal(format!(
3329                    "generated op lifecycle authority missing status for {id}"
3330                )));
3331            }
3332            None => OperationStatus::Absent,
3333        };
3334        ShellState::operation_public_result_class(id, status)
3335    }
3336
3337    fn classify_operation_completion_wake(
3338        &self,
3339        id: &OperationId,
3340        kind: OperationKind,
3341    ) -> Result<OperationCompletionWakeClass, OpsLifecycleError> {
3342        ShellState::operation_completion_wake_class(id, kind)
3343    }
3344
3345    fn classify_operation_transition_idempotence(
3346        &self,
3347        id: &OperationId,
3348        action: OperationLifecycleAction,
3349    ) -> Result<bool, OpsLifecycleError> {
3350        let state = self.read_state()?;
3351        let status = match state.status(id) {
3352            Some(status) => status,
3353            None if state.records.contains_key(id)
3354                || state.has_generated_operation_record_fact(id) =>
3355            {
3356                return Err(OpsLifecycleError::Internal(format!(
3357                    "generated op lifecycle authority missing status for {id}"
3358                )));
3359            }
3360            None => OperationStatus::Absent,
3361        };
3362        ShellState::operation_transition_rejection_is_idempotent(id, action, status)
3363    }
3364
3365    fn terminate_owner(&self, reason: String) -> Result<(), OpsLifecycleError> {
3366        let mut state = self.write_state()?;
3367        if state.owner_retired {
3368            return Ok(());
3369        }
3370        terminate_owner_locked(&mut state, &reason)
3371    }
3372
3373    fn collect_completed(
3374        &self,
3375    ) -> Result<Vec<(OperationId, OperationTerminalOutcome)>, OpsLifecycleError> {
3376        let mut state = self.write_state()?;
3377        state.ensure_owner_active()?;
3378
3379        let ids: Vec<OperationId> = state.completed_order.iter().cloned().collect();
3380        let mut collected = Vec::with_capacity(ids.len());
3381        for id in ids {
3382            let outcome = state.terminal_outcome(&id)?;
3383            state.dsl_apply(
3384                mm_dsl::MeerkatMachineInput::CollectCompletedOp {
3385                    operation_id: mm_dsl::OperationId::from_domain(&id).0,
3386                },
3387                "CollectCompletedOp",
3388            )?;
3389            state.completed_order.retain(|queued| queued != &id);
3390            state.records.remove(&id);
3391            if let Some(outcome) = outcome {
3392                collected.push((id, outcome));
3393            }
3394        }
3395        Ok(collected)
3396    }
3397
3398    fn completion_feed(&self) -> Option<Arc<dyn CompletionFeed>> {
3399        Some(self.completion_feed_handle())
3400    }
3401
3402    fn completion_cursor(
3403        &self,
3404        consumer: CompletionCursorConsumer,
3405    ) -> Result<Option<CompletionSeq>, OpsLifecycleError> {
3406        let state = self.read_state()?;
3407        Ok(Some(state.completion_cursor(consumer)))
3408    }
3409
3410    fn advance_completion_cursor(
3411        &self,
3412        consumer: CompletionCursorConsumer,
3413        cursor: CompletionSeq,
3414        projection: Option<&meerkat_core::EpochCursorState>,
3415    ) -> Result<CompletionSeq, OpsLifecycleError> {
3416        let mut state = self.write_state()?;
3417        state.ensure_owner_active()?;
3418        let input = match consumer {
3419            CompletionCursorConsumer::AgentApplied => {
3420                mm_dsl::MeerkatMachineInput::AdvanceAgentCompletionCursor { cursor }
3421            }
3422            CompletionCursorConsumer::RuntimeObserved => {
3423                mm_dsl::MeerkatMachineInput::AdvanceRuntimeObservedCompletionCursor { cursor }
3424            }
3425            CompletionCursorConsumer::RuntimeInjected => {
3426                mm_dsl::MeerkatMachineInput::AdvanceRuntimeInjectedCompletionCursor { cursor }
3427            }
3428        };
3429        let effects = state.dsl_apply_with_effects(input, "AdvanceCompletionCursor")?;
3430        let advanced = effects
3431            .iter()
3432            .find_map(|effect| match (consumer, effect) {
3433                (
3434                    CompletionCursorConsumer::AgentApplied,
3435                    mm_dsl::MeerkatMachineEffect::AgentCompletionCursorAdvanced { cursor },
3436                ) => Some(*cursor),
3437                (
3438                    CompletionCursorConsumer::RuntimeObserved,
3439                    mm_dsl::MeerkatMachineEffect::RuntimeObservedCompletionCursorAdvanced {
3440                        cursor,
3441                    },
3442                ) => Some(*cursor),
3443                (
3444                    CompletionCursorConsumer::RuntimeInjected,
3445                    mm_dsl::MeerkatMachineEffect::RuntimeInjectedCompletionCursorAdvanced {
3446                        cursor,
3447                    },
3448                ) => Some(*cursor),
3449                _ => None,
3450            })
3451            .ok_or_else(|| {
3452                OpsLifecycleError::Internal(format!(
3453                    "generated completion cursor transition emitted no feedback for {consumer:?}"
3454                ))
3455            })?;
3456        if let Some(projection) = projection {
3457            projection.project_authorized_completion_cursor(consumer, advanced);
3458        }
3459        Ok(advanced)
3460    }
3461
3462    fn wait_all(
3463        &self,
3464        run_id: &RunId,
3465        ids: &[OperationId],
3466    ) -> std::pin::Pin<
3467        Box<dyn std::future::Future<Output = Result<WaitAllResult, OpsLifecycleError>> + Send + '_>,
3468    > {
3469        let wait_request_id = WaitRequestId::new();
3470        let owned_ids = ids.to_vec();
3471
3472        let state = match self.write_state() {
3473            Ok(mut state) => {
3474                if let Err(error) = state.ensure_owner_active() {
3475                    return Box::pin(WaitAllFuture {
3476                        registry: self,
3477                        wait_request_id,
3478                        state: WaitAllFutureState::Ready(Some(Err(error))),
3479                    });
3480                }
3481                match state.begin_wait_all_authority(run_id, &wait_request_id, &owned_ids) {
3482                    Ok(WaitAllAuthorityPlan::AlreadySatisfied(satisfied)) => {
3483                        let outcomes =
3484                            state
3485                                .collect_wait_outcomes(&satisfied.operation_ids)
3486                                .map(|outcomes| WaitAllResult {
3487                                    outcomes,
3488                                    satisfied,
3489                                });
3490                        WaitAllFutureState::Ready(Some(outcomes))
3491                    }
3492                    Ok(WaitAllAuthorityPlan::ActivateBarrier) => {
3493                        if state.pending_wait.is_some() {
3494                            // Roll back the DSL barrier we just activated so the
3495                            // registry is not stuck in a wait-active state with
3496                            // no correlation oneshot to resolve. `CancelWaitAll`
3497                            // is the no-obligation clearer (members need not be
3498                            // terminal).
3499                            let rollback = state.dsl_apply(
3500                                mm_dsl::MeerkatMachineInput::CancelWaitAll,
3501                                "CancelWaitAll(rollback)",
3502                            );
3503                            return Box::pin(WaitAllFuture {
3504                                registry: self,
3505                                wait_request_id,
3506                                state: WaitAllFutureState::Ready(Some(Err(match rollback {
3507                                    Ok(()) => OpsLifecycleError::Internal(
3508                                        "wait_all started while a pending wait sender already existed"
3509                                            .into(),
3510                                    ),
3511                                    Err(err) => err,
3512                                }))),
3513                            });
3514                        }
3515                        state.wait_request_id = Some(wait_request_id.clone());
3516                        let (sender, receiver) = tokio::sync::oneshot::channel();
3517                        state.pending_wait = Some(PendingWaitState {
3518                            wait_request_id: wait_request_id.clone(),
3519                            sender,
3520                        });
3521                        WaitAllFutureState::Waiting(receiver)
3522                    }
3523                    Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3524                }
3525            }
3526            Err(err) => WaitAllFutureState::Ready(Some(Err(err))),
3527        };
3528
3529        Box::pin(WaitAllFuture {
3530            registry: self,
3531            wait_request_id,
3532            state,
3533        })
3534    }
3535}
3536
3537#[cfg(test)]
3538#[allow(clippy::unwrap_used, clippy::panic)]
3539mod tests {
3540    use super::*;
3541    use meerkat_core::comms::{PeerId, TrustedPeerDescriptor};
3542    use meerkat_core::lifecycle::RunId;
3543    use meerkat_core::ops_lifecycle::{OperationKind, OpsLifecycleRegistry};
3544    use meerkat_core::types::SessionId;
3545    use std::sync::atomic::Ordering;
3546    use uuid::Uuid;
3547
3548    fn test_run_id() -> RunId {
3549        RunId(Uuid::from_u128(1))
3550    }
3551
3552    fn background_spec(name: &str) -> OperationSpec {
3553        OperationSpec {
3554            id: OperationId::new(),
3555            kind: OperationKind::BackgroundToolOp,
3556            owner_session_id: SessionId::new(),
3557            display_name: name.into(),
3558            source_label: "test".into(),
3559            operation_source: None,
3560            child_session_id: None,
3561            expect_peer_channel: false,
3562        }
3563    }
3564
3565    #[tokio::test]
3566    async fn late_watchers_resolve_immediately() {
3567        let registry = RuntimeOpsLifecycleRegistry::new();
3568        let spec = background_spec("late");
3569        let op_id = spec.id.clone();
3570        registry.register_operation(spec).unwrap();
3571        registry.provisioning_succeeded(&op_id).unwrap();
3572        registry
3573            .complete_operation(
3574                &op_id,
3575                OperationResult {
3576                    id: op_id.clone(),
3577                    content: "done".into(),
3578                    is_error: false,
3579                    duration_ms: 1,
3580                    tokens_used: 0,
3581                },
3582            )
3583            .unwrap();
3584
3585        let watch = registry.register_watcher(&op_id).unwrap();
3586        match watch
3587            .await
3588            .expect("operation completion watch should resolve")
3589        {
3590            OperationTerminalOutcome::Completed(result) => assert_eq!(result.content, "done"),
3591            other => panic!("expected completed outcome, got {other:?}"),
3592        }
3593    }
3594
3595    #[tokio::test]
3596    async fn dropped_watch_sender_is_waiter_error_not_terminal_outcome() {
3597        let (tx, rx) = tokio::sync::oneshot::channel();
3598        let watch = operation_completion_watch_from_receiver(rx);
3599        drop(tx);
3600
3601        assert_eq!(
3602            watch.await,
3603            Err(meerkat_core::ops_lifecycle::OperationCompletionWatchError::ChannelClosed)
3604        );
3605    }
3606
3607    #[test]
3608    fn peer_ready_requires_peer_expectation() {
3609        let registry = RuntimeOpsLifecycleRegistry::new();
3610        let spec = background_spec("no-peer");
3611        let op_id = spec.id.clone();
3612        registry.register_operation(spec).unwrap();
3613        registry.provisioning_succeeded(&op_id).unwrap();
3614
3615        let result = registry.peer_ready(
3616            &op_id,
3617            OperationPeerHandle {
3618                peer_name: meerkat_core::comms::PeerName::new("peer").unwrap(),
3619                trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
3620                    "peer",
3621                    PeerId::new(),
3622                    "inproc://peer",
3623                )
3624                .unwrap(),
3625            },
3626        );
3627        assert!(matches!(result, Err(OpsLifecycleError::PeerNotExpected(_))));
3628    }
3629
3630    /// K8b pinning test: the terminal payload carried through the generated
3631    /// machine IS the typed domain outcome — no JSON codec exists in either
3632    /// direction. Every variant classifies to its matching discriminant, the
3633    /// fail-closed read returns the exact payload when the discriminant
3634    /// matches, and rejects any discriminant/variant disagreement.
3635    #[test]
3636    fn typed_terminal_payload_classifies_and_reads_back_each_variant() {
3637        let op_id = OperationId::new();
3638        let outcomes = vec![
3639            (
3640                OperationTerminalOutcome::Completed(OperationResult {
3641                    id: op_id.clone(),
3642                    content: "done".into(),
3643                    is_error: false,
3644                    duration_ms: 7,
3645                    tokens_used: 42,
3646                }),
3647                mm_dsl::OperationTerminalOutcomeKind::Completed,
3648            ),
3649            (
3650                OperationTerminalOutcome::Failed {
3651                    error: "boom".into(),
3652                },
3653                mm_dsl::OperationTerminalOutcomeKind::Failed,
3654            ),
3655            (
3656                OperationTerminalOutcome::Aborted {
3657                    reason: Some("user aborted".into()),
3658                },
3659                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3660            ),
3661            (
3662                OperationTerminalOutcome::Aborted { reason: None },
3663                mm_dsl::OperationTerminalOutcomeKind::Aborted,
3664            ),
3665            (
3666                OperationTerminalOutcome::Cancelled {
3667                    reason: Some("cancelled".into()),
3668                },
3669                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3670            ),
3671            (
3672                OperationTerminalOutcome::Cancelled { reason: None },
3673                mm_dsl::OperationTerminalOutcomeKind::Cancelled,
3674            ),
3675            (
3676                OperationTerminalOutcome::Retired,
3677                mm_dsl::OperationTerminalOutcomeKind::Retired,
3678            ),
3679            (
3680                OperationTerminalOutcome::Terminated {
3681                    reason: "owner stopped".into(),
3682                },
3683                mm_dsl::OperationTerminalOutcomeKind::Terminated,
3684            ),
3685        ];
3686
3687        for (outcome, expected_kind) in &outcomes {
3688            assert_eq!(
3689                mm_dsl::OperationTerminalOutcomeKind::from(outcome),
3690                *expected_kind,
3691                "typed payload {outcome:?} must classify to {expected_kind:?}"
3692            );
3693            let read = ShellState::checked_terminal_payload(
3694                *expected_kind,
3695                outcome,
3696                "test authority",
3697                "test-op",
3698            )
3699            .expect("matching discriminant must read back the exact payload");
3700            assert_eq!(&read, outcome);
3701        }
3702
3703        // Discriminant/variant disagreement fails closed.
3704        let err = ShellState::checked_terminal_payload(
3705            mm_dsl::OperationTerminalOutcomeKind::Completed,
3706            &OperationTerminalOutcome::Retired,
3707            "test authority",
3708            "test-op",
3709        )
3710        .expect_err("variant mismatch must be rejected");
3711        assert!(matches!(err, OpsLifecycleError::Internal(_)));
3712    }
3713
3714    /// K8b machine-ownership pin: a terminal payload whose variant disagrees
3715    /// with the transition's terminal kind is rejected by the generated
3716    /// machine guard (`payload_variant_matches_kind`) — not by any shell
3717    /// decode step. Drives the DSL input directly to prove the guard owns
3718    /// the invariant.
3719    #[test]
3720    fn generated_guard_rejects_terminal_payload_variant_mismatch() {
3721        let registry = RuntimeOpsLifecycleRegistry::new();
3722        let spec = background_spec("variant-mismatch");
3723        let op_id = spec.id.clone();
3724        registry.register_operation(spec).unwrap();
3725        registry.provisioning_succeeded(&op_id).unwrap();
3726
3727        let mut state = registry.write_state().unwrap();
3728        let err = state
3729            .dsl_apply(
3730                mm_dsl::MeerkatMachineInput::CompleteOp {
3731                    operation_id: mm_dsl::OperationId::from_domain(&op_id).0,
3732                    outcome: mm_dsl::OperationTerminalOutcomeKind::Completed,
3733                    // Variant mismatch: Completed kind with a Retired payload.
3734                    payload: OperationTerminalOutcome::Retired,
3735                },
3736                "CompleteOp",
3737            )
3738            .expect_err("machine must reject payload variant mismatch");
3739        // The kernel reports guard rejections without naming the guard; the
3740        // discriminating pin is the pair: identical input EXCEPT the payload
3741        // variant is guard-rejected here, then accepted below. Status
3742        // (`Running`) and discriminant (`Completed`) are identical in both
3743        // calls, so `payload_variant_matches_kind` is the only differing
3744        // guard.
3745        assert!(
3746            matches!(
3747                &err,
3748                OpsLifecycleError::Internal(message)
3749                    if message.contains("GuardRejected") && message.contains("CompleteOp")
3750            ),
3751            "expected generated guard rejection for CompleteOp, got: {err:?}"
3752        );
3753        drop(state);
3754
3755        // RetireCompletedOp requires the unit Retired payload; a data-carrying
3756        // payload is rejected by the same guard shape.
3757        registry
3758            .complete_operation(
3759                &op_id,
3760                OperationResult {
3761                    id: op_id.clone(),
3762                    content: "done".into(),
3763                    is_error: false,
3764                    duration_ms: 1,
3765                    tokens_used: 0,
3766                },
3767            )
3768            .expect("matching variant must complete");
3769    }
3770
3771    #[test]
3772    fn duplicate_registration_rejection_is_generated() {
3773        let registry = RuntimeOpsLifecycleRegistry::new();
3774        let spec = background_spec("duplicate");
3775        let op_id = spec.id.clone();
3776
3777        registry.register_operation(spec.clone()).unwrap();
3778        let result = registry.register_operation(spec);
3779
3780        assert!(matches!(
3781            result,
3782            Err(OpsLifecycleError::AlreadyRegistered(id)) if id == op_id
3783        ));
3784    }
3785
3786    #[test]
3787    fn invalid_transition_rejection_is_generated() {
3788        let registry = RuntimeOpsLifecycleRegistry::new();
3789        let spec = background_spec("invalid-transition");
3790        let op_id = spec.id.clone();
3791        registry.register_operation(spec).unwrap();
3792
3793        let result = registry.complete_operation(
3794            &op_id,
3795            OperationResult {
3796                id: op_id.clone(),
3797                content: "too-early".into(),
3798                is_error: false,
3799                duration_ms: 1,
3800                tokens_used: 0,
3801            },
3802        );
3803
3804        assert!(matches!(
3805            result,
3806            Err(OpsLifecycleError::InvalidTransition {
3807                id,
3808                status: OperationStatus::Provisioning,
3809                action: "complete_operation",
3810            }) if id == op_id
3811        ));
3812    }
3813
3814    #[tokio::test]
3815    async fn multi_listener_completion() {
3816        let registry = RuntimeOpsLifecycleRegistry::new();
3817        let spec = background_spec("multi");
3818        let op_id = spec.id.clone();
3819        registry.register_operation(spec).unwrap();
3820        registry.provisioning_succeeded(&op_id).unwrap();
3821
3822        let watch1 = registry.register_watcher(&op_id).unwrap();
3823        let watch2 = registry.register_watcher(&op_id).unwrap();
3824        let watch3 = registry.register_watcher(&op_id).unwrap();
3825
3826        registry
3827            .complete_operation(
3828                &op_id,
3829                OperationResult {
3830                    id: op_id.clone(),
3831                    content: "multi-done".into(),
3832                    is_error: false,
3833                    duration_ms: 1,
3834                    tokens_used: 0,
3835                },
3836            )
3837            .unwrap();
3838
3839        for watch in [watch1, watch2, watch3] {
3840            match watch
3841                .await
3842                .expect("operation completion watch should resolve")
3843            {
3844                OperationTerminalOutcome::Completed(result) => {
3845                    assert_eq!(result.content, "multi-done");
3846                }
3847                other => panic!("expected completed, got {other:?}"),
3848            }
3849        }
3850    }
3851
3852    #[tokio::test]
3853    async fn wait_all_returns_all_outcomes() {
3854        let registry = RuntimeOpsLifecycleRegistry::new();
3855
3856        let spec_a = background_spec("a");
3857        let id_a = spec_a.id.clone();
3858        registry.register_operation(spec_a).unwrap();
3859        registry.provisioning_succeeded(&id_a).unwrap();
3860
3861        let spec_b = background_spec("b");
3862        let id_b = spec_b.id.clone();
3863        registry.register_operation(spec_b).unwrap();
3864        registry.provisioning_succeeded(&id_b).unwrap();
3865
3866        registry
3867            .complete_operation(
3868                &id_a,
3869                OperationResult {
3870                    id: id_a.clone(),
3871                    content: "a-done".into(),
3872                    is_error: false,
3873                    duration_ms: 1,
3874                    tokens_used: 0,
3875                },
3876            )
3877            .unwrap();
3878        registry.fail_operation(&id_b, "b-error".into()).unwrap();
3879
3880        let wait_result = registry
3881            .wait_all(&test_run_id(), &[id_a.clone(), id_b.clone()])
3882            .await
3883            .unwrap();
3884        assert_eq!(wait_result.outcomes.len(), 2);
3885        assert_eq!(wait_result.outcomes[0].0, id_a);
3886        assert!(matches!(
3887            wait_result.outcomes[0].1,
3888            OperationTerminalOutcome::Completed(_)
3889        ));
3890        assert_eq!(wait_result.outcomes[1].0, id_b);
3891        assert!(matches!(
3892            wait_result.outcomes[1].1,
3893            OperationTerminalOutcome::Failed { .. }
3894        ));
3895        // Obligation carries the awaited IDs
3896        assert_eq!(wait_result.satisfied.operation_ids.len(), 2);
3897        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3898    }
3899
3900    /// Exercises the trait `wait_all` path (via `dyn OpsLifecycleRegistry`)
3901    /// which must submit WaitAll through the DSL for cross-machine handoff.
3902    #[tokio::test]
3903    async fn wait_all_trait_path_submits_through_authority() {
3904        let registry = RuntimeOpsLifecycleRegistry::new();
3905        let spec = background_spec("trait-wait");
3906        let op_id = spec.id.clone();
3907        registry.register_operation(spec).unwrap();
3908        registry.provisioning_succeeded(&op_id).unwrap();
3909        registry
3910            .complete_operation(
3911                &op_id,
3912                OperationResult {
3913                    id: op_id.clone(),
3914                    content: "done".into(),
3915                    is_error: false,
3916                    duration_ms: 1,
3917                    tokens_used: 0,
3918                },
3919            )
3920            .unwrap();
3921
3922        // Call through trait object to exercise the trait impl, not the inherent method.
3923        let trait_ref: &dyn OpsLifecycleRegistry = &registry;
3924        let wait_result = trait_ref
3925            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3926            .await
3927            .unwrap();
3928        assert_eq!(wait_result.outcomes.len(), 1);
3929        assert!(matches!(
3930            wait_result.outcomes[0].1,
3931            OperationTerminalOutcome::Completed(_)
3932        ));
3933        // Obligation carries the validated ID
3934        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id]);
3935        assert_ne!(wait_result.satisfied.wait_request_id.to_string(), "");
3936        let state = registry.read_state().unwrap();
3937        assert!(
3938            !state.wait_active(),
3939            "already-satisfied wait_all must be cleared by generated satisfaction authority"
3940        );
3941        assert!(state.wait_operation_ids().unwrap().is_empty());
3942    }
3943
3944    #[tokio::test]
3945    async fn wait_all_duplicate_rejection_is_generated() {
3946        let registry = RuntimeOpsLifecycleRegistry::new();
3947        let spec = background_spec("duplicate-wait");
3948        let op_id = spec.id.clone();
3949        registry.register_operation(spec).unwrap();
3950        registry.provisioning_succeeded(&op_id).unwrap();
3951
3952        let result = registry
3953            .wait_all(&test_run_id(), &[op_id.clone(), op_id.clone()])
3954            .await;
3955
3956        assert!(matches!(
3957            result,
3958            Err(OpsLifecycleError::DuplicateWaitOperation(id)) if id == op_id
3959        ));
3960        let state = registry.read_state().unwrap();
3961        assert!(
3962            !state.wait_active(),
3963            "duplicate wait rejection must not create a shell or machine barrier"
3964        );
3965        assert!(state.wait_operation_ids().unwrap().is_empty());
3966    }
3967
3968    #[tokio::test]
3969    async fn wait_all_active_rejection_is_generated() {
3970        let registry = RuntimeOpsLifecycleRegistry::new();
3971        let spec = background_spec("active-wait");
3972        let op_id = spec.id.clone();
3973        registry.register_operation(spec).unwrap();
3974        registry.provisioning_succeeded(&op_id).unwrap();
3975
3976        let active_wait = registry.wait_all(&test_run_id(), std::slice::from_ref(&op_id));
3977        let result = registry
3978            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3979            .await;
3980
3981        assert!(matches!(result, Err(OpsLifecycleError::WaitAlreadyActive)));
3982        drop(active_wait);
3983        let state = registry.read_state().unwrap();
3984        assert!(!state.wait_active());
3985        assert!(state.wait_operation_ids().unwrap().is_empty());
3986    }
3987
3988    #[tokio::test]
3989    async fn wait_all_unknown_operation_rejection_is_generated() {
3990        let registry = RuntimeOpsLifecycleRegistry::new();
3991        let op_id = OperationId::new();
3992
3993        let result = registry
3994            .wait_all(&test_run_id(), std::slice::from_ref(&op_id))
3995            .await;
3996
3997        assert!(matches!(result, Err(OpsLifecycleError::NotFound(id)) if id == op_id));
3998        let state = registry.read_state().unwrap();
3999        assert!(!state.wait_active());
4000        assert!(state.wait_operation_ids().unwrap().is_empty());
4001    }
4002
4003    #[tokio::test]
4004    async fn wait_all_resolves_from_authority_owned_wait_request() {
4005        let registry = RuntimeOpsLifecycleRegistry::new();
4006        let run_id = test_run_id();
4007
4008        let spec = background_spec("pending");
4009        let op_id = spec.id.clone();
4010        registry.register_operation(spec).unwrap();
4011        registry.provisioning_succeeded(&op_id).unwrap();
4012
4013        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
4014        tokio::pin!(wait_fut);
4015        assert!(
4016            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
4017                .await
4018                .is_err()
4019        );
4020
4021        let active_wait_request_id = {
4022            let state = registry.read_state().unwrap();
4023            let wait_request_id = match state.wait_request_id.clone() {
4024                Some(wait_request_id) => wait_request_id,
4025                None => panic!("wait request should be active"),
4026            };
4027            assert_eq!(
4028                state.wait_operation_ids().unwrap().as_slice(),
4029                std::slice::from_ref(&op_id)
4030            );
4031            wait_request_id
4032        };
4033
4034        registry
4035            .complete_operation(
4036                &op_id,
4037                OperationResult {
4038                    id: op_id.clone(),
4039                    content: "done".into(),
4040                    is_error: false,
4041                    duration_ms: 1,
4042                    tokens_used: 0,
4043                },
4044            )
4045            .unwrap();
4046
4047        let wait_result = wait_fut.await.unwrap();
4048        assert_eq!(
4049            wait_result.satisfied.wait_request_id,
4050            active_wait_request_id
4051        );
4052        assert_eq!(wait_result.satisfied.operation_ids, vec![op_id.clone()]);
4053        assert!(matches!(
4054            wait_result.outcomes.as_slice(),
4055            [(returned_id, OperationTerminalOutcome::Completed(_))] if *returned_id == op_id
4056        ));
4057        assert!(registry.read_state().unwrap().wait_request_id.is_none());
4058    }
4059
4060    #[tokio::test]
4061    async fn terminal_transition_rolls_back_publication_when_persistence_fails() {
4062        let registry = RuntimeOpsLifecycleRegistry::new();
4063        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
4064        registry.set_persistence_channel(
4065            tx,
4066            meerkat_core::RuntimeEpochId::new(),
4067            Arc::new(meerkat_core::EpochCursorState::new()),
4068        );
4069        let worker = std::thread::spawn(move || {
4070            let request = rx.blocking_recv().expect("persistence request");
4071            let _ = request.result_tx.send(Err(OpsLifecycleError::Internal(
4072                "injected persistence failure".to_string(),
4073            )));
4074        });
4075
4076        let spec = background_spec("feed-after-persist");
4077        let op_id = spec.id.clone();
4078        registry.register_operation(spec).unwrap();
4079        registry.provisioning_succeeded(&op_id).unwrap();
4080        let watch = registry.register_watcher(&op_id).unwrap();
4081
4082        let err = registry
4083            .complete_operation(
4084                &op_id,
4085                OperationResult {
4086                    id: op_id.clone(),
4087                    content: "done".into(),
4088                    is_error: false,
4089                    duration_ms: 1,
4090                    tokens_used: 0,
4091                },
4092            )
4093            .expect_err("persistence failure must fail the terminal transition");
4094        assert!(
4095            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("injected persistence failure")),
4096            "unexpected persistence error: {err:?}"
4097        );
4098        worker.join().expect("persistence worker");
4099
4100        let feed = registry.completion_feed_handle();
4101        let batch = feed.list_since(0);
4102        assert!(
4103            batch.entries.is_empty(),
4104            "completion feed must not publish entries before durable snapshot success"
4105        );
4106        assert_eq!(batch.watermark, 0);
4107        assert!(
4108            tokio::time::timeout(std::time::Duration::from_millis(10), watch)
4109                .await
4110                .is_err(),
4111            "watchers must not resolve before durable terminal snapshot success"
4112        );
4113        assert_eq!(
4114            registry.snapshot(&op_id).unwrap().unwrap().status,
4115            OperationStatus::Running,
4116            "failed persistence must roll the generated terminal transition back"
4117        );
4118
4119        let (tx, mut rx) = crate::tokio::sync::mpsc::unbounded_channel();
4120        registry.set_persistence_channel(
4121            tx,
4122            meerkat_core::RuntimeEpochId::new(),
4123            Arc::new(meerkat_core::EpochCursorState::new()),
4124        );
4125        let worker = std::thread::spawn(move || {
4126            let request = rx.blocking_recv().expect("second persistence request");
4127            let _ = request.result_tx.send(Ok(()));
4128        });
4129        registry
4130            .complete_operation(
4131                &op_id,
4132                OperationResult {
4133                    id: op_id.clone(),
4134                    content: "done".into(),
4135                    is_error: false,
4136                    duration_ms: 1,
4137                    tokens_used: 0,
4138                },
4139            )
4140            .expect("rolled-back terminal transition should remain retryable");
4141        worker.join().expect("second persistence worker");
4142
4143        let batch = feed.list_since(0);
4144        assert_eq!(batch.entries.len(), 1);
4145        assert_eq!(batch.entries[0].operation_id, op_id);
4146    }
4147
4148    #[test]
4149    fn bounded_completion_feed_reports_exact_evicted_cursor_gap() {
4150        let buffer = Arc::new(FeedBuffer::new(1));
4151        for seq in 1..=2 {
4152            let operation_id = OperationId::new();
4153            buffer.push(CompletionEntry {
4154                seq,
4155                operation_id: operation_id.clone(),
4156                kind: OperationKind::BackgroundToolOp,
4157                display_name: format!("completion-{seq}"),
4158                terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
4159                    id: operation_id,
4160                    content: "done".into(),
4161                    is_error: false,
4162                    duration_ms: 1,
4163                    tokens_used: 0,
4164                }),
4165                completed_at_ms: None,
4166            });
4167        }
4168        let feed = RuntimeCompletionFeed {
4169            buffer,
4170            authority: std::sync::Weak::new(),
4171        };
4172
4173        let batch = feed.list_since(0);
4174        assert_eq!(
4175            batch
4176                .entries
4177                .iter()
4178                .map(|entry| entry.seq)
4179                .collect::<Vec<_>>(),
4180            vec![2]
4181        );
4182        assert_eq!(
4183            batch.gap_after(0),
4184            Some(meerkat_core::completion_feed::CompletionFeedGap {
4185                requested_after_seq: 0,
4186                dropped_through: 1,
4187                watermark: 2,
4188            })
4189        );
4190        assert_eq!(
4191            batch.gap_after(1),
4192            None,
4193            "a cursor at the exact eviction boundary can read the retained suffix"
4194        );
4195    }
4196
4197    #[test]
4198    fn runtime_completion_feed_rehydrates_evicted_obligation_from_generated_authority() {
4199        let registry = RuntimeOpsLifecycleRegistry::new();
4200        {
4201            let state = registry.read_state().unwrap();
4202            let mut buffer = state
4203                .feed_buffer
4204                .inner
4205                .write()
4206                .unwrap_or_else(std::sync::PoisonError::into_inner);
4207            buffer.max_retained = 1;
4208        }
4209
4210        for label in ["first", "second"] {
4211            let spec = background_spec(label);
4212            let operation_id = spec.id.clone();
4213            registry.register_operation(spec).unwrap();
4214            registry.provisioning_succeeded(&operation_id).unwrap();
4215            registry
4216                .complete_operation(
4217                    &operation_id,
4218                    OperationResult {
4219                        id: operation_id.clone(),
4220                        content: "done".into(),
4221                        is_error: false,
4222                        duration_ms: 1,
4223                        tokens_used: 0,
4224                    },
4225                )
4226                .unwrap();
4227        }
4228
4229        let batch = registry.completion_feed_handle().list_since(0);
4230        assert_eq!(
4231            batch
4232                .entries
4233                .iter()
4234                .map(|entry| entry.seq)
4235                .collect::<Vec<_>>(),
4236            vec![1, 2],
4237            "the generated authority must reconstruct the evicted completion"
4238        );
4239        assert_eq!(
4240            batch.gap_after(0),
4241            None,
4242            "successful authority reconstruction closes the bounded projection gap"
4243        );
4244    }
4245
4246    #[test]
4247    fn generated_and_projection_retention_expose_one_typed_gap() {
4248        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4249            max_completed: 1,
4250            max_concurrent: None,
4251        });
4252        for label in ["first", "second"] {
4253            let spec = background_spec(label);
4254            let operation_id = spec.id.clone();
4255            registry.register_operation(spec).unwrap();
4256            registry.provisioning_succeeded(&operation_id).unwrap();
4257            registry
4258                .complete_operation(
4259                    &operation_id,
4260                    OperationResult {
4261                        id: operation_id.clone(),
4262                        content: "done".into(),
4263                        is_error: false,
4264                        duration_ms: 1,
4265                        tokens_used: 0,
4266                    },
4267                )
4268                .unwrap();
4269        }
4270
4271        let batch = registry.completion_feed_handle().list_since(0);
4272        assert_eq!(
4273            batch
4274                .entries
4275                .iter()
4276                .map(|entry| entry.seq)
4277                .collect::<Vec<_>>(),
4278            vec![2]
4279        );
4280        assert_eq!(
4281            batch.gap_after(0),
4282            Some(meerkat_core::completion_feed::CompletionFeedGap {
4283                requested_after_seq: 0,
4284                dropped_through: 1,
4285                watermark: 2,
4286            }),
4287            "authority and projection eviction must meet at one typed retention boundary"
4288        );
4289    }
4290
4291    #[tokio::test]
4292    async fn dropping_wait_all_future_cancels_active_wait_request() {
4293        let registry = RuntimeOpsLifecycleRegistry::new();
4294        let run_id = test_run_id();
4295
4296        let spec = background_spec("cancelled-wait");
4297        let op_id = spec.id.clone();
4298        registry.register_operation(spec).unwrap();
4299        registry.provisioning_succeeded(&op_id).unwrap();
4300
4301        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
4302        drop(wait_fut);
4303
4304        let state = registry.read_state().unwrap();
4305        assert!(state.wait_request_id.is_none());
4306        assert!(state.wait_operation_ids().unwrap().is_empty());
4307        assert!(!state.wait_active());
4308    }
4309
4310    /// id 101: an authority-invariant corruption surfaced during a terminal
4311    /// transition must NOT report the op terminal with a silently-hung barrier.
4312    /// The terminal call must return the typed `Internal` fault, AND the awaited
4313    /// `wait_all` future must resolve to `Err` (via the dropped-sender arm)
4314    /// instead of hanging forever.
4315    #[tokio::test]
4316    async fn satisfy_wait_authority_fault_fails_terminal_and_unblocks_waiter() {
4317        let registry = RuntimeOpsLifecycleRegistry::new();
4318        let run_id = test_run_id();
4319
4320        let spec = background_spec("corrupt-barrier");
4321        let op_id = spec.id.clone();
4322        registry.register_operation(spec).unwrap();
4323        registry.provisioning_succeeded(&op_id).unwrap();
4324
4325        // Activate a barrier whose waiter is still pending.
4326        let wait_fut = registry.wait_all(&run_id, std::slice::from_ref(&op_id));
4327        tokio::pin!(wait_fut);
4328        assert!(
4329            tokio::time::timeout(std::time::Duration::from_millis(10), &mut wait_fut)
4330                .await
4331                .is_err(),
4332            "barrier waiter must still be pending before corruption"
4333        );
4334        assert!(registry.read_state().unwrap().wait_request_id.is_some());
4335
4336        // Corrupt the generated wait authority: an active wait_request_id with
4337        // no wait_run_id forces `try_satisfy_wait_all_authority` down its
4338        // non-GuardRejected `Internal` arm during the terminal transition.
4339        {
4340            let mut state = registry.write_state().unwrap();
4341            let mut machine_state = state.dsl.0.state().clone();
4342            machine_state.wait_run_id = None;
4343            state.dsl = DslAuthority(Box::new(
4344                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4345            ));
4346        }
4347
4348        // The terminal transition must FAIL with the typed authority fault, not
4349        // report the op complete.
4350        let err = registry
4351            .complete_operation(
4352                &op_id,
4353                OperationResult {
4354                    id: op_id.clone(),
4355                    content: "done".into(),
4356                    is_error: false,
4357                    duration_ms: 1,
4358                    tokens_used: 0,
4359                },
4360            )
4361            .expect_err("corrupt wait authority must fail the terminal transition");
4362        assert!(
4363            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("active wait without run id")),
4364            "unexpected terminal error: {err:?}"
4365        );
4366
4367        // The waiter must resolve to Err (dropped sender), not hang.
4368        let waiter_result = tokio::time::timeout(std::time::Duration::from_secs(1), &mut wait_fut)
4369            .await
4370            .expect("waiter must resolve, not hang, after authority corruption");
4371        match waiter_result {
4372            Err(OpsLifecycleError::Internal(message)) => assert!(
4373                message.contains("wait_all completion channel dropped"),
4374                "unexpected waiter error message: {message}"
4375            ),
4376            other => panic!("expected dropped-channel Internal error, got {other:?}"),
4377        }
4378    }
4379
4380    /// id 97: a poisoned registry lock must surface as a typed `Internal` fault
4381    /// from `completion_cursor`, NOT laundered into `Ok(None)` (which means "no
4382    /// generated cursor authority") or a bare `None`.
4383    #[test]
4384    fn completion_cursor_propagates_poison_not_none() {
4385        let registry = std::sync::Arc::new(RuntimeOpsLifecycleRegistry::new());
4386
4387        // Poison the registry RwLock by panicking while holding the write guard.
4388        let poison_registry = std::sync::Arc::clone(&registry);
4389        let join = std::thread::spawn(move || {
4390            let _guard = poison_registry.write_state().unwrap();
4391            panic!("intentional panic to poison ops lifecycle registry lock");
4392        });
4393        assert!(
4394            join.join().is_err(),
4395            "poisoning thread must have panicked while holding the write guard"
4396        );
4397
4398        let trait_ref: &dyn OpsLifecycleRegistry = registry.as_ref();
4399        let result = trait_ref.completion_cursor(CompletionCursorConsumer::AgentApplied);
4400        match result {
4401            Err(OpsLifecycleError::Internal(message)) => assert!(
4402                message.contains("ops lifecycle registry poisoned"),
4403                "unexpected cursor error message: {message}"
4404            ),
4405            other => panic!("poisoned registry must surface typed Internal fault, got {other:?}"),
4406        }
4407    }
4408
4409    #[test]
4410    fn terminate_owner_only_targets_non_terminal_operations() {
4411        let registry = RuntimeOpsLifecycleRegistry::new();
4412
4413        let running_spec = background_spec("running");
4414        let running_id = running_spec.id.clone();
4415        registry.register_operation(running_spec).unwrap();
4416        registry.provisioning_succeeded(&running_id).unwrap();
4417
4418        let completed_spec = background_spec("completed");
4419        let completed_id = completed_spec.id.clone();
4420        registry.register_operation(completed_spec).unwrap();
4421        registry.provisioning_succeeded(&completed_id).unwrap();
4422        registry
4423            .complete_operation(
4424                &completed_id,
4425                OperationResult {
4426                    id: completed_id.clone(),
4427                    content: "done".into(),
4428                    is_error: false,
4429                    duration_ms: 1,
4430                    tokens_used: 0,
4431                },
4432            )
4433            .unwrap();
4434
4435        registry.terminate_owner("shutdown".into()).unwrap();
4436
4437        assert!(matches!(
4438            registry.snapshot(&running_id).unwrap().unwrap().status,
4439            OperationStatus::Terminated
4440        ));
4441        assert!(matches!(
4442            registry.snapshot(&completed_id).unwrap().unwrap().status,
4443            OperationStatus::Completed
4444        ));
4445    }
4446
4447    #[test]
4448    fn unregister_retirement_fences_detached_late_callbacks() {
4449        let registry = RuntimeOpsLifecycleRegistry::new();
4450        let running_spec = background_spec("detached-running");
4451        let running_id = running_spec.id.clone();
4452        registry.register_operation(running_spec).unwrap();
4453        registry.provisioning_succeeded(&running_id).unwrap();
4454
4455        registry
4456            .retire_owner_for_unregister("owner unregistered".into())
4457            .unwrap();
4458        assert!(matches!(
4459            registry.snapshot(&running_id).unwrap().unwrap().status,
4460            OperationStatus::Terminated
4461        ));
4462
4463        let late_result = OperationResult {
4464            id: running_id.clone(),
4465            content: "late detached completion".into(),
4466            is_error: false,
4467            duration_ms: 1,
4468            tokens_used: 0,
4469        };
4470        assert_eq!(
4471            registry.complete_operation(&running_id, late_result),
4472            Err(OpsLifecycleError::OwnerRetired)
4473        );
4474        assert_eq!(
4475            registry.register_operation(background_spec("late-new-op")),
4476            Err(OpsLifecycleError::OwnerRetired)
4477        );
4478        assert_eq!(
4479            registry.report_progress(
4480                &running_id,
4481                OperationProgressUpdate {
4482                    message: "late progress".into(),
4483                    percent: None,
4484                },
4485            ),
4486            Err(OpsLifecycleError::OwnerRetired)
4487        );
4488    }
4489
4490    #[test]
4491    fn collect_completed_drains_terminal_operations() {
4492        let registry = RuntimeOpsLifecycleRegistry::new();
4493
4494        let spec_a = background_spec("a");
4495        let id_a = spec_a.id.clone();
4496        registry.register_operation(spec_a).unwrap();
4497        registry.provisioning_succeeded(&id_a).unwrap();
4498        registry
4499            .complete_operation(
4500                &id_a,
4501                OperationResult {
4502                    id: id_a.clone(),
4503                    content: "done".into(),
4504                    is_error: false,
4505                    duration_ms: 1,
4506                    tokens_used: 0,
4507                },
4508            )
4509            .unwrap();
4510
4511        let spec_b = background_spec("b");
4512        let id_b = spec_b.id.clone();
4513        registry.register_operation(spec_b).unwrap();
4514
4515        let collected = registry.collect_completed().unwrap();
4516        assert_eq!(collected.len(), 1);
4517        assert_eq!(collected[0].0, id_a);
4518
4519        assert!(registry.snapshot(&id_a).unwrap().is_none());
4520        assert!(registry.snapshot(&id_b).unwrap().is_some());
4521
4522        let collected2 = registry.collect_completed().unwrap();
4523        assert!(collected2.is_empty());
4524    }
4525
4526    #[test]
4527    fn bounded_completed_retention_evicts_oldest() {
4528        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
4529            max_completed: 3,
4530            max_concurrent: None,
4531        });
4532
4533        let mut ids = Vec::new();
4534        for i in 0..5 {
4535            let spec = background_spec(&format!("op-{i}"));
4536            let id = spec.id.clone();
4537            registry.register_operation(spec).unwrap();
4538            registry.provisioning_succeeded(&id).unwrap();
4539            registry
4540                .complete_operation(
4541                    &id,
4542                    OperationResult {
4543                        id: id.clone(),
4544                        content: format!("done-{i}"),
4545                        is_error: false,
4546                        duration_ms: 1,
4547                        tokens_used: 0,
4548                    },
4549                )
4550                .unwrap();
4551            ids.push(id);
4552        }
4553
4554        assert!(registry.snapshot(&ids[0]).unwrap().is_none());
4555        assert!(registry.snapshot(&ids[1]).unwrap().is_none());
4556        assert!(registry.snapshot(&ids[2]).unwrap().is_some());
4557        assert!(registry.snapshot(&ids[3]).unwrap().is_some());
4558        assert!(registry.snapshot(&ids[4]).unwrap().is_some());
4559    }
4560
4561    #[test]
4562    fn recovered_snapshot_retains_only_machine_accepted_terminal_records() {
4563        let registry = RuntimeOpsLifecycleRegistry::new();
4564
4565        let completed_spec = background_spec("completed");
4566        let completed_id = completed_spec.id.clone();
4567        registry.register_operation(completed_spec).unwrap();
4568        registry.provisioning_succeeded(&completed_id).unwrap();
4569        registry
4570            .complete_operation(
4571                &completed_id,
4572                OperationResult {
4573                    id: completed_id.clone(),
4574                    content: "done".into(),
4575                    is_error: false,
4576                    duration_ms: 1,
4577                    tokens_used: 0,
4578                },
4579            )
4580            .unwrap();
4581
4582        let running_spec = background_spec("running");
4583        let running_id = running_spec.id.clone();
4584        registry.register_operation(running_spec).unwrap();
4585        registry.provisioning_succeeded(&running_id).unwrap();
4586
4587        let cursor_state = meerkat_core::EpochCursorState::new();
4588        let snapshot = registry
4589            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4590            .unwrap();
4591        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4592
4593        assert!(recovered.snapshot(&completed_id).unwrap().is_some());
4594        assert!(recovered.snapshot(&running_id).unwrap().is_none());
4595
4596        let collected = recovered.collect_completed().unwrap();
4597        assert_eq!(collected.len(), 1);
4598        assert_eq!(collected[0].0, completed_id);
4599    }
4600
4601    #[test]
4602    fn recovered_snapshot_retains_running_detached_job_wait_binding() {
4603        let registry = RuntimeOpsLifecycleRegistry::new();
4604        let owner_session_id = SessionId::new();
4605        let job_id = "job_recovery_wait";
4606        let operation_id = OperationId::for_detached_job_wait(&owner_session_id, "realm-a", job_id);
4607        let source = OperationSource::detached_job("realm-a", job_id);
4608        registry
4609            .register_operation(OperationSpec {
4610                id: operation_id.clone(),
4611                kind: OperationKind::DetachedJobWait,
4612                owner_session_id: owner_session_id.clone(),
4613                display_name: "await detached job".into(),
4614                source_label: "await_job".into(),
4615                operation_source: Some(source.clone()),
4616                child_session_id: None,
4617                expect_peer_channel: false,
4618            })
4619            .unwrap();
4620        registry.provisioning_succeeded(&operation_id).unwrap();
4621
4622        let cursor_state = meerkat_core::EpochCursorState::new();
4623        let snapshot = registry
4624            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4625            .unwrap();
4626        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4627        let wait = recovered
4628            .snapshot(&operation_id)
4629            .unwrap()
4630            .expect("running await binding survives recovery");
4631
4632        assert_eq!(wait.status, OperationStatus::Running);
4633        assert_eq!(wait.owner_session_id, owner_session_id);
4634        assert_eq!(wait.operation_source, Some(source));
4635        assert!(wait.terminal_outcome.is_none());
4636        assert!(
4637            recovered
4638                .completion_feed()
4639                .expect("completion feed")
4640                .list_since(0)
4641                .entries
4642                .is_empty(),
4643            "recovery of a wait binding must not mint completion"
4644        );
4645    }
4646
4647    #[test]
4648    fn capacity_slot_terminal_is_not_persisted_or_recovered() {
4649        let registry = RuntimeOpsLifecycleRegistry::new();
4650
4651        let mut spec = background_spec("capacity");
4652        spec.kind = OperationKind::BackgroundToolCapacitySlot;
4653        let operation_id = spec.id.clone();
4654        registry.register_operation(spec).unwrap();
4655        registry.provisioning_succeeded(&operation_id).unwrap();
4656        registry.mark_retired(&operation_id).unwrap();
4657
4658        assert!(registry.snapshot(&operation_id).unwrap().is_none());
4659
4660        let cursor_state = meerkat_core::EpochCursorState::new();
4661        let snapshot = registry
4662            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4663            .unwrap();
4664        assert!(
4665            !snapshot
4666                .authority_state
4667                .operations
4668                .contains_key(&operation_id)
4669        );
4670        assert!(!snapshot.operation_specs.contains_key(&operation_id));
4671        assert!(snapshot.completion_entries.is_empty());
4672
4673        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4674        assert!(recovered.snapshot(&operation_id).unwrap().is_none());
4675    }
4676
4677    #[test]
4678    fn rollback_unreturned_background_operation_restores_absence_without_completion() {
4679        let registry = RuntimeOpsLifecycleRegistry::new();
4680        let spec = background_spec("unreturned");
4681        let operation_id = spec.id.clone();
4682
4683        registry
4684            .register_operation(spec)
4685            .expect("register unreturned operation");
4686        registry
4687            .provisioning_succeeded(&operation_id)
4688            .expect("start unreturned operation");
4689        registry
4690            .rollback_unreturned_operation(&operation_id)
4691            .expect("generated rollback should restore absence");
4692
4693        assert!(
4694            registry
4695                .snapshot(&operation_id)
4696                .expect("snapshot after rollback")
4697                .is_none(),
4698            "never-returned operation must not retain public lifecycle state"
4699        );
4700        let feed = registry
4701            .completion_feed()
4702            .expect("runtime registry exposes completion feed");
4703        assert!(
4704            feed.list_since(0).entries.is_empty(),
4705            "never-returned operation must not mint a phantom completion"
4706        );
4707    }
4708
4709    #[test]
4710    fn recovered_snapshot_uses_authority_operation_source() {
4711        let registry = RuntimeOpsLifecycleRegistry::new();
4712        let child_session_id = SessionId::new();
4713        let operation_source = OperationSource::session_child(child_session_id.clone());
4714        let spec = OperationSpec {
4715            id: OperationId::new(),
4716            kind: OperationKind::MobMemberChild,
4717            owner_session_id: SessionId::new(),
4718            display_name: "source-recovery".into(),
4719            source_label: "test".into(),
4720            operation_source: Some(operation_source.clone()),
4721            child_session_id: Some(child_session_id),
4722            expect_peer_channel: true,
4723        };
4724        let operation_id = spec.id.clone();
4725
4726        registry.register_operation(spec).unwrap();
4727        registry.provisioning_succeeded(&operation_id).unwrap();
4728        registry.mark_retired(&operation_id).unwrap();
4729
4730        let cursor_state = meerkat_core::EpochCursorState::new();
4731        let mut snapshot = registry
4732            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4733            .unwrap();
4734        assert_eq!(
4735            snapshot
4736                .authority_state
4737                .operations
4738                .get(&operation_id)
4739                .and_then(|state| state.operation_source.as_ref()),
4740            Some(&operation_source)
4741        );
4742
4743        snapshot
4744            .operation_specs
4745            .get_mut(&operation_id)
4746            .expect("persisted spec")
4747            .operation_source = None;
4748        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
4749        assert_eq!(
4750            recovered
4751                .snapshot(&operation_id)
4752                .unwrap()
4753                .unwrap()
4754                .operation_source,
4755            Some(operation_source)
4756        );
4757    }
4758
4759    #[test]
4760    fn recovered_snapshot_rejects_operation_source_mirror_drift() {
4761        let registry = RuntimeOpsLifecycleRegistry::new();
4762        let child_session_id = SessionId::new();
4763        let operation_source = OperationSource::session_child(child_session_id.clone());
4764        let spec = OperationSpec {
4765            id: OperationId::new(),
4766            kind: OperationKind::MobMemberChild,
4767            owner_session_id: SessionId::new(),
4768            display_name: "source-drift".into(),
4769            source_label: "test".into(),
4770            operation_source: Some(operation_source),
4771            child_session_id: Some(child_session_id),
4772            expect_peer_channel: true,
4773        };
4774        let operation_id = spec.id.clone();
4775
4776        registry.register_operation(spec).unwrap();
4777        registry.provisioning_succeeded(&operation_id).unwrap();
4778        registry.mark_retired(&operation_id).unwrap();
4779
4780        let cursor_state = meerkat_core::EpochCursorState::new();
4781        let mut snapshot = registry
4782            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4783            .unwrap();
4784        snapshot
4785            .operation_specs
4786            .get_mut(&operation_id)
4787            .expect("persisted spec")
4788            .operation_source = Some(OperationSource::session_child(SessionId::new()));
4789
4790        let err = RuntimeOpsLifecycleRegistry::from_recovered(snapshot)
4791            .expect_err("source mirror drift must fail recovery");
4792        assert!(
4793            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("operation source mirror")),
4794            "unexpected recovery error: {err:?}"
4795        );
4796    }
4797
4798    #[test]
4799    fn persisted_authority_state_serializes_explicit_no_operation_source() {
4800        let registry = RuntimeOpsLifecycleRegistry::new();
4801
4802        let spec = background_spec("explicit-no-source");
4803        let operation_id = spec.id.clone();
4804        registry.register_operation(spec).unwrap();
4805        registry.provisioning_succeeded(&operation_id).unwrap();
4806
4807        let cursor_state = meerkat_core::EpochCursorState::new();
4808        let snapshot = registry
4809            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4810            .unwrap();
4811        let value = serde_json::to_value(&snapshot).unwrap();
4812        let operations = value
4813            .get("authority_state")
4814            .and_then(|state| state.get("operations"))
4815            .and_then(serde_json::Value::as_object)
4816            .expect("serialized authority operations");
4817        let persisted_state = operations
4818            .values()
4819            .next()
4820            .and_then(serde_json::Value::as_object)
4821            .expect("serialized operation state");
4822
4823        assert!(
4824            persisted_state
4825                .get("operation_source")
4826                .is_some_and(serde_json::Value::is_null),
4827            "generated explicit no-source fact must be serialized as present null: {persisted_state:?}"
4828        );
4829
4830        let recovered_snapshot = serde_json::from_value::<PersistedOpsSnapshot>(value).unwrap();
4831        assert_eq!(
4832            recovered_snapshot
4833                .authority_state
4834                .operations
4835                .get(&operation_id)
4836                .expect("round-tripped operation")
4837                .operation_source,
4838            None
4839        );
4840    }
4841
4842    #[test]
4843    fn persisted_authority_state_rejects_missing_operation_source_fact() {
4844        let registry = RuntimeOpsLifecycleRegistry::new();
4845
4846        let spec = background_spec("missing-source-fact");
4847        registry.register_operation(spec).unwrap();
4848
4849        let cursor_state = meerkat_core::EpochCursorState::new();
4850        let snapshot = registry
4851            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4852            .unwrap();
4853        let mut value = serde_json::to_value(&snapshot).unwrap();
4854        let operations = value
4855            .get_mut("authority_state")
4856            .and_then(|state| state.get_mut("operations"))
4857            .and_then(serde_json::Value::as_object_mut)
4858            .expect("serialized authority operations");
4859        let operation_state = operations
4860            .values_mut()
4861            .next()
4862            .and_then(serde_json::Value::as_object_mut)
4863            .expect("serialized operation state");
4864        assert!(operation_state.remove("operation_source").is_some());
4865
4866        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4867            .expect_err("missing generated source fact must fail recovery snapshot decoding");
4868        assert!(
4869            err.to_string().contains("operation_source"),
4870            "unexpected decode error: {err}"
4871        );
4872    }
4873
4874    #[test]
4875    fn persisted_authority_state_rejects_missing_completion_feed_authority() {
4876        let registry = RuntimeOpsLifecycleRegistry::new();
4877
4878        let spec = background_spec("missing-feed-authority");
4879        let operation_id = spec.id.clone();
4880        registry.register_operation(spec).unwrap();
4881        registry.provisioning_succeeded(&operation_id).unwrap();
4882        registry
4883            .complete_operation(
4884                &operation_id,
4885                OperationResult {
4886                    id: operation_id.clone(),
4887                    content: "done".into(),
4888                    is_error: false,
4889                    duration_ms: 1,
4890                    tokens_used: 0,
4891                },
4892            )
4893            .unwrap();
4894
4895        let cursor_state = meerkat_core::EpochCursorState::new();
4896        let snapshot = registry
4897            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4898            .unwrap();
4899        let mut value = serde_json::to_value(&snapshot).unwrap();
4900        let authority_state = value
4901            .get_mut("authority_state")
4902            .and_then(serde_json::Value::as_object_mut)
4903            .expect("serialized authority state");
4904        assert!(authority_state.remove("completion_feed_entries").is_some());
4905
4906        let err = serde_json::from_value::<PersistedOpsSnapshot>(value)
4907            .expect_err("missing generated feed authority must fail recovery snapshot decoding");
4908        assert!(
4909            err.to_string().contains("completion_feed_entries"),
4910            "unexpected decode error: {err}"
4911        );
4912    }
4913
4914    #[test]
4915    fn public_child_session_projection_uses_authority_operation_source() {
4916        let registry = RuntimeOpsLifecycleRegistry::new();
4917        let authority_child_session_id = SessionId::new();
4918        let stale_shell_child_session_id = SessionId::new();
4919        let operation_source = OperationSource::session_child(authority_child_session_id.clone());
4920        let spec = OperationSpec {
4921            id: OperationId::new(),
4922            kind: OperationKind::MobMemberChild,
4923            owner_session_id: SessionId::new(),
4924            display_name: "child-projection".into(),
4925            source_label: "test".into(),
4926            operation_source: Some(operation_source),
4927            child_session_id: Some(stale_shell_child_session_id),
4928            expect_peer_channel: true,
4929        };
4930        let operation_id = spec.id.clone();
4931
4932        registry.register_operation(spec).unwrap();
4933
4934        assert_eq!(
4935            registry
4936                .snapshot(&operation_id)
4937                .unwrap()
4938                .unwrap()
4939                .child_session_id,
4940            Some(authority_child_session_id.clone())
4941        );
4942
4943        let cursor_state = meerkat_core::EpochCursorState::new();
4944        let snapshot = registry
4945            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
4946            .unwrap();
4947        assert_eq!(
4948            snapshot
4949                .operation_specs
4950                .get(&operation_id)
4951                .expect("persisted spec")
4952                .child_session_id,
4953            Some(authority_child_session_id)
4954        );
4955    }
4956
4957    #[test]
4958    fn generated_terminal_payload_projection_fails_closed() {
4959        let registry = RuntimeOpsLifecycleRegistry::new();
4960
4961        let spec = background_spec("terminal-payload-drift");
4962        let operation_id = spec.id.clone();
4963        registry.register_operation(spec).unwrap();
4964        registry.provisioning_succeeded(&operation_id).unwrap();
4965        registry
4966            .complete_operation(
4967                &operation_id,
4968                OperationResult {
4969                    id: operation_id.clone(),
4970                    content: "done".into(),
4971                    is_error: false,
4972                    duration_ms: 1,
4973                    tokens_used: 0,
4974                },
4975            )
4976            .unwrap();
4977
4978        {
4979            let mut state = registry.write_state().unwrap();
4980            let mut machine_state = state.dsl.0.state().clone();
4981            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
4982            machine_state
4983                .op_terminal_payload
4984                .insert(operation_id_key, OperationTerminalOutcome::Retired);
4985            state.dsl = DslAuthority(Box::new(
4986                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
4987            ));
4988        }
4989
4990        let err = match registry.register_watcher(&operation_id) {
4991            Ok(_) => panic!("invalid generated terminal payload must reject watcher projection"),
4992            Err(err) => err,
4993        };
4994        assert!(
4995            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
4996            "unexpected watcher error: {err:?}"
4997        );
4998        let err = registry
4999            .snapshot(&operation_id)
5000            .expect_err("invalid generated terminal payload must reject public snapshot");
5001        assert!(
5002            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("does not match terminal outcome discriminant")),
5003            "unexpected public snapshot error: {err:?}"
5004        );
5005
5006        let cursor_state = meerkat_core::EpochCursorState::new();
5007        let err = match registry
5008            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5009        {
5010            Ok(_) => panic!("invalid generated terminal payload must reject persistence snapshot"),
5011            Err(err) => err,
5012        };
5013        assert!(
5014            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
5015            "unexpected snapshot error: {err:?}"
5016        );
5017
5018        let err = match registry.collect_completed() {
5019            Ok(_) => panic!("invalid generated terminal payload must reject collection"),
5020            Err(err) => err,
5021        };
5022        assert!(
5023            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("payload variant") && message.contains("does not match terminal outcome discriminant")),
5024            "unexpected collection error: {err:?}"
5025        );
5026    }
5027
5028    #[test]
5029    fn generated_terminal_payload_missing_projection_fails_closed() {
5030        let registry = RuntimeOpsLifecycleRegistry::new();
5031
5032        let spec = background_spec("terminal-payload-missing");
5033        let operation_id = spec.id.clone();
5034        registry.register_operation(spec).unwrap();
5035        registry.provisioning_succeeded(&operation_id).unwrap();
5036        registry
5037            .fail_operation(&operation_id, "boom".into())
5038            .unwrap();
5039
5040        {
5041            let mut state = registry.write_state().unwrap();
5042            let mut machine_state = state.dsl.0.state().clone();
5043            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5044            machine_state.op_terminal_payload.remove(&operation_id_key);
5045            state.dsl = DslAuthority(Box::new(
5046                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5047            ));
5048        }
5049
5050        let err = match registry.register_watcher(&operation_id) {
5051            Ok(_) => panic!("missing generated terminal payload must reject watcher projection"),
5052            Err(err) => err,
5053        };
5054        assert!(
5055            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
5056            "unexpected watcher error: {err:?}"
5057        );
5058        let err = registry
5059            .snapshot(&operation_id)
5060            .expect_err("missing generated terminal payload must reject public snapshot");
5061        assert!(
5062            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal payload")),
5063            "unexpected public snapshot error: {err:?}"
5064        );
5065    }
5066
5067    #[test]
5068    fn generated_terminal_status_without_outcome_fails_closed() {
5069        let registry = RuntimeOpsLifecycleRegistry::new();
5070
5071        let spec = background_spec("terminal-outcome-missing");
5072        let operation_id = spec.id.clone();
5073        registry.register_operation(spec).unwrap();
5074        registry.provisioning_succeeded(&operation_id).unwrap();
5075        registry
5076            .fail_operation(&operation_id, "boom".into())
5077            .unwrap();
5078
5079        {
5080            let mut state = registry.write_state().unwrap();
5081            let mut machine_state = state.dsl.0.state().clone();
5082            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5083            machine_state.op_terminal_outcomes.remove(&operation_id_key);
5084            state.dsl = DslAuthority(Box::new(
5085                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5086            ));
5087        }
5088
5089        let err = registry
5090            .snapshot(&operation_id)
5091            .expect_err("terminal status without outcome must reject public snapshot");
5092        assert!(
5093            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
5094            "unexpected public snapshot error: {err:?}"
5095        );
5096
5097        let err = match registry.collect_completed() {
5098            Ok(_) => panic!("terminal status without outcome must reject collection"),
5099            Err(err) => err,
5100        };
5101        assert!(
5102            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing terminal outcome")),
5103            "unexpected collection error: {err:?}"
5104        );
5105    }
5106
5107    #[test]
5108    fn generated_operation_source_projection_fails_closed() {
5109        let registry = RuntimeOpsLifecycleRegistry::new();
5110        let child_session_id = SessionId::new();
5111        let operation_source = OperationSource::session_child(child_session_id.clone());
5112        let spec = OperationSpec {
5113            id: OperationId::new(),
5114            kind: OperationKind::MobMemberChild,
5115            owner_session_id: SessionId::new(),
5116            display_name: "source-authority-drift".into(),
5117            source_label: "test".into(),
5118            operation_source: Some(operation_source),
5119            child_session_id: Some(child_session_id),
5120            expect_peer_channel: true,
5121        };
5122        let operation_id = spec.id.clone();
5123
5124        registry.register_operation(spec).unwrap();
5125
5126        {
5127            let mut state = registry.write_state().unwrap();
5128            let mut machine_state = state.dsl.0.state().clone();
5129            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5130            machine_state
5131                .op_sources
5132                .get_mut(&operation_id_key)
5133                .expect("generated operation source")
5134                .session_id = None;
5135            state.dsl = DslAuthority(Box::new(
5136                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5137            ));
5138        }
5139
5140        let err = registry
5141            .snapshot(&operation_id)
5142            .expect_err("invalid generated operation source must reject public snapshot");
5143        assert!(
5144            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
5145            "unexpected public snapshot error: {err:?}"
5146        );
5147        let err = registry
5148            .list_operations()
5149            .expect_err("invalid generated operation source must reject public operation list");
5150        assert!(
5151            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
5152            "unexpected operation list error: {err:?}"
5153        );
5154
5155        let cursor_state = meerkat_core::EpochCursorState::new();
5156        let err = match registry
5157            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5158        {
5159            Ok(_) => panic!("invalid generated operation source must reject persistence snapshot"),
5160            Err(err) => err,
5161        };
5162        assert!(
5163            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("generated operation source authority has invalid source")),
5164            "unexpected snapshot error: {err:?}"
5165        );
5166    }
5167
5168    #[test]
5169    fn generated_operation_id_projection_fails_closed() {
5170        let registry = RuntimeOpsLifecycleRegistry::new();
5171
5172        {
5173            let mut state = registry.write_state().unwrap();
5174            let mut machine_state = state.dsl.0.state().clone();
5175            machine_state.op_statuses.insert(
5176                "not-json-operation-id".into(),
5177                mm_dsl::OperationStatus::Running,
5178            );
5179            state.dsl = DslAuthority(Box::new(
5180                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5181            ));
5182        }
5183
5184        let err = registry
5185            .list_operations()
5186            .expect_err("invalid generated operation id must reject public operation list");
5187        assert!(
5188            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
5189            "unexpected operation list error: {err:?}"
5190        );
5191
5192        let cursor_state = meerkat_core::EpochCursorState::new();
5193        let err = match registry
5194            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5195        {
5196            Ok(_) => panic!("invalid generated operation id must reject persistence snapshot"),
5197            Err(err) => err,
5198        };
5199        assert!(
5200            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("invalid operation id key")),
5201            "unexpected persistence snapshot error: {err:?}"
5202        );
5203    }
5204
5205    #[test]
5206    fn generated_missing_kind_projection_fails_closed() {
5207        let registry = RuntimeOpsLifecycleRegistry::new();
5208        let spec = background_spec("missing-kind");
5209        let operation_id = spec.id.clone();
5210        registry.register_operation(spec).unwrap();
5211
5212        {
5213            let mut state = registry.write_state().unwrap();
5214            let mut machine_state = state.dsl.0.state().clone();
5215            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5216            machine_state.op_kinds.remove(&operation_id_key);
5217            state.dsl = DslAuthority(Box::new(
5218                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5219            ));
5220        }
5221
5222        let err = registry
5223            .snapshot(&operation_id)
5224            .expect_err("missing generated kind must reject public snapshot");
5225        assert!(
5226            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
5227            "unexpected public snapshot error: {err:?}"
5228        );
5229        let err = registry
5230            .list_operations()
5231            .expect_err("missing generated kind must reject public list");
5232        assert!(
5233            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
5234            "unexpected public list error: {err:?}"
5235        );
5236
5237        let cursor_state = meerkat_core::EpochCursorState::new();
5238        let err = registry
5239            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5240            .expect_err("missing generated kind must reject persistence snapshot");
5241        assert!(
5242            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing kind")),
5243            "unexpected persistence snapshot error: {err:?}"
5244        );
5245    }
5246
5247    #[test]
5248    fn generated_missing_status_projection_fails_closed() {
5249        let registry = RuntimeOpsLifecycleRegistry::new();
5250        let spec = background_spec("missing-status");
5251        let operation_id = spec.id.clone();
5252        registry.register_operation(spec).unwrap();
5253
5254        {
5255            let mut state = registry.write_state().unwrap();
5256            let mut machine_state = state.dsl.0.state().clone();
5257            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5258            machine_state.op_statuses.remove(&operation_id_key);
5259            state.dsl = DslAuthority(Box::new(
5260                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5261            ));
5262        }
5263
5264        let err = registry
5265            .snapshot(&operation_id)
5266            .expect_err("missing generated status must reject public snapshot");
5267        assert!(
5268            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
5269            "unexpected public snapshot error: {err:?}"
5270        );
5271        let err = registry
5272            .list_operations()
5273            .expect_err("missing generated status must reject public list");
5274        assert!(
5275            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
5276            "unexpected public list error: {err:?}"
5277        );
5278        let err = registry
5279            .classify_operation_public_result(&operation_id)
5280            .expect_err("missing generated status must reject public-result classification");
5281        assert!(
5282            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing status")),
5283            "unexpected public-result error: {err:?}"
5284        );
5285    }
5286
5287    #[test]
5288    fn generated_retiring_public_result_remains_running_until_terminal() {
5289        let registry = RuntimeOpsLifecycleRegistry::new();
5290        let spec = background_spec("retiring-public-result");
5291        let operation_id = spec.id.clone();
5292        registry.register_operation(spec).unwrap();
5293        registry.provisioning_succeeded(&operation_id).unwrap();
5294        registry.request_retire(&operation_id).unwrap();
5295
5296        let snapshot = registry.snapshot(&operation_id).unwrap().unwrap();
5297        assert_eq!(snapshot.status, OperationStatus::Retiring);
5298        assert!(snapshot.terminal_outcome.is_none());
5299        assert!(!snapshot.terminal);
5300        assert_eq!(
5301            snapshot.public_result_class,
5302            OperationPublicResultClass::Running
5303        );
5304        assert_eq!(
5305            registry
5306                .classify_operation_public_result(&operation_id)
5307                .unwrap(),
5308            OperationPublicResultClass::Running
5309        );
5310    }
5311
5312    #[test]
5313    fn generated_missing_peer_ready_projection_fails_closed() {
5314        let registry = RuntimeOpsLifecycleRegistry::new();
5315        let spec = background_spec("missing-peer-ready");
5316        let operation_id = spec.id.clone();
5317        registry.register_operation(spec).unwrap();
5318
5319        {
5320            let mut state = registry.write_state().unwrap();
5321            let mut machine_state = state.dsl.0.state().clone();
5322            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5323            machine_state.op_peer_ready.remove(&operation_id_key);
5324            state.dsl = DslAuthority(Box::new(
5325                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5326            ));
5327        }
5328
5329        let err = registry
5330            .snapshot(&operation_id)
5331            .expect_err("missing generated peer-ready fact must reject public snapshot");
5332        assert!(
5333            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
5334            "unexpected public snapshot error: {err:?}"
5335        );
5336
5337        let cursor_state = meerkat_core::EpochCursorState::new();
5338        let err = registry
5339            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5340            .expect_err("missing generated peer-ready fact must reject persistence snapshot");
5341        assert!(
5342            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing peer-ready")),
5343            "unexpected persistence snapshot error: {err:?}"
5344        );
5345    }
5346
5347    #[test]
5348    fn generated_missing_progress_count_projection_fails_closed() {
5349        let registry = RuntimeOpsLifecycleRegistry::new();
5350        let spec = background_spec("missing-progress-count");
5351        let operation_id = spec.id.clone();
5352        registry.register_operation(spec).unwrap();
5353
5354        {
5355            let mut state = registry.write_state().unwrap();
5356            let mut machine_state = state.dsl.0.state().clone();
5357            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5358            machine_state.op_progress_counts.remove(&operation_id_key);
5359            state.dsl = DslAuthority(Box::new(
5360                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5361            ));
5362        }
5363
5364        let err = registry
5365            .snapshot(&operation_id)
5366            .expect_err("missing generated progress count must reject public snapshot");
5367        assert!(
5368            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
5369            "unexpected public snapshot error: {err:?}"
5370        );
5371
5372        let cursor_state = meerkat_core::EpochCursorState::new();
5373        let err = registry
5374            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5375            .expect_err("missing generated progress count must reject persistence snapshot");
5376        assert!(
5377            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing progress count")),
5378            "unexpected persistence snapshot error: {err:?}"
5379        );
5380    }
5381
5382    #[test]
5383    fn generated_terminal_sequence_missing_persistence_fails_closed() {
5384        let registry = RuntimeOpsLifecycleRegistry::new();
5385        let spec = background_spec("terminal-sequence-missing");
5386        let operation_id = spec.id.clone();
5387        registry.register_operation(spec).unwrap();
5388        registry.provisioning_succeeded(&operation_id).unwrap();
5389        registry
5390            .complete_operation(
5391                &operation_id,
5392                OperationResult {
5393                    id: operation_id.clone(),
5394                    content: "done".into(),
5395                    is_error: false,
5396                    duration_ms: 1,
5397                    tokens_used: 0,
5398                },
5399            )
5400            .unwrap();
5401
5402        {
5403            let mut state = registry.write_state().unwrap();
5404            let mut machine_state = state.dsl.0.state().clone();
5405            let operation_id_key = mm_dsl::OperationId::from_domain(&operation_id).0;
5406            machine_state.op_completion_seq.remove(&operation_id_key);
5407            state.dsl = DslAuthority(Box::new(
5408                mm_dsl::MeerkatMachineAuthority::recover_from_state(machine_state).unwrap(),
5409            ));
5410        }
5411
5412        let cursor_state = meerkat_core::EpochCursorState::new();
5413        let err = registry
5414            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5415            .expect_err("missing generated terminal sequence must reject persistence snapshot");
5416        assert!(
5417            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("missing completion sequence")),
5418            "unexpected persistence snapshot error: {err:?}"
5419        );
5420    }
5421
5422    #[test]
5423    fn generated_record_without_shell_projection_fails_closed() {
5424        let registry = RuntimeOpsLifecycleRegistry::new();
5425        let spec = background_spec("missing-shell-record");
5426        let operation_id = spec.id.clone();
5427        registry.register_operation(spec).unwrap();
5428
5429        {
5430            let mut state = registry.write_state().unwrap();
5431            state.records.remove(&operation_id);
5432        }
5433
5434        let err = registry
5435            .snapshot(&operation_id)
5436            .expect_err("generated operation without shell record must reject public snapshot");
5437        assert!(
5438            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5439            "unexpected public snapshot error: {err:?}"
5440        );
5441        let err = registry
5442            .list_operations()
5443            .expect_err("generated operation without shell record must reject public list");
5444        assert!(
5445            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("without shell projection record")),
5446            "unexpected public list error: {err:?}"
5447        );
5448    }
5449
5450    #[test]
5451    fn recovered_snapshot_rebuilds_child_session_mirror_from_authority() {
5452        let registry = RuntimeOpsLifecycleRegistry::new();
5453        let child_session_id = SessionId::new();
5454        let operation_source = OperationSource::session_child(child_session_id.clone());
5455        let spec = OperationSpec {
5456            id: OperationId::new(),
5457            kind: OperationKind::MobMemberChild,
5458            owner_session_id: SessionId::new(),
5459            display_name: "child-drift".into(),
5460            source_label: "test".into(),
5461            operation_source: Some(operation_source),
5462            child_session_id: Some(child_session_id.clone()),
5463            expect_peer_channel: true,
5464        };
5465        let operation_id = spec.id.clone();
5466
5467        registry.register_operation(spec).unwrap();
5468        registry.provisioning_succeeded(&operation_id).unwrap();
5469        registry.mark_retired(&operation_id).unwrap();
5470
5471        let cursor_state = meerkat_core::EpochCursorState::new();
5472        let mut snapshot = registry
5473            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5474            .unwrap();
5475        snapshot
5476            .operation_specs
5477            .get_mut(&operation_id)
5478            .expect("persisted spec")
5479            .child_session_id = Some(SessionId::new());
5480
5481        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5482        assert_eq!(
5483            recovered
5484                .snapshot(&operation_id)
5485                .unwrap()
5486                .unwrap()
5487                .child_session_id,
5488            Some(child_session_id)
5489        );
5490    }
5491
5492    #[test]
5493    fn completion_wake_class_is_generated_by_operation_kind() {
5494        let registry = RuntimeOpsLifecycleRegistry::new();
5495        let operation_id = OperationId::new();
5496
5497        assert_eq!(
5498            registry
5499                .classify_operation_completion_wake(&operation_id, OperationKind::BackgroundToolOp)
5500                .unwrap(),
5501            OperationCompletionWakeClass::Wake
5502        );
5503        assert_eq!(
5504            registry
5505                .classify_operation_completion_wake(&operation_id, OperationKind::MobMemberChild)
5506                .unwrap(),
5507            OperationCompletionWakeClass::Ignore
5508        );
5509        assert_eq!(
5510            registry
5511                .classify_operation_completion_wake(
5512                    &operation_id,
5513                    OperationKind::BackgroundToolCapacitySlot,
5514                )
5515                .unwrap(),
5516            OperationCompletionWakeClass::Ignore
5517        );
5518    }
5519
5520    #[test]
5521    fn recovered_snapshot_rejects_completion_feed_without_generated_record() {
5522        let registry = RuntimeOpsLifecycleRegistry::new();
5523
5524        let running_spec = background_spec("running");
5525        let running_id = running_spec.id.clone();
5526        registry.register_operation(running_spec.clone()).unwrap();
5527        registry.provisioning_succeeded(&running_id).unwrap();
5528
5529        let cursor_state = meerkat_core::EpochCursorState::new();
5530        let mut snapshot = registry
5531            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5532            .unwrap();
5533        snapshot.completion_entries.push(CompletionEntry {
5534            seq: 1,
5535            operation_id: running_id.clone(),
5536            kind: running_spec.kind,
5537            display_name: running_spec.display_name,
5538            terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5539                id: running_id,
5540                content: "phantom".into(),
5541                is_error: false,
5542                duration_ms: 1,
5543                tokens_used: 0,
5544            }),
5545            completed_at_ms: None,
5546        });
5547
5548        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5549            Ok(_) => panic!("public completion feed must not recover without generated op truth"),
5550            Err(err) => err,
5551        };
5552        assert!(
5553            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated feed authority")),
5554            "unexpected recovery error: {err:?}"
5555        );
5556    }
5557
5558    #[test]
5559    fn captured_snapshot_rejects_completion_feed_projection_without_generated_authority() {
5560        let registry = RuntimeOpsLifecycleRegistry::new();
5561        let phantom_id = OperationId::new();
5562        {
5563            let state = registry.read_state().unwrap();
5564            state.feed_buffer.push(CompletionEntry {
5565                seq: 1,
5566                operation_id: phantom_id.clone(),
5567                kind: OperationKind::BackgroundToolOp,
5568                display_name: "phantom".into(),
5569                terminal_outcome: OperationTerminalOutcome::Completed(OperationResult {
5570                    id: phantom_id,
5571                    content: "phantom".into(),
5572                    is_error: false,
5573                    duration_ms: 1,
5574                    tokens_used: 0,
5575                }),
5576                completed_at_ms: None,
5577            });
5578        }
5579
5580        let cursor_state = meerkat_core::EpochCursorState::new();
5581        let err = match registry
5582            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5583        {
5584            Ok(_) => panic!("phantom public completion projection must fail snapshot capture"),
5585            Err(err) => err,
5586        };
5587        assert!(
5588            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("no generated authority")),
5589            "unexpected capture error: {err:?}"
5590        );
5591    }
5592
5593    #[test]
5594    fn recovered_snapshot_rejects_feed_authority_beyond_completion_cursor() {
5595        let registry = RuntimeOpsLifecycleRegistry::new();
5596
5597        let spec = background_spec("terminal");
5598        let operation_id = spec.id.clone();
5599        registry.register_operation(spec).unwrap();
5600        registry.provisioning_succeeded(&operation_id).unwrap();
5601        registry
5602            .complete_operation(
5603                &operation_id,
5604                OperationResult {
5605                    id: operation_id.clone(),
5606                    content: "done".into(),
5607                    is_error: false,
5608                    duration_ms: 1,
5609                    tokens_used: 0,
5610                },
5611            )
5612            .unwrap();
5613
5614        let cursor_state = meerkat_core::EpochCursorState::new();
5615        let mut snapshot = registry
5616            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5617            .unwrap();
5618        let phantom_id = OperationId::new();
5619        let phantom_result = OperationResult {
5620            id: phantom_id.clone(),
5621            content: "phantom".into(),
5622            is_error: false,
5623            duration_ms: 1,
5624            tokens_used: 0,
5625        };
5626        let phantom_entry = CompletionFeedCanonicalState {
5627            seq: snapshot.authority_state.next_completion_seq + 1,
5628            kind: OperationKind::BackgroundToolOp,
5629            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result.clone()),
5630        };
5631        snapshot
5632            .authority_state
5633            .completion_feed_entries
5634            .insert(phantom_id.clone(), phantom_entry);
5635        snapshot.completion_entries.push(CompletionEntry {
5636            seq: snapshot.authority_state.next_completion_seq + 1,
5637            operation_id: phantom_id.clone(),
5638            kind: OperationKind::BackgroundToolOp,
5639            display_name: "phantom".into(),
5640            terminal_outcome: OperationTerminalOutcome::Completed(phantom_result),
5641            completed_at_ms: None,
5642        });
5643
5644        let err = match RuntimeOpsLifecycleRegistry::from_recovered(snapshot) {
5645            Ok(_) => panic!("feed authority must not advance the recovered completion cursor"),
5646            Err(err) => err,
5647        };
5648        assert!(
5649            matches!(&err, OpsLifecycleError::Internal(message) if message.contains("RecoverCompletionFeedEntry")),
5650            "unexpected recovery error: {err:?}"
5651        );
5652    }
5653
5654    #[test]
5655    fn recovered_completed_order_uses_generated_completion_sequences() {
5656        let registry = RuntimeOpsLifecycleRegistry::new();
5657
5658        let spec_a = background_spec("a");
5659        let id_a = spec_a.id.clone();
5660        registry.register_operation(spec_a).unwrap();
5661        registry.provisioning_succeeded(&id_a).unwrap();
5662        registry
5663            .complete_operation(
5664                &id_a,
5665                OperationResult {
5666                    id: id_a.clone(),
5667                    content: "a".into(),
5668                    is_error: false,
5669                    duration_ms: 1,
5670                    tokens_used: 0,
5671                },
5672            )
5673            .unwrap();
5674
5675        let spec_b = background_spec("b");
5676        let id_b = spec_b.id.clone();
5677        registry.register_operation(spec_b).unwrap();
5678        registry.provisioning_succeeded(&id_b).unwrap();
5679        registry
5680            .complete_operation(
5681                &id_b,
5682                OperationResult {
5683                    id: id_b.clone(),
5684                    content: "b".into(),
5685                    is_error: false,
5686                    duration_ms: 1,
5687                    tokens_used: 0,
5688                },
5689            )
5690            .unwrap();
5691
5692        let cursor_state = meerkat_core::EpochCursorState::new();
5693        let mut snapshot = registry
5694            .capture_persistence_snapshot(meerkat_core::RuntimeEpochId::new(), &cursor_state)
5695            .unwrap();
5696        snapshot.authority_state.completed_order = VecDeque::from([id_b.clone(), id_a.clone()]);
5697
5698        let recovered = RuntimeOpsLifecycleRegistry::from_recovered(snapshot).unwrap();
5699        let collected = recovered.collect_completed().unwrap();
5700
5701        assert_eq!(collected[0].0, id_a);
5702        assert_eq!(collected[1].0, id_b);
5703    }
5704
5705    #[test]
5706    fn max_concurrent_enforcement() {
5707        let registry = RuntimeOpsLifecycleRegistry::with_config(OpsLifecycleConfig {
5708            max_completed: DEFAULT_MAX_COMPLETED,
5709            max_concurrent: Some(2),
5710        });
5711
5712        let spec_a = background_spec("a");
5713        let id_a = spec_a.id.clone();
5714        registry.register_operation(spec_a).unwrap();
5715
5716        let spec_b = background_spec("b");
5717        registry.register_operation(spec_b).unwrap();
5718
5719        let spec_c = background_spec("c");
5720        let result = registry.register_operation(spec_c);
5721        assert!(matches!(
5722            result,
5723            Err(OpsLifecycleError::MaxConcurrentExceeded {
5724                limit: 2,
5725                active: 2,
5726            })
5727        ));
5728
5729        registry.provisioning_succeeded(&id_a).unwrap();
5730        registry
5731            .complete_operation(
5732                &id_a,
5733                OperationResult {
5734                    id: id_a.clone(),
5735                    content: "done".into(),
5736                    is_error: false,
5737                    duration_ms: 1,
5738                    tokens_used: 0,
5739                },
5740            )
5741            .unwrap();
5742
5743        let spec_d = background_spec("d");
5744        assert!(registry.register_operation(spec_d).is_ok());
5745    }
5746
5747    #[test]
5748    fn snapshot_includes_timestamps() {
5749        let registry = RuntimeOpsLifecycleRegistry::new();
5750        let spec = background_spec("timed");
5751        let op_id = spec.id.clone();
5752        registry.register_operation(spec).unwrap();
5753
5754        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5755        assert!(snap1.created_at_ms > 0);
5756        assert!(snap1.started_at_ms.is_none());
5757        assert!(snap1.completed_at_ms.is_none());
5758        assert!(snap1.elapsed_ms.is_none());
5759
5760        registry.provisioning_succeeded(&op_id).unwrap();
5761        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5762        assert!(snap2.started_at_ms.is_some());
5763        assert!(snap2.started_at_ms.unwrap() >= snap2.created_at_ms);
5764
5765        registry
5766            .complete_operation(
5767                &op_id,
5768                OperationResult {
5769                    id: op_id.clone(),
5770                    content: "done".into(),
5771                    is_error: false,
5772                    duration_ms: 1,
5773                    tokens_used: 0,
5774                },
5775            )
5776            .unwrap();
5777        let snap3 = registry.snapshot(&op_id).unwrap().unwrap();
5778        assert!(snap3.completed_at_ms.is_some());
5779        assert!(snap3.elapsed_ms.is_some());
5780        assert!(snap3.completed_at_ms.unwrap() >= snap3.started_at_ms.unwrap());
5781    }
5782
5783    #[test]
5784    fn snapshot_includes_peer_handle() {
5785        let registry = RuntimeOpsLifecycleRegistry::new();
5786        let child_session_id = SessionId::new();
5787        let spec = OperationSpec {
5788            id: OperationId::new(),
5789            kind: OperationKind::MobMemberChild,
5790            owner_session_id: SessionId::new(),
5791            display_name: "peer-test".into(),
5792            source_label: "test".into(),
5793            operation_source: Some(OperationSource::session_child(child_session_id.clone())),
5794            child_session_id: Some(child_session_id),
5795            expect_peer_channel: true,
5796        };
5797        let op_id = spec.id.clone();
5798        registry.register_operation(spec).unwrap();
5799        registry.provisioning_succeeded(&op_id).unwrap();
5800
5801        let snap1 = registry.snapshot(&op_id).unwrap().unwrap();
5802        assert!(snap1.peer_handle.is_none());
5803
5804        let handle = OperationPeerHandle {
5805            peer_name: meerkat_core::comms::PeerName::new("member-x").unwrap(),
5806            trusted_peer: TrustedPeerDescriptor::test_only_unsigned_typed(
5807                "member-x",
5808                PeerId::new(),
5809                "inproc://x",
5810            )
5811            .unwrap(),
5812        };
5813        registry.peer_ready(&op_id, handle).unwrap();
5814
5815        let snap2 = registry.snapshot(&op_id).unwrap().unwrap();
5816        assert_eq!(
5817            snap2.peer_handle.as_ref().unwrap().peer_name.as_str(),
5818            "member-x"
5819        );
5820    }
5821}