Skip to main content

liminal_protocol/lifecycle/
observer_recovery.rs

1use alloc::collections::{BTreeMap, BTreeSet};
2use alloc::vec::Vec;
3
4use crate::wire::{
5    ConversationId, DeliverySeq, InvalidObserverEpoch, InvalidObserverEpochList, ObserverEpoch,
6    ObserverProgressStatus, ObserverRecoveryAccepted, ObserverRecoveryHandshake,
7    ObserverRecoveryResponse,
8};
9
10/// Move-only exact observer-progress projection emitted by a committed source.
11///
12/// Participant acknowledgement and binding-ending transitions own the progress
13/// value. Consuming code can persist and publish this projection but cannot
14/// construct one from a guessed maximum or record-delivery observation.
15///
16/// ```compile_fail
17/// use liminal_protocol::lifecycle::ObserverProgressProjection;
18///
19/// fn require_clone<T: Clone>() {}
20/// require_clone::<ObserverProgressProjection>();
21/// ```
22#[derive(Debug, PartialEq, Eq)]
23pub struct ObserverProgressProjection {
24    conversation_id: ConversationId,
25    new_observer_progress: DeliverySeq,
26}
27
28impl ObserverProgressProjection {
29    pub(in crate::lifecycle) const fn new(
30        conversation_id: ConversationId,
31        new_observer_progress: DeliverySeq,
32    ) -> Self {
33        Self {
34            conversation_id,
35            new_observer_progress,
36        }
37    }
38
39    /// Returns the conversation whose hard observer progress may advance.
40    #[must_use]
41    pub const fn conversation_id(&self) -> ConversationId {
42        self.conversation_id
43    }
44
45    /// Returns the exact protocol-produced hard observer progress.
46    #[must_use]
47    pub const fn new_observer_progress(&self) -> DeliverySeq {
48        self.new_observer_progress
49    }
50}
51
52/// Restore-time validation failure for the owned observer-recovery aggregate.
53#[derive(Clone, Copy, Debug, PartialEq, Eq)]
54pub enum ObserverRecoveryAggregateRestoreError {
55    /// One conversation appears twice in the durable progress rows.
56    DuplicateProgress {
57        /// Duplicated conversation.
58        conversation_id: ConversationId,
59    },
60    /// One conversation appears twice in the durable arm rows.
61    DuplicateArm {
62        /// Duplicated conversation.
63        conversation_id: ConversationId,
64    },
65    /// A durable arm names a conversation with no durable progress row.
66    ArmWithoutProgress {
67        /// Conversation named by the orphaned arm.
68        conversation_id: ConversationId,
69    },
70    /// A durable arm's epoch differs from its conversation's progress.
71    ///
72    /// An installed arm is always equal-epoch: it is installed at the exact
73    /// read progress and fired (removed) by the same mutation that advances
74    /// progress past it, so any durable disagreement is corruption.
75    ArmEpochMismatch {
76        /// Conversation whose arm disagrees.
77        conversation_id: ConversationId,
78        /// Epoch stored with the arm.
79        armed_epoch: ObserverEpoch,
80        /// Durable hard observer progress.
81        current_observer_progress: DeliverySeq,
82    },
83}
84
85/// Failure while registering a newly tracked conversation.
86#[derive(Clone, Copy, Debug, PartialEq, Eq)]
87pub enum ObserverProgressTrackError {
88    /// The conversation already has an authoritative progress row.
89    AlreadyTracked {
90        /// Conversation presented twice.
91        conversation_id: ConversationId,
92    },
93}
94
95/// Failure while advancing hard observer progress.
96#[derive(Clone, Copy, Debug, PartialEq, Eq)]
97pub enum ObserverProgressAdvanceError {
98    /// The conversation has no authoritative progress row.
99    ConversationUnknown {
100        /// Unknown conversation.
101        conversation_id: ConversationId,
102    },
103    /// The presented progress does not strictly advance the current value.
104    NotAdvancing {
105        /// Conversation whose progress was presented.
106        conversation_id: ConversationId,
107        /// Current hard observer progress.
108        current_observer_progress: DeliverySeq,
109        /// Non-advancing presented progress.
110        presented_progress: DeliverySeq,
111    },
112}
113
114/// Exclusively owned observer-recovery aggregate: per-conversation hard
115/// observer progress plus every installed equal-epoch arm, as ONE owned unit.
116///
117/// This is the A4 transactional surface
118/// (`docs/design/LP-GAP-CLOSURE-GOAL.md`): the equal-epoch progress read and
119/// the arm installation share one serialization boundary *by construction*,
120/// because [`Self::decide_recovery`] consumes the aggregate and only
121/// [`ObserverRecoveryTransaction::commit`] or
122/// [`ObserverRecoveryTransaction::abort`] returns it. No acknowledgement or
123/// binding fate can advance progress between the read and the installation,
124/// and a crash while the transaction is pending installs nothing.
125///
126/// The acknowledgement/binding-fate feed carries the same barrier:
127/// [`Self::decide_progress_advance`] consumes the aggregate and only
128/// [`ObserverProgressAdvanceTransaction::commit`] applies the progress write
129/// and surrenders the fired arm — after the caller's durable append is
130/// confirmed — while [`ObserverProgressAdvanceTransaction::abort`] returns
131/// the aggregate byte-for-byte unchanged, arm still installed. So does the
132/// registration feed: [`Self::decide_track`] consumes the aggregate and only
133/// [`ObserverProgressTrackTransaction::commit`] installs the new progress
134/// row, after the caller's durable append is confirmed, so a recovery batch
135/// can never plan an arm against a progress row that is not yet durable. No
136/// mutation in this module is exempt from the decide/commit/abort
137/// discipline, so live and durable state can never disagree about progress
138/// or an installed arm.
139///
140/// The aggregate is deliberately not `Clone`: at most one owner may read
141/// progress for arm selection.
142///
143/// ```compile_fail
144/// use liminal_protocol::lifecycle::ObserverRecoveryAggregate;
145///
146/// fn require_clone<T: Clone>() {}
147/// require_clone::<ObserverRecoveryAggregate>();
148/// ```
149#[derive(Debug, Default, PartialEq, Eq)]
150pub struct ObserverRecoveryAggregate {
151    progress: BTreeMap<ConversationId, DeliverySeq>,
152    armed: BTreeMap<ConversationId, ObserverEpoch>,
153}
154
155impl ObserverRecoveryAggregate {
156    /// Creates an empty aggregate tracking no conversations.
157    #[must_use]
158    pub const fn new() -> Self {
159        Self {
160            progress: BTreeMap::new(),
161            armed: BTreeMap::new(),
162        }
163    }
164
165    /// Rebuilds the aggregate from durable progress and arm rows.
166    ///
167    /// # Errors
168    ///
169    /// Returns [`ObserverRecoveryAggregateRestoreError`] for duplicate rows,
170    /// an arm without its progress row, or an arm whose epoch is not the
171    /// exact current progress of its conversation.
172    pub fn restore(
173        progress_rows: &[(ConversationId, DeliverySeq)],
174        armed_rows: &[(ConversationId, ObserverEpoch)],
175    ) -> Result<Self, ObserverRecoveryAggregateRestoreError> {
176        let mut progress = BTreeMap::new();
177        for (conversation_id, observer_progress) in progress_rows {
178            if progress
179                .insert(*conversation_id, *observer_progress)
180                .is_some()
181            {
182                return Err(ObserverRecoveryAggregateRestoreError::DuplicateProgress {
183                    conversation_id: *conversation_id,
184                });
185            }
186        }
187        let mut armed = BTreeMap::new();
188        for (conversation_id, armed_epoch) in armed_rows {
189            let Some(current_observer_progress) = progress.get(conversation_id).copied() else {
190                return Err(ObserverRecoveryAggregateRestoreError::ArmWithoutProgress {
191                    conversation_id: *conversation_id,
192                });
193            };
194            if current_observer_progress != *armed_epoch {
195                return Err(ObserverRecoveryAggregateRestoreError::ArmEpochMismatch {
196                    conversation_id: *conversation_id,
197                    armed_epoch: *armed_epoch,
198                    current_observer_progress,
199                });
200            }
201            if armed.insert(*conversation_id, *armed_epoch).is_some() {
202                return Err(ObserverRecoveryAggregateRestoreError::DuplicateArm {
203                    conversation_id: *conversation_id,
204                });
205            }
206        }
207        Ok(Self { progress, armed })
208    }
209
210    /// Returns the current hard observer progress for one conversation.
211    #[must_use]
212    pub fn observer_progress(&self, conversation_id: ConversationId) -> Option<DeliverySeq> {
213        self.progress.get(&conversation_id).copied()
214    }
215
216    /// Returns the installed equal-epoch arm for one conversation, if any.
217    #[must_use]
218    pub fn armed_epoch(&self, conversation_id: ConversationId) -> Option<ObserverEpoch> {
219        self.armed.get(&conversation_id).copied()
220    }
221
222    /// Returns every durable progress row in conversation order.
223    #[must_use]
224    pub fn progress_rows(&self) -> Vec<(ConversationId, DeliverySeq)> {
225        self.progress
226            .iter()
227            .map(|(conversation_id, observer_progress)| (*conversation_id, *observer_progress))
228            .collect()
229    }
230
231    /// Returns every installed arm row in conversation order.
232    #[must_use]
233    pub fn armed_rows(&self) -> Vec<(ConversationId, ObserverEpoch)> {
234        self.armed
235            .iter()
236            .map(|(conversation_id, armed_epoch)| (*conversation_id, *armed_epoch))
237            .collect()
238    }
239
240    /// Consumes the aggregate into one registration-track transaction.
241    ///
242    /// This is the registration feed for newly tracked conversations, and it
243    /// carries the same barrier as the other two mutations. Validation
244    /// happens here — an already tracked conversation refuses with the
245    /// aggregate unchanged — and a validated registration returns a pending
246    /// [`ObserverProgressTrackTransaction`] carrying the new progress row for
247    /// the caller's durable append. Nothing mutates until
248    /// [`ObserverProgressTrackTransaction::commit`] confirms the append;
249    /// [`ObserverProgressTrackTransaction::abort`] returns the aggregate
250    /// byte-for-byte unchanged, conversation still untracked. A recovery
251    /// batch therefore can never read a progress row whose durable append
252    /// has not been confirmed, so a crash after an arm install can never
253    /// strand a durable arm row without its progress row
254    /// ([`ObserverRecoveryAggregateRestoreError::ArmWithoutProgress`]).
255    #[must_use]
256    pub fn decide_track(
257        self,
258        conversation_id: ConversationId,
259        observer_progress: DeliverySeq,
260    ) -> ObserverProgressTrackDecision {
261        if self.progress.contains_key(&conversation_id) {
262            return ObserverProgressTrackDecision::Refuse {
263                aggregate: self,
264                error: ObserverProgressTrackError::AlreadyTracked { conversation_id },
265            };
266        }
267        ObserverProgressTrackDecision::Commit(ObserverProgressTrackTransaction {
268            aggregate: self,
269            conversation_id,
270            observer_progress,
271        })
272    }
273
274    /// Consumes the aggregate into one progress-advance transaction.
275    ///
276    /// This is the acknowledgement/binding-fate feed. Validation happens
277    /// here — unknown conversation and non-advancing progress refuse with
278    /// the aggregate unchanged — and a validated advance returns a pending
279    /// [`ObserverProgressAdvanceTransaction`] carrying the new progress row
280    /// and the fired-arm plan for the caller's durable append. Nothing
281    /// mutates until [`ObserverProgressAdvanceTransaction::commit`] confirms
282    /// the append; [`ObserverProgressAdvanceTransaction::abort`] returns the
283    /// aggregate byte-for-byte unchanged. Every installed arm is equal-epoch
284    /// with its conversation's progress, so a strictly advancing write always
285    /// fires the installed arm: the fired arm is surrendered by `commit` so
286    /// the caller wakes the parked rows in the same durable transaction
287    /// (LAW-1: wake on the event, never poll).
288    #[must_use]
289    pub fn decide_progress_advance(
290        self,
291        conversation_id: ConversationId,
292        presented_progress: DeliverySeq,
293    ) -> ObserverProgressAdvanceDecision {
294        let Some(current) = self.progress.get(&conversation_id).copied() else {
295            return ObserverProgressAdvanceDecision::Refuse {
296                aggregate: self,
297                error: ObserverProgressAdvanceError::ConversationUnknown { conversation_id },
298            };
299        };
300        if presented_progress <= current {
301            return ObserverProgressAdvanceDecision::Refuse {
302                aggregate: self,
303                error: ObserverProgressAdvanceError::NotAdvancing {
304                    conversation_id,
305                    current_observer_progress: current,
306                    presented_progress,
307                },
308            };
309        }
310        let fired = self
311            .armed
312            .get(&conversation_id)
313            .copied()
314            .map(|refused_epoch| ObserverRecoveryArm {
315                conversation_id,
316                refused_epoch,
317            });
318        ObserverProgressAdvanceDecision::Commit(ObserverProgressAdvanceTransaction {
319            aggregate: self,
320            conversation_id,
321            presented_progress,
322            fired,
323        })
324    }
325
326    /// Consumes the aggregate into one observer-recovery transaction.
327    ///
328    /// This is the only public door to arm selection: it binds the
329    /// crate-internal selection's progress reads to the owned aggregate so
330    /// the read and the arm installation are one owned unit by construction.
331    /// A refused batch returns the aggregate unchanged alongside the exact
332    /// refusal response.
333    #[must_use]
334    pub fn decide_recovery(
335        self,
336        request: &ObserverRecoveryHandshake,
337        max_entries: u64,
338        connection_conversation_limit: u64,
339        tracked_conversations: &[ConversationId],
340    ) -> ObserverRecoveryTransactionDecision {
341        let decision = apply_observer_recovery(
342            request,
343            max_entries,
344            connection_conversation_limit,
345            tracked_conversations,
346            |conversation_id| self.progress.get(&conversation_id).copied(),
347        );
348        match decision {
349            ObserverRecoveryDecision::Respond(response) => {
350                ObserverRecoveryTransactionDecision::Respond {
351                    aggregate: self,
352                    response,
353                }
354            }
355            ObserverRecoveryDecision::Commit(commit) => {
356                ObserverRecoveryTransactionDecision::Commit(ObserverRecoveryTransaction {
357                    aggregate: self,
358                    commit,
359                })
360            }
361        }
362    }
363}
364
365/// Total transactional observer-recovery decision against the owned aggregate.
366#[derive(Debug, PartialEq, Eq)]
367pub enum ObserverRecoveryTransactionDecision {
368    /// The whole batch was refused; the aggregate is unchanged.
369    Respond {
370        /// Unchanged aggregate returned to its owner.
371        aggregate: ObserverRecoveryAggregate,
372        /// Exact refusal response.
373        response: ObserverRecoveryResponse,
374    },
375    /// Every entry validated; the arm plan may commit atomically.
376    Commit(ObserverRecoveryTransaction),
377}
378
379/// Ownership barrier between arm selection and atomic arm installation.
380///
381/// The aggregate is unreachable while the transaction is pending, so no
382/// progress can advance between the equal-epoch read and the installation.
383/// Consuming [`Self::commit`] installs every arm of the plan at once — a
384/// subset is unrepresentable — and consuming [`Self::abort`] installs none,
385/// so a crash at any point leaves either the complete arm plan or no arm at
386/// all, never a partially-armed request.
387#[derive(Debug, PartialEq, Eq)]
388pub struct ObserverRecoveryTransaction {
389    aggregate: ObserverRecoveryAggregate,
390    commit: ObserverRecoveryCommit,
391}
392
393impl ObserverRecoveryTransaction {
394    /// Borrows the complete equal-epoch arm plan for the durable append.
395    #[must_use]
396    pub fn arms(&self) -> &[ObserverRecoveryArm] {
397        self.commit.arms()
398    }
399
400    /// Borrows the exact request-ordered success response.
401    #[must_use]
402    pub const fn outcome(&self) -> &ObserverRecoveryAccepted {
403        self.commit.outcome()
404    }
405
406    /// Installs the whole arm plan after a confirmed durable append.
407    ///
408    /// Installation is idempotent: replaying a durable recovery against the
409    /// post-state reinstalls the identical equal-epoch arms, so at most one
410    /// arm per conversation ever exists and crash replay converges.
411    #[must_use]
412    pub fn commit(mut self) -> (ObserverRecoveryAggregate, ObserverRecoveryAccepted) {
413        let (arms, outcome) = self.commit.into_parts();
414        for arm in arms {
415            self.aggregate
416                .armed
417                .insert(arm.conversation_id(), arm.refused_epoch());
418        }
419        (self.aggregate, outcome)
420    }
421
422    /// Cancels a failed durable append; not one arm is installed.
423    #[must_use]
424    pub fn abort(self) -> ObserverRecoveryAggregate {
425        self.aggregate
426    }
427}
428
429/// Total transactional progress-advance decision against the owned aggregate.
430#[derive(Debug, PartialEq, Eq)]
431pub enum ObserverProgressAdvanceDecision {
432    /// The advance was refused; the aggregate is unchanged.
433    Refuse {
434        /// Unchanged aggregate returned to its owner.
435        aggregate: ObserverRecoveryAggregate,
436        /// Exact refusal.
437        error: ObserverProgressAdvanceError,
438    },
439    /// The advance validated; the progress write and arm fire may commit
440    /// atomically.
441    Commit(ObserverProgressAdvanceTransaction),
442}
443
444/// Ownership barrier between a validated progress advance and its durable
445/// append.
446///
447/// The aggregate is unreachable while the transaction is pending, so no
448/// recovery batch can read the not-yet-durable progress and no other advance
449/// can race the fired-arm plan. Consuming [`Self::commit`] applies the
450/// progress write and removes the fired arm as ONE mutation — after the
451/// caller has confirmed the durable append of the progress row, the arm-row
452/// deletion, and the wake — and consuming [`Self::abort`] returns the
453/// aggregate byte-for-byte unchanged, arm still installed, so a failed
454/// append never leaves live progress ahead of durable state or a
455/// phantom-fired arm.
456#[derive(Debug, PartialEq, Eq)]
457pub struct ObserverProgressAdvanceTransaction {
458    aggregate: ObserverRecoveryAggregate,
459    conversation_id: ConversationId,
460    presented_progress: DeliverySeq,
461    fired: Option<ObserverRecoveryArm>,
462}
463
464impl ObserverProgressAdvanceTransaction {
465    /// Returns the conversation whose progress row the durable append writes.
466    #[must_use]
467    pub const fn conversation_id(&self) -> ConversationId {
468        self.conversation_id
469    }
470
471    /// Returns the new hard observer progress for the durable append.
472    #[must_use]
473    pub const fn presented_progress(&self) -> DeliverySeq {
474        self.presented_progress
475    }
476
477    /// Returns the fired-arm plan for the durable append.
478    ///
479    /// `Some` names the installed arm whose durable row the same append must
480    /// delete and whose parked rows the same append must wake; `None` means
481    /// the conversation carried no arm.
482    #[must_use]
483    pub const fn fired_arm(&self) -> Option<ObserverRecoveryArm> {
484        self.fired
485    }
486
487    /// Applies the advance after a confirmed durable append.
488    ///
489    /// The progress write and the arm removal are one mutation: the returned
490    /// arm is exactly [`Self::fired_arm`], surrendered so the caller wakes
491    /// the parked rows it durably committed to waking.
492    #[must_use]
493    pub fn commit(mut self) -> (ObserverRecoveryAggregate, Option<ObserverRecoveryArm>) {
494        self.aggregate
495            .progress
496            .insert(self.conversation_id, self.presented_progress);
497        if self.fired.is_some() {
498            self.aggregate.armed.remove(&self.conversation_id);
499        }
500        (self.aggregate, self.fired)
501    }
502
503    /// Cancels a failed durable append; neither progress nor arm changes.
504    #[must_use]
505    pub fn abort(self) -> ObserverRecoveryAggregate {
506        self.aggregate
507    }
508}
509
510/// Total transactional registration decision against the owned aggregate.
511#[derive(Debug, PartialEq, Eq)]
512pub enum ObserverProgressTrackDecision {
513    /// The registration was refused; the aggregate is unchanged.
514    Refuse {
515        /// Unchanged aggregate returned to its owner.
516        aggregate: ObserverRecoveryAggregate,
517        /// Exact refusal.
518        error: ObserverProgressTrackError,
519    },
520    /// The registration validated; the progress row may commit atomically.
521    Commit(ObserverProgressTrackTransaction),
522}
523
524/// Ownership barrier between a validated registration and its durable append.
525///
526/// The aggregate is unreachable while the transaction is pending, so no
527/// recovery batch can plan an equal-epoch arm against a progress row whose
528/// durable append is not yet confirmed. Consuming [`Self::commit`] installs
529/// the new progress row — after the caller has confirmed the durable append —
530/// and consuming [`Self::abort`] returns the aggregate byte-for-byte
531/// unchanged, conversation still untracked, so a failed append never leaves
532/// a live progress row that durable state does not hold.
533#[derive(Debug, PartialEq, Eq)]
534pub struct ObserverProgressTrackTransaction {
535    aggregate: ObserverRecoveryAggregate,
536    conversation_id: ConversationId,
537    observer_progress: DeliverySeq,
538}
539
540impl ObserverProgressTrackTransaction {
541    /// Returns the conversation whose progress row the durable append writes.
542    #[must_use]
543    pub const fn conversation_id(&self) -> ConversationId {
544        self.conversation_id
545    }
546
547    /// Returns the registered hard observer progress for the durable append.
548    #[must_use]
549    pub const fn observer_progress(&self) -> DeliverySeq {
550        self.observer_progress
551    }
552
553    /// Installs the registered progress row after a confirmed durable append.
554    #[must_use]
555    pub fn commit(mut self) -> ObserverRecoveryAggregate {
556        self.aggregate
557            .progress
558            .insert(self.conversation_id, self.observer_progress);
559        self.aggregate
560    }
561
562    /// Cancels a failed durable append; the conversation stays untracked.
563    #[must_use]
564    pub fn abort(self) -> ObserverRecoveryAggregate {
565        self.aggregate
566    }
567}
568
569/// One equal observer epoch that must be armed by an accepted recovery batch.
570#[derive(Clone, Copy, Debug, PartialEq, Eq)]
571pub struct ObserverRecoveryArm {
572    conversation_id: ConversationId,
573    refused_epoch: ObserverEpoch,
574}
575
576impl ObserverRecoveryArm {
577    /// Returns the conversation whose progress event must wake the parked rows.
578    #[must_use]
579    pub const fn conversation_id(self) -> ConversationId {
580        self.conversation_id
581    }
582
583    /// Returns the exact refusal epoch being armed.
584    #[must_use]
585    pub const fn refused_epoch(self) -> ObserverEpoch {
586        self.refused_epoch
587    }
588}
589
590/// Whole-batch observer-recovery commit selected after exhaustive validation.
591///
592/// The arm list and response are produced together. A consumer must persist all
593/// arms atomically before sending [`Self::outcome`], so no validation failure or
594/// crash can expose a partially armed request.
595#[derive(Clone, Debug, PartialEq, Eq)]
596pub struct ObserverRecoveryCommit {
597    arms: Vec<ObserverRecoveryArm>,
598    outcome: ObserverRecoveryAccepted,
599}
600
601impl ObserverRecoveryCommit {
602    /// Borrows the complete equal-epoch arm plan in request order.
603    #[must_use]
604    pub fn arms(&self) -> &[ObserverRecoveryArm] {
605        &self.arms
606    }
607
608    /// Borrows the exact request-ordered success response.
609    #[must_use]
610    pub const fn outcome(&self) -> &ObserverRecoveryAccepted {
611        &self.outcome
612    }
613
614    /// Consumes the commit into the atomic arm plan and success response.
615    #[must_use]
616    pub fn into_parts(self) -> (Vec<ObserverRecoveryArm>, ObserverRecoveryAccepted) {
617        (self.arms, self.outcome)
618    }
619}
620
621/// Total observer-recovery decision.
622#[derive(Clone, Debug, PartialEq, Eq)]
623pub enum ObserverRecoveryDecision {
624    /// Validation or connection-capacity preflight refused the whole batch.
625    Respond(ObserverRecoveryResponse),
626    /// Every entry validated and the complete arm plan may commit atomically.
627    Commit(ObserverRecoveryCommit),
628}
629
630fn wire_count(value: usize) -> u64 {
631    u64::try_from(value).map_or(u64::MAX, core::convert::identity)
632}
633
634/// Applies the observer-recovery list, capacity, and epoch precedence.
635///
636/// Validation order is the frozen R-D1 order: list length, duplicate
637/// conversation, request-index connection capacity, then request-index unknown
638/// or ahead epoch. Only after every entry passes is an arm plan produced.
639/// `observer_progress` must return the current hard observer progress for a
640/// known conversation and `None` for an unknown conversation.
641///
642/// This raw selection is deliberately crate-internal: its progress reads and
643/// the installation of every returned arm must share one serialization
644/// boundary (releasing it between evaluation and installation would violate
645/// the equal-epoch subscribe-then-snapshot rule), and that requirement is
646/// enforceable only by ownership.
647/// [`ObserverRecoveryAggregate::decide_recovery`] is the one public door to
648/// arm selection: it binds these progress reads to the owned aggregate, so
649/// no caller can interleave an acknowledgement or fate advance between
650/// evaluation and installation.
651#[must_use]
652pub(super) fn apply_observer_recovery<F>(
653    request: &ObserverRecoveryHandshake,
654    max_entries: u64,
655    connection_conversation_limit: u64,
656    tracked_conversations: &[ConversationId],
657    mut observer_progress: F,
658) -> ObserverRecoveryDecision
659where
660    F: FnMut(ConversationId) -> Option<DeliverySeq>,
661{
662    let presented_entries = wire_count(request.observer_refusals.len());
663    if presented_entries > max_entries {
664        return ObserverRecoveryDecision::Respond(
665            ObserverRecoveryResponse::invalid_observer_epoch_list(
666                InvalidObserverEpochList::TooManyEntries {
667                    presented_entries,
668                    max_entries,
669                },
670            ),
671        );
672    }
673
674    let mut first_indices = BTreeMap::new();
675    for (index, refusal) in request.observer_refusals.iter().enumerate() {
676        let request_index = wire_count(index);
677        if let Some(first_index) = first_indices.insert(refusal.conversation_id, request_index) {
678            return ObserverRecoveryDecision::Respond(
679                ObserverRecoveryResponse::invalid_observer_epoch_list(
680                    InvalidObserverEpochList::DuplicateConversation {
681                        conversation_id: refusal.conversation_id,
682                        first_index,
683                        duplicate_index: request_index,
684                    },
685                ),
686            );
687        }
688    }
689
690    let mut tracked: BTreeSet<_> = tracked_conversations.iter().copied().collect();
691    for refusal in &request.observer_refusals {
692        if tracked.contains(&refusal.conversation_id) {
693            continue;
694        }
695        let occupied = wire_count(tracked.len());
696        if occupied >= connection_conversation_limit {
697            return ObserverRecoveryDecision::Respond(
698                ObserverRecoveryResponse::connection_capacity_exceeded(
699                    refusal.conversation_id,
700                    connection_conversation_limit,
701                ),
702            );
703        }
704        tracked.insert(refusal.conversation_id);
705    }
706
707    let mut validated = Vec::with_capacity(request.observer_refusals.len());
708    for refusal in &request.observer_refusals {
709        let Some(current_observer_progress) = observer_progress(refusal.conversation_id) else {
710            return ObserverRecoveryDecision::Respond(
711                ObserverRecoveryResponse::invalid_observer_epoch(
712                    InvalidObserverEpoch::ConversationUnknown {
713                        conversation_id: refusal.conversation_id,
714                        presented_epoch: refusal.refused_epoch,
715                    },
716                ),
717            );
718        };
719        if refusal.refused_epoch > current_observer_progress {
720            return ObserverRecoveryDecision::Respond(
721                ObserverRecoveryResponse::invalid_observer_epoch(
722                    InvalidObserverEpoch::EpochAhead {
723                        conversation_id: refusal.conversation_id,
724                        presented_epoch: refusal.refused_epoch,
725                        current_observer_progress,
726                    },
727                ),
728            );
729        }
730        validated.push((refusal, current_observer_progress));
731    }
732
733    let mut arms = Vec::new();
734    let mut statuses = Vec::with_capacity(validated.len());
735    for (refusal, current_observer_progress) in validated {
736        let armed = refusal.refused_epoch == current_observer_progress;
737        if armed {
738            arms.push(ObserverRecoveryArm {
739                conversation_id: refusal.conversation_id,
740                refused_epoch: refusal.refused_epoch,
741            });
742        }
743        statuses.push(ObserverProgressStatus {
744            conversation_id: refusal.conversation_id,
745            refused_epoch: refusal.refused_epoch,
746            current_observer_progress,
747            armed,
748            progressed: !armed,
749        });
750    }
751
752    ObserverRecoveryDecision::Commit(ObserverRecoveryCommit {
753        arms,
754        outcome: ObserverRecoveryAccepted { statuses },
755    })
756}