liminal_protocol/lifecycle/observer_recovery.rs
1use alloc::collections::{BTreeMap, BTreeSet};
2use alloc::vec::Vec;
3
4use crate::wire::{
5 ConversationId, DeliverySeq, InvalidObserverEpoch, InvalidObserverEpochList, ObserverEpoch,
6 ObserverProgressStatus, ObserverRecoveryAccepted, ObserverRecoveryHandshake,
7 ObserverRecoveryResponse,
8};
9
10/// Move-only exact observer-progress projection emitted by a committed source.
11///
12/// Participant acknowledgement and binding-ending transitions own the progress
13/// value. Consuming code can persist and publish this projection but cannot
14/// construct one from a guessed maximum or record-delivery observation.
15///
16/// ```compile_fail
17/// use liminal_protocol::lifecycle::ObserverProgressProjection;
18///
19/// fn require_clone<T: Clone>() {}
20/// require_clone::<ObserverProgressProjection>();
21/// ```
22#[derive(Debug, PartialEq, Eq)]
23pub struct ObserverProgressProjection {
24 conversation_id: ConversationId,
25 new_observer_progress: DeliverySeq,
26}
27
28impl ObserverProgressProjection {
29 pub(in crate::lifecycle) const fn new(
30 conversation_id: ConversationId,
31 new_observer_progress: DeliverySeq,
32 ) -> Self {
33 Self {
34 conversation_id,
35 new_observer_progress,
36 }
37 }
38
39 /// Returns the conversation whose hard observer progress may advance.
40 #[must_use]
41 pub const fn conversation_id(&self) -> ConversationId {
42 self.conversation_id
43 }
44
45 /// Returns the exact protocol-produced hard observer progress.
46 #[must_use]
47 pub const fn new_observer_progress(&self) -> DeliverySeq {
48 self.new_observer_progress
49 }
50}
51
52/// Restore-time validation failure for the owned observer-recovery aggregate.
53#[derive(Clone, Copy, Debug, PartialEq, Eq)]
54pub enum ObserverRecoveryAggregateRestoreError {
55 /// One conversation appears twice in the durable progress rows.
56 DuplicateProgress {
57 /// Duplicated conversation.
58 conversation_id: ConversationId,
59 },
60 /// One conversation appears twice in the durable arm rows.
61 DuplicateArm {
62 /// Duplicated conversation.
63 conversation_id: ConversationId,
64 },
65 /// A durable arm names a conversation with no durable progress row.
66 ArmWithoutProgress {
67 /// Conversation named by the orphaned arm.
68 conversation_id: ConversationId,
69 },
70 /// A durable arm's epoch differs from its conversation's progress.
71 ///
72 /// An installed arm is always equal-epoch: it is installed at the exact
73 /// read progress and fired (removed) by the same mutation that advances
74 /// progress past it, so any durable disagreement is corruption.
75 ArmEpochMismatch {
76 /// Conversation whose arm disagrees.
77 conversation_id: ConversationId,
78 /// Epoch stored with the arm.
79 armed_epoch: ObserverEpoch,
80 /// Durable hard observer progress.
81 current_observer_progress: DeliverySeq,
82 },
83}
84
85/// Failure while registering a newly tracked conversation.
86#[derive(Clone, Copy, Debug, PartialEq, Eq)]
87pub enum ObserverProgressTrackError {
88 /// The conversation already has an authoritative progress row.
89 AlreadyTracked {
90 /// Conversation presented twice.
91 conversation_id: ConversationId,
92 },
93}
94
95/// Failure while advancing hard observer progress.
96#[derive(Clone, Copy, Debug, PartialEq, Eq)]
97pub enum ObserverProgressAdvanceError {
98 /// The conversation has no authoritative progress row.
99 ConversationUnknown {
100 /// Unknown conversation.
101 conversation_id: ConversationId,
102 },
103 /// The presented progress does not strictly advance the current value.
104 NotAdvancing {
105 /// Conversation whose progress was presented.
106 conversation_id: ConversationId,
107 /// Current hard observer progress.
108 current_observer_progress: DeliverySeq,
109 /// Non-advancing presented progress.
110 presented_progress: DeliverySeq,
111 },
112}
113
114/// Exclusively owned observer-recovery aggregate: per-conversation hard
115/// observer progress plus every installed equal-epoch arm, as ONE owned unit.
116///
117/// This is the A4 transactional surface
118/// (`docs/design/LP-GAP-CLOSURE-GOAL.md`): the equal-epoch progress read and
119/// the arm installation share one serialization boundary *by construction*,
120/// because [`Self::decide_recovery`] consumes the aggregate and only
121/// [`ObserverRecoveryTransaction::commit`] or
122/// [`ObserverRecoveryTransaction::abort`] returns it. No acknowledgement or
123/// binding fate can advance progress between the read and the installation,
124/// and a crash while the transaction is pending installs nothing.
125///
126/// The acknowledgement/binding-fate feed carries the same barrier:
127/// [`Self::decide_progress_advance`] consumes the aggregate and only
128/// [`ObserverProgressAdvanceTransaction::commit`] applies the progress write
129/// and surrenders the fired arm — after the caller's durable append is
130/// confirmed — while [`ObserverProgressAdvanceTransaction::abort`] returns
131/// the aggregate byte-for-byte unchanged, arm still installed. So does the
132/// registration feed: [`Self::decide_track`] consumes the aggregate and only
133/// [`ObserverProgressTrackTransaction::commit`] installs the new progress
134/// row, after the caller's durable append is confirmed, so a recovery batch
135/// can never plan an arm against a progress row that is not yet durable. No
136/// mutation in this module is exempt from the decide/commit/abort
137/// discipline, so live and durable state can never disagree about progress
138/// or an installed arm.
139///
140/// The aggregate is deliberately not `Clone`: at most one owner may read
141/// progress for arm selection.
142///
143/// ```compile_fail
144/// use liminal_protocol::lifecycle::ObserverRecoveryAggregate;
145///
146/// fn require_clone<T: Clone>() {}
147/// require_clone::<ObserverRecoveryAggregate>();
148/// ```
149#[derive(Debug, Default, PartialEq, Eq)]
150pub struct ObserverRecoveryAggregate {
151 progress: BTreeMap<ConversationId, DeliverySeq>,
152 armed: BTreeMap<ConversationId, ObserverEpoch>,
153}
154
155impl ObserverRecoveryAggregate {
156 /// Creates an empty aggregate tracking no conversations.
157 #[must_use]
158 pub const fn new() -> Self {
159 Self {
160 progress: BTreeMap::new(),
161 armed: BTreeMap::new(),
162 }
163 }
164
165 /// Rebuilds the aggregate from durable progress and arm rows.
166 ///
167 /// # Errors
168 ///
169 /// Returns [`ObserverRecoveryAggregateRestoreError`] for duplicate rows,
170 /// an arm without its progress row, or an arm whose epoch is not the
171 /// exact current progress of its conversation.
172 pub fn restore(
173 progress_rows: &[(ConversationId, DeliverySeq)],
174 armed_rows: &[(ConversationId, ObserverEpoch)],
175 ) -> Result<Self, ObserverRecoveryAggregateRestoreError> {
176 let mut progress = BTreeMap::new();
177 for (conversation_id, observer_progress) in progress_rows {
178 if progress
179 .insert(*conversation_id, *observer_progress)
180 .is_some()
181 {
182 return Err(ObserverRecoveryAggregateRestoreError::DuplicateProgress {
183 conversation_id: *conversation_id,
184 });
185 }
186 }
187 let mut armed = BTreeMap::new();
188 for (conversation_id, armed_epoch) in armed_rows {
189 let Some(current_observer_progress) = progress.get(conversation_id).copied() else {
190 return Err(ObserverRecoveryAggregateRestoreError::ArmWithoutProgress {
191 conversation_id: *conversation_id,
192 });
193 };
194 if current_observer_progress != *armed_epoch {
195 return Err(ObserverRecoveryAggregateRestoreError::ArmEpochMismatch {
196 conversation_id: *conversation_id,
197 armed_epoch: *armed_epoch,
198 current_observer_progress,
199 });
200 }
201 if armed.insert(*conversation_id, *armed_epoch).is_some() {
202 return Err(ObserverRecoveryAggregateRestoreError::DuplicateArm {
203 conversation_id: *conversation_id,
204 });
205 }
206 }
207 Ok(Self { progress, armed })
208 }
209
210 /// Returns the current hard observer progress for one conversation.
211 #[must_use]
212 pub fn observer_progress(&self, conversation_id: ConversationId) -> Option<DeliverySeq> {
213 self.progress.get(&conversation_id).copied()
214 }
215
216 /// Returns the installed equal-epoch arm for one conversation, if any.
217 #[must_use]
218 pub fn armed_epoch(&self, conversation_id: ConversationId) -> Option<ObserverEpoch> {
219 self.armed.get(&conversation_id).copied()
220 }
221
222 /// Returns every durable progress row in conversation order.
223 #[must_use]
224 pub fn progress_rows(&self) -> Vec<(ConversationId, DeliverySeq)> {
225 self.progress
226 .iter()
227 .map(|(conversation_id, observer_progress)| (*conversation_id, *observer_progress))
228 .collect()
229 }
230
231 /// Returns every installed arm row in conversation order.
232 #[must_use]
233 pub fn armed_rows(&self) -> Vec<(ConversationId, ObserverEpoch)> {
234 self.armed
235 .iter()
236 .map(|(conversation_id, armed_epoch)| (*conversation_id, *armed_epoch))
237 .collect()
238 }
239
240 /// Consumes the aggregate into one registration-track transaction.
241 ///
242 /// This is the registration feed for newly tracked conversations, and it
243 /// carries the same barrier as the other two mutations. Validation
244 /// happens here — an already tracked conversation refuses with the
245 /// aggregate unchanged — and a validated registration returns a pending
246 /// [`ObserverProgressTrackTransaction`] carrying the new progress row for
247 /// the caller's durable append. Nothing mutates until
248 /// [`ObserverProgressTrackTransaction::commit`] confirms the append;
249 /// [`ObserverProgressTrackTransaction::abort`] returns the aggregate
250 /// byte-for-byte unchanged, conversation still untracked. A recovery
251 /// batch therefore can never read a progress row whose durable append
252 /// has not been confirmed, so a crash after an arm install can never
253 /// strand a durable arm row without its progress row
254 /// ([`ObserverRecoveryAggregateRestoreError::ArmWithoutProgress`]).
255 #[must_use]
256 pub fn decide_track(
257 self,
258 conversation_id: ConversationId,
259 observer_progress: DeliverySeq,
260 ) -> ObserverProgressTrackDecision {
261 if self.progress.contains_key(&conversation_id) {
262 return ObserverProgressTrackDecision::Refuse {
263 aggregate: self,
264 error: ObserverProgressTrackError::AlreadyTracked { conversation_id },
265 };
266 }
267 ObserverProgressTrackDecision::Commit(ObserverProgressTrackTransaction {
268 aggregate: self,
269 conversation_id,
270 observer_progress,
271 })
272 }
273
274 /// Consumes the aggregate into one progress-advance transaction.
275 ///
276 /// This is the acknowledgement/binding-fate feed. Validation happens
277 /// here — unknown conversation and non-advancing progress refuse with
278 /// the aggregate unchanged — and a validated advance returns a pending
279 /// [`ObserverProgressAdvanceTransaction`] carrying the new progress row
280 /// and the fired-arm plan for the caller's durable append. Nothing
281 /// mutates until [`ObserverProgressAdvanceTransaction::commit`] confirms
282 /// the append; [`ObserverProgressAdvanceTransaction::abort`] returns the
283 /// aggregate byte-for-byte unchanged. Every installed arm is equal-epoch
284 /// with its conversation's progress, so a strictly advancing write always
285 /// fires the installed arm: the fired arm is surrendered by `commit` so
286 /// the caller wakes the parked rows in the same durable transaction
287 /// (LAW-1: wake on the event, never poll).
288 #[must_use]
289 pub fn decide_progress_advance(
290 self,
291 conversation_id: ConversationId,
292 presented_progress: DeliverySeq,
293 ) -> ObserverProgressAdvanceDecision {
294 let Some(current) = self.progress.get(&conversation_id).copied() else {
295 return ObserverProgressAdvanceDecision::Refuse {
296 aggregate: self,
297 error: ObserverProgressAdvanceError::ConversationUnknown { conversation_id },
298 };
299 };
300 if presented_progress <= current {
301 return ObserverProgressAdvanceDecision::Refuse {
302 aggregate: self,
303 error: ObserverProgressAdvanceError::NotAdvancing {
304 conversation_id,
305 current_observer_progress: current,
306 presented_progress,
307 },
308 };
309 }
310 let fired = self
311 .armed
312 .get(&conversation_id)
313 .copied()
314 .map(|refused_epoch| ObserverRecoveryArm {
315 conversation_id,
316 refused_epoch,
317 });
318 ObserverProgressAdvanceDecision::Commit(ObserverProgressAdvanceTransaction {
319 aggregate: self,
320 conversation_id,
321 presented_progress,
322 fired,
323 })
324 }
325
326 /// Consumes the aggregate into one observer-recovery transaction.
327 ///
328 /// This is the only public door to arm selection: it binds the
329 /// crate-internal selection's progress reads to the owned aggregate so
330 /// the read and the arm installation are one owned unit by construction.
331 /// A refused batch returns the aggregate unchanged alongside the exact
332 /// refusal response.
333 #[must_use]
334 pub fn decide_recovery(
335 self,
336 request: &ObserverRecoveryHandshake,
337 max_entries: u64,
338 connection_conversation_limit: u64,
339 tracked_conversations: &[ConversationId],
340 ) -> ObserverRecoveryTransactionDecision {
341 let decision = apply_observer_recovery(
342 request,
343 max_entries,
344 connection_conversation_limit,
345 tracked_conversations,
346 |conversation_id| self.progress.get(&conversation_id).copied(),
347 );
348 match decision {
349 ObserverRecoveryDecision::Respond(response) => {
350 ObserverRecoveryTransactionDecision::Respond {
351 aggregate: self,
352 response,
353 }
354 }
355 ObserverRecoveryDecision::Commit(commit) => {
356 ObserverRecoveryTransactionDecision::Commit(ObserverRecoveryTransaction {
357 aggregate: self,
358 commit,
359 })
360 }
361 }
362 }
363}
364
365/// Total transactional observer-recovery decision against the owned aggregate.
366#[derive(Debug, PartialEq, Eq)]
367pub enum ObserverRecoveryTransactionDecision {
368 /// The whole batch was refused; the aggregate is unchanged.
369 Respond {
370 /// Unchanged aggregate returned to its owner.
371 aggregate: ObserverRecoveryAggregate,
372 /// Exact refusal response.
373 response: ObserverRecoveryResponse,
374 },
375 /// Every entry validated; the arm plan may commit atomically.
376 Commit(ObserverRecoveryTransaction),
377}
378
379/// Ownership barrier between arm selection and atomic arm installation.
380///
381/// The aggregate is unreachable while the transaction is pending, so no
382/// progress can advance between the equal-epoch read and the installation.
383/// Consuming [`Self::commit`] installs every arm of the plan at once — a
384/// subset is unrepresentable — and consuming [`Self::abort`] installs none,
385/// so a crash at any point leaves either the complete arm plan or no arm at
386/// all, never a partially-armed request.
387#[derive(Debug, PartialEq, Eq)]
388pub struct ObserverRecoveryTransaction {
389 aggregate: ObserverRecoveryAggregate,
390 commit: ObserverRecoveryCommit,
391}
392
393impl ObserverRecoveryTransaction {
394 /// Borrows the complete equal-epoch arm plan for the durable append.
395 #[must_use]
396 pub fn arms(&self) -> &[ObserverRecoveryArm] {
397 self.commit.arms()
398 }
399
400 /// Borrows the exact request-ordered success response.
401 #[must_use]
402 pub const fn outcome(&self) -> &ObserverRecoveryAccepted {
403 self.commit.outcome()
404 }
405
406 /// Installs the whole arm plan after a confirmed durable append.
407 ///
408 /// Installation is idempotent: replaying a durable recovery against the
409 /// post-state reinstalls the identical equal-epoch arms, so at most one
410 /// arm per conversation ever exists and crash replay converges.
411 #[must_use]
412 pub fn commit(mut self) -> (ObserverRecoveryAggregate, ObserverRecoveryAccepted) {
413 let (arms, outcome) = self.commit.into_parts();
414 for arm in arms {
415 self.aggregate
416 .armed
417 .insert(arm.conversation_id(), arm.refused_epoch());
418 }
419 (self.aggregate, outcome)
420 }
421
422 /// Cancels a failed durable append; not one arm is installed.
423 #[must_use]
424 pub fn abort(self) -> ObserverRecoveryAggregate {
425 self.aggregate
426 }
427}
428
429/// Total transactional progress-advance decision against the owned aggregate.
430#[derive(Debug, PartialEq, Eq)]
431pub enum ObserverProgressAdvanceDecision {
432 /// The advance was refused; the aggregate is unchanged.
433 Refuse {
434 /// Unchanged aggregate returned to its owner.
435 aggregate: ObserverRecoveryAggregate,
436 /// Exact refusal.
437 error: ObserverProgressAdvanceError,
438 },
439 /// The advance validated; the progress write and arm fire may commit
440 /// atomically.
441 Commit(ObserverProgressAdvanceTransaction),
442}
443
444/// Ownership barrier between a validated progress advance and its durable
445/// append.
446///
447/// The aggregate is unreachable while the transaction is pending, so no
448/// recovery batch can read the not-yet-durable progress and no other advance
449/// can race the fired-arm plan. Consuming [`Self::commit`] applies the
450/// progress write and removes the fired arm as ONE mutation — after the
451/// caller has confirmed the durable append of the progress row, the arm-row
452/// deletion, and the wake — and consuming [`Self::abort`] returns the
453/// aggregate byte-for-byte unchanged, arm still installed, so a failed
454/// append never leaves live progress ahead of durable state or a
455/// phantom-fired arm.
456#[derive(Debug, PartialEq, Eq)]
457pub struct ObserverProgressAdvanceTransaction {
458 aggregate: ObserverRecoveryAggregate,
459 conversation_id: ConversationId,
460 presented_progress: DeliverySeq,
461 fired: Option<ObserverRecoveryArm>,
462}
463
464impl ObserverProgressAdvanceTransaction {
465 /// Returns the conversation whose progress row the durable append writes.
466 #[must_use]
467 pub const fn conversation_id(&self) -> ConversationId {
468 self.conversation_id
469 }
470
471 /// Returns the new hard observer progress for the durable append.
472 #[must_use]
473 pub const fn presented_progress(&self) -> DeliverySeq {
474 self.presented_progress
475 }
476
477 /// Returns the fired-arm plan for the durable append.
478 ///
479 /// `Some` names the installed arm whose durable row the same append must
480 /// delete and whose parked rows the same append must wake; `None` means
481 /// the conversation carried no arm.
482 #[must_use]
483 pub const fn fired_arm(&self) -> Option<ObserverRecoveryArm> {
484 self.fired
485 }
486
487 /// Applies the advance after a confirmed durable append.
488 ///
489 /// The progress write and the arm removal are one mutation: the returned
490 /// arm is exactly [`Self::fired_arm`], surrendered so the caller wakes
491 /// the parked rows it durably committed to waking.
492 #[must_use]
493 pub fn commit(mut self) -> (ObserverRecoveryAggregate, Option<ObserverRecoveryArm>) {
494 self.aggregate
495 .progress
496 .insert(self.conversation_id, self.presented_progress);
497 if self.fired.is_some() {
498 self.aggregate.armed.remove(&self.conversation_id);
499 }
500 (self.aggregate, self.fired)
501 }
502
503 /// Cancels a failed durable append; neither progress nor arm changes.
504 #[must_use]
505 pub fn abort(self) -> ObserverRecoveryAggregate {
506 self.aggregate
507 }
508}
509
510/// Total transactional registration decision against the owned aggregate.
511#[derive(Debug, PartialEq, Eq)]
512pub enum ObserverProgressTrackDecision {
513 /// The registration was refused; the aggregate is unchanged.
514 Refuse {
515 /// Unchanged aggregate returned to its owner.
516 aggregate: ObserverRecoveryAggregate,
517 /// Exact refusal.
518 error: ObserverProgressTrackError,
519 },
520 /// The registration validated; the progress row may commit atomically.
521 Commit(ObserverProgressTrackTransaction),
522}
523
524/// Ownership barrier between a validated registration and its durable append.
525///
526/// The aggregate is unreachable while the transaction is pending, so no
527/// recovery batch can plan an equal-epoch arm against a progress row whose
528/// durable append is not yet confirmed. Consuming [`Self::commit`] installs
529/// the new progress row — after the caller has confirmed the durable append —
530/// and consuming [`Self::abort`] returns the aggregate byte-for-byte
531/// unchanged, conversation still untracked, so a failed append never leaves
532/// a live progress row that durable state does not hold.
533#[derive(Debug, PartialEq, Eq)]
534pub struct ObserverProgressTrackTransaction {
535 aggregate: ObserverRecoveryAggregate,
536 conversation_id: ConversationId,
537 observer_progress: DeliverySeq,
538}
539
540impl ObserverProgressTrackTransaction {
541 /// Returns the conversation whose progress row the durable append writes.
542 #[must_use]
543 pub const fn conversation_id(&self) -> ConversationId {
544 self.conversation_id
545 }
546
547 /// Returns the registered hard observer progress for the durable append.
548 #[must_use]
549 pub const fn observer_progress(&self) -> DeliverySeq {
550 self.observer_progress
551 }
552
553 /// Installs the registered progress row after a confirmed durable append.
554 #[must_use]
555 pub fn commit(mut self) -> ObserverRecoveryAggregate {
556 self.aggregate
557 .progress
558 .insert(self.conversation_id, self.observer_progress);
559 self.aggregate
560 }
561
562 /// Cancels a failed durable append; the conversation stays untracked.
563 #[must_use]
564 pub fn abort(self) -> ObserverRecoveryAggregate {
565 self.aggregate
566 }
567}
568
569/// One equal observer epoch that must be armed by an accepted recovery batch.
570#[derive(Clone, Copy, Debug, PartialEq, Eq)]
571pub struct ObserverRecoveryArm {
572 conversation_id: ConversationId,
573 refused_epoch: ObserverEpoch,
574}
575
576impl ObserverRecoveryArm {
577 /// Returns the conversation whose progress event must wake the parked rows.
578 #[must_use]
579 pub const fn conversation_id(self) -> ConversationId {
580 self.conversation_id
581 }
582
583 /// Returns the exact refusal epoch being armed.
584 #[must_use]
585 pub const fn refused_epoch(self) -> ObserverEpoch {
586 self.refused_epoch
587 }
588}
589
590/// Whole-batch observer-recovery commit selected after exhaustive validation.
591///
592/// The arm list and response are produced together. A consumer must persist all
593/// arms atomically before sending [`Self::outcome`], so no validation failure or
594/// crash can expose a partially armed request.
595#[derive(Clone, Debug, PartialEq, Eq)]
596pub struct ObserverRecoveryCommit {
597 arms: Vec<ObserverRecoveryArm>,
598 outcome: ObserverRecoveryAccepted,
599}
600
601impl ObserverRecoveryCommit {
602 /// Borrows the complete equal-epoch arm plan in request order.
603 #[must_use]
604 pub fn arms(&self) -> &[ObserverRecoveryArm] {
605 &self.arms
606 }
607
608 /// Borrows the exact request-ordered success response.
609 #[must_use]
610 pub const fn outcome(&self) -> &ObserverRecoveryAccepted {
611 &self.outcome
612 }
613
614 /// Consumes the commit into the atomic arm plan and success response.
615 #[must_use]
616 pub fn into_parts(self) -> (Vec<ObserverRecoveryArm>, ObserverRecoveryAccepted) {
617 (self.arms, self.outcome)
618 }
619}
620
621/// Total observer-recovery decision.
622#[derive(Clone, Debug, PartialEq, Eq)]
623pub enum ObserverRecoveryDecision {
624 /// Validation or connection-capacity preflight refused the whole batch.
625 Respond(ObserverRecoveryResponse),
626 /// Every entry validated and the complete arm plan may commit atomically.
627 Commit(ObserverRecoveryCommit),
628}
629
630fn wire_count(value: usize) -> u64 {
631 u64::try_from(value).map_or(u64::MAX, core::convert::identity)
632}
633
634/// Applies the observer-recovery list, capacity, and epoch precedence.
635///
636/// Validation order is the frozen R-D1 order: list length, duplicate
637/// conversation, request-index connection capacity, then request-index unknown
638/// or ahead epoch. Only after every entry passes is an arm plan produced.
639/// `observer_progress` must return the current hard observer progress for a
640/// known conversation and `None` for an unknown conversation.
641///
642/// This raw selection is deliberately crate-internal: its progress reads and
643/// the installation of every returned arm must share one serialization
644/// boundary (releasing it between evaluation and installation would violate
645/// the equal-epoch subscribe-then-snapshot rule), and that requirement is
646/// enforceable only by ownership.
647/// [`ObserverRecoveryAggregate::decide_recovery`] is the one public door to
648/// arm selection: it binds these progress reads to the owned aggregate, so
649/// no caller can interleave an acknowledgement or fate advance between
650/// evaluation and installation.
651#[must_use]
652pub(super) fn apply_observer_recovery<F>(
653 request: &ObserverRecoveryHandshake,
654 max_entries: u64,
655 connection_conversation_limit: u64,
656 tracked_conversations: &[ConversationId],
657 mut observer_progress: F,
658) -> ObserverRecoveryDecision
659where
660 F: FnMut(ConversationId) -> Option<DeliverySeq>,
661{
662 let presented_entries = wire_count(request.observer_refusals.len());
663 if presented_entries > max_entries {
664 return ObserverRecoveryDecision::Respond(
665 ObserverRecoveryResponse::invalid_observer_epoch_list(
666 InvalidObserverEpochList::TooManyEntries {
667 presented_entries,
668 max_entries,
669 },
670 ),
671 );
672 }
673
674 let mut first_indices = BTreeMap::new();
675 for (index, refusal) in request.observer_refusals.iter().enumerate() {
676 let request_index = wire_count(index);
677 if let Some(first_index) = first_indices.insert(refusal.conversation_id, request_index) {
678 return ObserverRecoveryDecision::Respond(
679 ObserverRecoveryResponse::invalid_observer_epoch_list(
680 InvalidObserverEpochList::DuplicateConversation {
681 conversation_id: refusal.conversation_id,
682 first_index,
683 duplicate_index: request_index,
684 },
685 ),
686 );
687 }
688 }
689
690 let mut tracked: BTreeSet<_> = tracked_conversations.iter().copied().collect();
691 for refusal in &request.observer_refusals {
692 if tracked.contains(&refusal.conversation_id) {
693 continue;
694 }
695 let occupied = wire_count(tracked.len());
696 if occupied >= connection_conversation_limit {
697 return ObserverRecoveryDecision::Respond(
698 ObserverRecoveryResponse::connection_capacity_exceeded(
699 refusal.conversation_id,
700 connection_conversation_limit,
701 ),
702 );
703 }
704 tracked.insert(refusal.conversation_id);
705 }
706
707 let mut validated = Vec::with_capacity(request.observer_refusals.len());
708 for refusal in &request.observer_refusals {
709 let Some(current_observer_progress) = observer_progress(refusal.conversation_id) else {
710 return ObserverRecoveryDecision::Respond(
711 ObserverRecoveryResponse::invalid_observer_epoch(
712 InvalidObserverEpoch::ConversationUnknown {
713 conversation_id: refusal.conversation_id,
714 presented_epoch: refusal.refused_epoch,
715 },
716 ),
717 );
718 };
719 if refusal.refused_epoch > current_observer_progress {
720 return ObserverRecoveryDecision::Respond(
721 ObserverRecoveryResponse::invalid_observer_epoch(
722 InvalidObserverEpoch::EpochAhead {
723 conversation_id: refusal.conversation_id,
724 presented_epoch: refusal.refused_epoch,
725 current_observer_progress,
726 },
727 ),
728 );
729 }
730 validated.push((refusal, current_observer_progress));
731 }
732
733 let mut arms = Vec::new();
734 let mut statuses = Vec::with_capacity(validated.len());
735 for (refusal, current_observer_progress) in validated {
736 let armed = refusal.refused_epoch == current_observer_progress;
737 if armed {
738 arms.push(ObserverRecoveryArm {
739 conversation_id: refusal.conversation_id,
740 refused_epoch: refusal.refused_epoch,
741 });
742 }
743 statuses.push(ObserverProgressStatus {
744 conversation_id: refusal.conversation_id,
745 refused_epoch: refusal.refused_epoch,
746 current_observer_progress,
747 armed,
748 progressed: !armed,
749 });
750 }
751
752 ObserverRecoveryDecision::Commit(ObserverRecoveryCommit {
753 arms,
754 outcome: ObserverRecoveryAccepted { statuses },
755 })
756}