Skip to main content

arkhe_forge_platform/
projection.rs

1//! L2 Projection observer pipeline.
2//!
3//! L0 emits deterministic events; L2 turns those events into denormalized
4//! read-model rows that PG (or another store) serves to higher layers. This
5//! module exposes: the [`Projection`] trait, a
6//! [`ProjectionRouter`] that dispatches [`EventRecord`]s by `TypeCode`,
7//! a [`ProjectionStore`] abstraction, an in-memory store, and active /
8//! passive / draining lifecycle transitions.
9//!
10//! The PG-backed store, the L0 observer bridge, and the
11//! `kernel_projection_state` chain-anchored view route through the trait surface.
12
13use core::marker::PhantomData;
14use std::collections::HashMap;
15
16use arkhe_forge_core::activity::{ActivityId, ActivityRecord, EntityShellId};
17use arkhe_forge_core::actor::{ActorId, ActorProfile, UserBinding};
18use arkhe_forge_core::brand::ShellId;
19use arkhe_forge_core::context::EventRecord;
20use arkhe_forge_core::entry::{EntryBody, EntryCore, EntryId, EntryParentDepth};
21use arkhe_forge_core::event::{ArkheEvent, CrossShellActivity};
22use arkhe_forge_core::space::{ParentChainDepth, SpaceConfig, SpaceId, SpaceMembership};
23use arkhe_kernel::abi::{InstanceId, Tick, TypeCode};
24use bytes::Bytes;
25use serde::{Deserialize, Serialize};
26
27use crate::manifest::ManifestSnapshot;
28
29// ===================== Lifecycle + Context + Errors =====================
30
31/// Observer worker lifecycle (active-passive HA).
32///
33/// * `Passive` — read-only secondary. Consumes events for warm standby but
34///   does not commit writes upstream.
35/// * `Active` — primary writer. The only projection state permitted to call
36///   `ProjectionStore` mutators.
37/// * `Draining` — graceful shutdown. Rejects new work, flushes in-flight.
38#[derive(Debug, Clone, Copy, PartialEq, Eq)]
39#[non_exhaustive]
40pub enum ObserverState {
41    /// Not primary; events are observed but writes are blocked.
42    Passive,
43    /// Primary writer.
44    Active,
45    /// Winding down — rejects new work.
46    Draining,
47}
48
49/// Outcome of an auto-promote policy evaluation.
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51#[non_exhaustive]
52pub enum PromotionDecision {
53    /// Policy + elapsed time justify a `Passive → Active` transition.
54    Promote,
55    /// Policy requires additional wait / manual operator approval.
56    Wait,
57}
58
59/// Minimum number of KMS health channels that must report `Healthy` for the
60/// `after_60min` auto-promote policy to clear its guardrail. Matches the
61/// default 2-of-3 quorum — operators that provision more channels
62/// can re-tune via a future manifest field.
63pub const HF2_HEALTH_QUORUM_MIN: usize = 2;
64
65/// Per-dispatch context carried alongside an [`EventRecord`]. The `'i`
66/// lifetime reserves the slot that binds to the L0
67/// `Effect<'i, Authorized>` borrow, and also scopes the optional manifest
68/// snapshot reference.
69pub struct ProjectionContext<'i> {
70    /// Tick at which the event is being applied.
71    pub tick: Tick,
72    /// Runtime instance identifier.
73    pub instance_id: InstanceId,
74    /// Active manifest snapshot, if one has been loaded. `None` is legal
75    /// for Tier-0 dev bootstrap paths that run before the first manifest
76    /// has been emitted via `RuntimeBootstrap`.
77    pub manifest: Option<&'i ManifestSnapshot>,
78    _phantom: PhantomData<&'i ()>,
79}
80
81impl<'i> ProjectionContext<'i> {
82    /// Construct a projection dispatch context without a manifest.
83    #[inline]
84    #[must_use]
85    pub fn new(tick: Tick, instance_id: InstanceId) -> Self {
86        Self {
87            tick,
88            instance_id,
89            manifest: None,
90            _phantom: PhantomData,
91        }
92    }
93
94    /// Construct a projection dispatch context with an attached manifest
95    /// snapshot. Callers that have loaded a manifest should use this path
96    /// so projection workers can key on shell policy (tier, cipher, etc.).
97    #[inline]
98    #[must_use]
99    pub fn with_manifest(
100        tick: Tick,
101        instance_id: InstanceId,
102        manifest: &'i ManifestSnapshot,
103    ) -> Self {
104        Self {
105            tick,
106            instance_id,
107            manifest: Some(manifest),
108            _phantom: PhantomData,
109        }
110    }
111}
112
113/// Projection-side failure taxonomy.
114#[derive(Debug, thiserror::Error)]
115#[non_exhaustive]
116pub enum ProjectionError {
117    /// The event stream moved backward (corruption, mis-ordered redelivery,
118    /// or two computes sharing one tick — see
119    /// [`ProjectionRouter::dispatch`]).
120    #[error("projection stream backward: last {last:?}, incoming {incoming:?}")]
121    SequenceBackward {
122        /// Stream head the incoming event was ordered against — the last
123        /// accepted event or, when a failed dispatch is awaiting retry,
124        /// the pinned unapplied position.
125        last: ProjectionCursor,
126        /// Position of the rejected incoming event.
127        incoming: ProjectionCursor,
128    },
129
130    /// The stream skipped an event — a same-tick sequence jump, or a new
131    /// tick whose batch does not start at sequence 0. The replay harness
132    /// needs to fetch the missing range before dispatch can advance.
133    #[error("projection stream gap: last {last:?}, incoming {incoming:?}")]
134    SequenceGap {
135        /// Stream head the incoming event was ordered against — the last
136        /// accepted event or, when a failed dispatch is awaiting retry,
137        /// the pinned unapplied position.
138        last: ProjectionCursor,
139        /// Position of the event that exposed the gap.
140        incoming: ProjectionCursor,
141    },
142
143    /// A DIFFERENT event arrived at a position the router has already
144    /// pinned — the accepted cursor position, or a failed dispatch's
145    /// pending-retry pin. Two same-tick computes collided on the
146    /// `(tick, sequence)` identity (driver contract violation),
147    /// distinguished from a true redelivery/retry by event identity
148    /// (type code + payload).
149    #[error("projection stream position conflict at {at:?}")]
150    PositionConflict {
151        /// The contested stream position.
152        at: ProjectionCursor,
153    },
154
155    /// Caller attempted a mutation in a non-`Active` state (observer is
156    /// Passive or Draining).
157    #[error("observer not active: current state {state:?}")]
158    NotActive {
159        /// Observer state at the time of the attempted mutation.
160        state: ObserverState,
161    },
162
163    /// Storage-layer error (in-memory corruption, PG driver, …).
164    #[error("projection storage error: {0}")]
165    Storage(&'static str),
166
167    /// Event payload failed to decode.
168    #[error("event decode failed: {0}")]
169    DecodeFailed(&'static str),
170
171    /// An event targeted an Actor / Space / Entry / Activity that the
172    /// projection has no row for.
173    #[error("projection row missing")]
174    MissingRow,
175}
176
177// ===================== Projection trait =====================
178
179/// L2 projection worker. Each implementor owns a read-model view that is
180/// kept in sync with the L1 event stream for a specific set of `TypeCode`s.
181///
182/// Implementors must be `Send + Sync` so the `ProjectionRouter` can run
183/// across worker threads. Stream dedup / gap detection is centralised in
184/// the router's own cursor; the router additionally consults
185/// [`Projection::last_applied`] to skip events a projection has already
186/// applied (catch-up after a router rebuild or a partially failed
187/// fan-out).
188pub trait Projection: Send + Sync {
189    /// TypeCodes this projection observes — the router filters incoming
190    /// events against this slice.
191    fn observes(&self) -> &[TypeCode];
192
193    /// Apply an event. Called only after router-side dedup + gap checks
194    /// have succeeded. Implementations must:
195    ///
196    /// 1. Update their internal view state.
197    /// 2. Bump `last_applied` to the event's `(sequence, tick)`.
198    fn on_event(
199        &mut self,
200        event: &EventRecord,
201        ctx: &ProjectionContext<'_>,
202    ) -> Result<(), ProjectionError>;
203
204    /// React to a worker-state transition (Passive ↔ Active ↔ Draining).
205    /// Default is no-op.
206    fn on_state_change(&mut self, _new_state: ObserverState) -> Result<(), ProjectionError> {
207        Ok(())
208    }
209
210    /// Last `(sequence, tick)` applied — `None` if the projection is fresh.
211    /// The router skips any event at or before this position (tick-major
212    /// order), so the `on_event` bump is what makes application
213    /// at-most-once across redeliveries.
214    fn last_applied(&self) -> Option<(u64, Tick)>;
215}
216
217// ===================== Projection view structs =====================
218
219/// Composite event-stream position. `EventRecord.sequence` is
220/// per-compute — it restarts at 0 for every action's `ActionContext` — so
221/// an event is identified in the stream by the `(tick, sequence)` pair,
222/// ordered tick-major.
223#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)]
224pub struct ProjectionCursor {
225    /// Sequence number within the emitting compute's batch.
226    pub sequence: u64,
227    /// Tick at which the event was emitted.
228    pub tick: Tick,
229}
230
231/// Actor-facing read-model row — `ActorProfile` + optional `UserBinding`.
232#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
233pub struct ActorProjection {
234    /// Wire schema version.
235    pub schema_version: u16,
236    /// Actor identity.
237    pub actor_id: ActorId,
238    /// Authoritative `ActorProfile` Component.
239    pub profile: ActorProfile,
240    /// `UserBinding` is present iff the actor is `Authenticated` (E-actor-2).
241    pub user_binding: Option<UserBinding>,
242    /// Event cursor — dedup / gap anchor.
243    pub cursor: Option<ProjectionCursor>,
244}
245
246/// Space read-model row — `SpaceConfig` + parent-chain cache + membership.
247#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
248pub struct SpaceProjection {
249    /// Wire schema version.
250    pub schema_version: u16,
251    /// Space identity.
252    pub space_id: SpaceId,
253    /// Authoritative `SpaceConfig` Component.
254    pub config: SpaceConfig,
255    /// Cached parent-chain depth (E-space-4 O(1)).
256    pub parent_chain_depth: Option<ParentChainDepth>,
257    /// Membership list for PrivateInvite Spaces.
258    pub membership: Option<SpaceMembership>,
259    /// Event cursor.
260    pub cursor: Option<ProjectionCursor>,
261}
262
263/// Entry read-model row — `EntryCore` + body metadata + depth cache.
264#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
265pub struct EntryProjection {
266    /// Wire schema version.
267    pub schema_version: u16,
268    /// Entry identity.
269    pub entry_id: EntryId,
270    /// Authoritative `EntryCore` Component.
271    pub core: EntryCore,
272    /// `EntryBody` — absent when soft-deleted (E-entry-5).
273    pub body: Option<EntryBody>,
274    /// Cached parent-chain depth (E-entry-3).
275    pub parent_depth: Option<EntryParentDepth>,
276    /// Event cursor.
277    pub cursor: Option<ProjectionCursor>,
278}
279
280/// Activity read-model row — `ActivityRecord` + optional Extension-target
281/// shell marker.
282#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
283pub struct ActivityProjection {
284    /// Wire schema version.
285    pub schema_version: u16,
286    /// Activity identity.
287    pub activity_id: ActivityId,
288    /// Authoritative `ActivityRecord` Component.
289    pub record: ActivityRecord,
290    /// Shell marker for Extension-target Activities (E-act-7).
291    pub entity_shell_id: Option<EntityShellId>,
292    /// Event cursor.
293    pub cursor: Option<ProjectionCursor>,
294}
295
296// ===================== ProjectionStore =====================
297
298/// Storage abstraction for projection rows. An in-memory
299/// implementation ships with this crate; PG-backed storage plugs in
300/// at the L2 service layer.
301pub trait ProjectionStore: Send + Sync {
302    /// Upsert an Actor row. `Active` observer only.
303    fn upsert_actor(&mut self, row: &ActorProjection) -> Result<(), ProjectionError>;
304
305    /// Upsert a Space row. `Active` observer only.
306    fn upsert_space(&mut self, row: &SpaceProjection) -> Result<(), ProjectionError>;
307
308    /// Upsert an Entry row. `Active` observer only.
309    fn upsert_entry(&mut self, row: &EntryProjection) -> Result<(), ProjectionError>;
310
311    /// Upsert an Activity row. `Active` observer only.
312    fn upsert_activity(&mut self, row: &ActivityProjection) -> Result<(), ProjectionError>;
313
314    /// Read an Actor row.
315    fn get_actor(&self, actor_id: ActorId) -> Option<ActorProjection>;
316    /// Read a Space row.
317    fn get_space(&self, space_id: SpaceId) -> Option<SpaceProjection>;
318    /// Read an Entry row.
319    fn get_entry(&self, entry_id: EntryId) -> Option<EntryProjection>;
320    /// Read an Activity row.
321    fn get_activity(&self, activity_id: ActivityId) -> Option<ActivityProjection>;
322}
323
324/// In-memory [`ProjectionStore`] — intended for tests and Tier-0 dev runs.
325#[derive(Debug, Default)]
326pub struct InMemoryProjectionStore {
327    actors: HashMap<ActorId, ActorProjection>,
328    spaces: HashMap<SpaceId, SpaceProjection>,
329    entries: HashMap<EntryId, EntryProjection>,
330    activities: HashMap<ActivityId, ActivityProjection>,
331}
332
333impl InMemoryProjectionStore {
334    /// Construct an empty store.
335    #[inline]
336    #[must_use]
337    pub fn new() -> Self {
338        Self::default()
339    }
340}
341
342impl ProjectionStore for InMemoryProjectionStore {
343    fn upsert_actor(&mut self, row: &ActorProjection) -> Result<(), ProjectionError> {
344        self.actors.insert(row.actor_id, row.clone());
345        Ok(())
346    }
347    fn upsert_space(&mut self, row: &SpaceProjection) -> Result<(), ProjectionError> {
348        self.spaces.insert(row.space_id, row.clone());
349        Ok(())
350    }
351    fn upsert_entry(&mut self, row: &EntryProjection) -> Result<(), ProjectionError> {
352        self.entries.insert(row.entry_id, row.clone());
353        Ok(())
354    }
355    fn upsert_activity(&mut self, row: &ActivityProjection) -> Result<(), ProjectionError> {
356        self.activities.insert(row.activity_id, row.clone());
357        Ok(())
358    }
359    fn get_actor(&self, actor_id: ActorId) -> Option<ActorProjection> {
360        self.actors.get(&actor_id).cloned()
361    }
362    fn get_space(&self, space_id: SpaceId) -> Option<SpaceProjection> {
363        self.spaces.get(&space_id).cloned()
364    }
365    fn get_entry(&self, entry_id: EntryId) -> Option<EntryProjection> {
366        self.entries.get(&entry_id).cloned()
367    }
368    fn get_activity(&self, activity_id: ActivityId) -> Option<ActivityProjection> {
369        self.activities.get(&activity_id).cloned()
370    }
371}
372
373// ===================== Router =====================
374
375/// Event-stream dispatcher. Matches incoming events to registered
376/// projections by `TypeCode`, enforces stream-order dedup + gap detection
377/// via a router-held `(tick, sequence)` cursor, and propagates observer
378/// state transitions.
379pub struct ProjectionRouter {
380    projections: Vec<Box<dyn Projection>>,
381    /// `TypeCode -> projection indices that observe it`, built at
382    /// `register()` time. Indices are stored in registration order, so
383    /// dispatch visits matching projections in the same order a linear scan
384    /// would — dispatch-order semantics are preserved (see `dispatch`).
385    observers_by_code: HashMap<TypeCode, Vec<usize>>,
386    /// Position of the last event accepted by `dispatch`, including events
387    /// no projection observes — gap detection needs the full stream, not
388    /// the per-projection filtered view. Advanced only after a fully
389    /// successful fan-out so a failed dispatch stays retryable.
390    cursor: Option<ProjectionCursor>,
391    /// Identity (type code + payload) of the last accepted event — lets
392    /// the cursor-equality path distinguish a true redelivery (`Ok(0)`)
393    /// from a DIFFERENT event colliding on the same position
394    /// ([`ProjectionError::PositionConflict`]).
395    last_event: Option<(u32, Bytes)>,
396    /// Position + identity (type code, payload) of the event a failed
397    /// fan-out left unapplied — set on every failed fan-out (a
398    /// projection apply error), first or mid-stream; ordering
399    /// rejections never move it (re-pinning to a stray position would
400    /// deadlock the legitimate retry). Only a retry of this exact
401    /// EVENT may proceed: a skip-ahead would silently advance past the
402    /// lost event, and a DIFFERENT event at the same position would
403    /// conflate same-tick compute streams.
404    pending_retry: Option<(ProjectionCursor, u32, Bytes)>,
405    state: ObserverState,
406}
407
408impl ProjectionRouter {
409    /// Build a router in the `Passive` state — promote to `Active` before
410    /// committing writes.
411    #[inline]
412    #[must_use]
413    pub fn new() -> Self {
414        Self {
415            projections: Vec::new(),
416            observers_by_code: HashMap::new(),
417            cursor: None,
418            last_event: None,
419            pending_retry: None,
420            state: ObserverState::Passive,
421        }
422    }
423
424    /// Register a projection. Its observed `TypeCode`s are folded into the
425    /// dispatch index so `dispatch` only visits matching projections.
426    pub fn register(&mut self, projection: Box<dyn Projection>) {
427        let idx = self.projections.len();
428        for &tc in projection.observes() {
429            self.observers_by_code.entry(tc).or_default().push(idx);
430        }
431        self.projections.push(projection);
432    }
433
434    /// Current observer state.
435    #[inline]
436    #[must_use]
437    pub fn state(&self) -> ObserverState {
438        self.state
439    }
440
441    /// Transition to `Active` — fails if currently `Draining`.
442    pub fn promote_to_active(&mut self) -> Result<(), ProjectionError> {
443        self.transition(ObserverState::Active)
444    }
445
446    /// Evaluate the shell's `kms_auto_promote` policy against three inputs:
447    /// the elapsed outage, the multi-channel KMS health quorum, and the
448    /// threshold-HSM share readiness.
449    ///
450    /// Policy values:
451    ///
452    /// | `kms_auto_promote`  | Decision matrix |
453    /// |---------------------|-----------------|
454    /// | `"manual"`          | Always `Some(Wait)` — operator drives the promotion manually. |
455    /// | `"after_60min"`     | `Some(Promote)` iff `primary_down_duration >= 1h` **and** the KMS health quorum has at least [`HF2_HEALTH_QUORUM_MIN`] channels healthy; otherwise `Some(Wait)`. |
456    /// | `"threshold_hsm"`   | `Some(Promote)` iff `threshold_ready` (t-of-n shares collected); otherwise `Some(Wait)`. |
457    /// | other               | `None` — unrecognised policy string, operator intervention required. |
458    ///
459    /// Returning `None` is conservative by design: callers treat unknown
460    /// policies as "do not auto-promote" and fall back to manual operator
461    /// action.
462    #[must_use]
463    pub fn evaluate_auto_promote(
464        &self,
465        manifest: &crate::manifest::ManifestSnapshot,
466        primary_down_duration: core::time::Duration,
467        health: &crate::hf2_kms::health::MultiChannelHealth,
468        threshold_ready: bool,
469    ) -> Option<PromotionDecision> {
470        match manifest.audit.kms_auto_promote.as_str() {
471            "manual" => Some(PromotionDecision::Wait),
472            "after_60min" => {
473                let elapsed_ok = primary_down_duration >= core::time::Duration::from_secs(60 * 60);
474                let health_ok = health.healthy_count() >= HF2_HEALTH_QUORUM_MIN;
475                if elapsed_ok && health_ok {
476                    Some(PromotionDecision::Promote)
477                } else {
478                    Some(PromotionDecision::Wait)
479                }
480            }
481            "threshold_hsm" => {
482                if threshold_ready {
483                    Some(PromotionDecision::Promote)
484                } else {
485                    Some(PromotionDecision::Wait)
486                }
487            }
488            _ => None,
489        }
490    }
491
492    /// Transition to `Passive`. Used when ceding primary to a peer.
493    pub fn demote_to_passive(&mut self) -> Result<(), ProjectionError> {
494        self.transition(ObserverState::Passive)
495    }
496
497    /// Transition to `Draining`. Terminal — no further state changes.
498    pub fn begin_draining(&mut self) -> Result<(), ProjectionError> {
499        self.transition(ObserverState::Draining)
500    }
501
502    fn transition(&mut self, next: ObserverState) -> Result<(), ProjectionError> {
503        // Draining is terminal: once a graceful shutdown begins, no
504        // transition path (including Draining → Passive → Active) may
505        // resurrect the router. Gating here covers every public
506        // transition method with a single guard.
507        if self.state == ObserverState::Draining {
508            return Err(ProjectionError::NotActive { state: self.state });
509        }
510        for p in &mut self.projections {
511            p.on_state_change(next)?;
512        }
513        self.state = next;
514        Ok(())
515    }
516
517    /// Dispatch an event to every matching projection. Returns `Ok(n)`
518    /// where `n` is the number of projections that applied the event.
519    ///
520    /// Matching projections are visited in registration order (the dispatch
521    /// index stores their indices in that order), so a projection registered
522    /// earlier always sees an event before one registered later — callers may
523    /// rely on that ordering. A `TypeCode` with no registered observer is a
524    /// cheap miss (no index entry) returning `Ok(0)` — the event still
525    /// advances the stream cursor.
526    ///
527    /// ## Stream-order contract
528    ///
529    /// `EventRecord.sequence` is per-compute: every compute's batch starts
530    /// at 0 and is contiguous, so an event's stream identity is the
531    /// composite `(tick, sequence)` pair, ordered tick-major. Callers must
532    /// feed every drained event of a compute in emission order and must
533    /// advance the tick between computes. The kernel scheduler permits
534    /// same-tick scheduled actions, but two same-tick computes emit
535    /// colliding positions — this router rejects the collision loudly
536    /// rather than conflating the streams: a colliding head behind the
537    /// cursor is [`ProjectionError::SequenceBackward`], and a DIFFERENT
538    /// event at exactly the cursor position (the single-event-batch
539    /// shape) is [`ProjectionError::PositionConflict`], distinguished
540    /// from a true redelivery by event identity (type code + payload).
541    ///
542    /// Checks against the stream cursor (last accepted event):
543    ///
544    /// * the same `(tick, sequence)` redelivered with the same identity —
545    ///   duplicate, `Ok(0)`;
546    /// * a different event at the cursor position —
547    ///   [`ProjectionError::PositionConflict`];
548    /// * a position behind the cursor —
549    ///   [`ProjectionError::SequenceBackward`];
550    /// * a same-tick sequence jump, or a new tick whose batch does not
551    ///   start at sequence 0 — [`ProjectionError::SequenceGap`];
552    /// * a fresh router (no cursor yet) accepts any position as its
553    ///   resume anchor.
554    ///
555    /// The cursor advances only after a fully successful fan-out. A
556    /// FAILED fan-out (first or mid-stream) pins the unapplied event's
557    /// position AND identity: only a retry of that exact event may
558    /// proceed — a skip-ahead is a gap (it would silently advance past
559    /// the lost event), a different event at the pinned position is a
560    /// [`ProjectionError::PositionConflict`] — and the projections that
561    /// already applied it before the failure are skipped on the retry
562    /// via [`Projection::last_applied`].
563    ///
564    /// Only the `Active` state may dispatch — `Passive` and `Draining`
565    /// reject with [`ProjectionError::NotActive`]. The `Passive` rejection
566    /// is the production guardrail for active-passive HA: a secondary
567    /// observer that mistakenly accepts writes would create split-brain
568    /// rows in the PG-backed store. The dylint `arkhe-trait-default-check`
569    /// CI gate ensures the contract is honoured by every L2 deployment.
570    pub fn dispatch(
571        &mut self,
572        event: &EventRecord,
573        ctx: &ProjectionContext<'_>,
574    ) -> Result<usize, ProjectionError> {
575        if self.state != ObserverState::Active {
576            return Err(ProjectionError::NotActive { state: self.state });
577        }
578        let incoming = ProjectionCursor {
579            sequence: event.sequence,
580            tick: event.tick,
581        };
582        if let Some(last) = self.cursor {
583            if incoming == last {
584                // Same position: a true redelivery is a no-op, but a
585                // DIFFERENT event here means two same-tick computes
586                // collided on the cursor — never conflate silently.
587                let is_redelivery = self.last_event.as_ref().is_some_and(|(tc, payload)| {
588                    *tc == event.type_code && *payload == event.payload
589                });
590                if is_redelivery {
591                    return Ok(0);
592                }
593                return Err(ProjectionError::PositionConflict { at: incoming });
594            }
595        }
596        if let Some((pending, pending_tc, pending_payload)) = &self.pending_retry {
597            // A prior dispatch failed at `pending` (first or mid-stream)
598            // and was never applied — accept only the retry of that exact
599            // EVENT: a skip-ahead would lose it silently, and a different
600            // event at the same position would conflate same-tick compute
601            // streams. The pending position already passed the ordering
602            // checks against the cursor when it first arrived.
603            let pending = *pending;
604            if (incoming.tick, incoming.sequence) < (pending.tick, pending.sequence) {
605                return Err(ProjectionError::SequenceBackward {
606                    last: pending,
607                    incoming,
608                });
609            }
610            if incoming != pending {
611                return Err(ProjectionError::SequenceGap {
612                    last: pending,
613                    incoming,
614                });
615            }
616            if *pending_tc != event.type_code || *pending_payload != event.payload {
617                return Err(ProjectionError::PositionConflict { at: incoming });
618            }
619        } else if let Some(last) = self.cursor {
620            if (incoming.tick, incoming.sequence) < (last.tick, last.sequence) {
621                return Err(ProjectionError::SequenceBackward { last, incoming });
622            }
623            let contiguous = if incoming.tick == last.tick {
624                incoming.sequence == last.sequence.saturating_add(1)
625            } else {
626                // A new tick means a new compute whose sequence restarts
627                // at 0 — anything else is a missing batch head.
628                incoming.sequence == 0
629            };
630            if !contiguous {
631                return Err(ProjectionError::SequenceGap { last, incoming });
632            }
633        }
634        let tc = TypeCode(event.type_code);
635        // Split-borrow the disjoint fields: the index slice (immutable) and
636        // the projection vec (mutable) are borrowed independently so the loop
637        // can mutate `projections[i]` while reading the matching index list.
638        let Self {
639            projections,
640            observers_by_code,
641            cursor,
642            last_event,
643            pending_retry,
644            ..
645        } = self;
646        // Indices are already in registration order.
647        let mut applied = 0usize;
648        if let Some(matching) = observers_by_code.get(&tc) {
649            for &i in matching {
650                let p = &mut projections[i];
651                if let Some((last_seq, last_tick)) = p.last_applied() {
652                    // Already applied — catch-up redelivery into a rebuilt
653                    // router or a retry after a partial fan-out failure.
654                    if (incoming.tick, incoming.sequence) <= (last_tick, last_seq) {
655                        continue;
656                    }
657                }
658                if let Err(e) = p.on_event(event, ctx) {
659                    *pending_retry = Some((incoming, event.type_code, event.payload.clone()));
660                    return Err(e);
661                }
662                applied += 1;
663            }
664        }
665        *cursor = Some(incoming);
666        *last_event = Some((event.type_code, event.payload.clone()));
667        *pending_retry = None;
668        Ok(applied)
669    }
670}
671
672impl Default for ProjectionRouter {
673    fn default() -> Self {
674        Self::new()
675    }
676}
677
678// ===================== CrossShellActivity fanout =====================
679
680/// Read-only fanout projection for `CrossShellActivity`. Stores cross-shell
681/// notifications keyed by the target shell — never touches the source
682/// shell's rows, preserving shell isolation (E-act-2 RA tier).
683#[derive(Debug)]
684pub struct CrossShellActivityFanout {
685    observes: [TypeCode; 1],
686    by_target_shell: HashMap<ShellId, Vec<CrossShellActivity>>,
687    cursor: Option<ProjectionCursor>,
688}
689
690impl Default for CrossShellActivityFanout {
691    fn default() -> Self {
692        Self::new()
693    }
694}
695
696impl CrossShellActivityFanout {
697    /// Construct a fresh fanout, pre-wired to observe `CrossShellActivity`.
698    #[inline]
699    #[must_use]
700    pub fn new() -> Self {
701        Self {
702            observes: [TypeCode(CrossShellActivity::TYPE_CODE)],
703            by_target_shell: HashMap::new(),
704            cursor: None,
705        }
706    }
707
708    /// Borrow the notification queue for a shell (read-only).
709    #[inline]
710    #[must_use]
711    pub fn notifications_for(&self, shell: &ShellId) -> &[CrossShellActivity] {
712        self.by_target_shell
713            .get(shell)
714            .map(Vec::as_slice)
715            .unwrap_or(&[])
716    }
717}
718
719impl Projection for CrossShellActivityFanout {
720    fn observes(&self) -> &[TypeCode] {
721        &self.observes
722    }
723
724    fn on_event(
725        &mut self,
726        event: &EventRecord,
727        _ctx: &ProjectionContext<'_>,
728    ) -> Result<(), ProjectionError> {
729        let notice: CrossShellActivity = postcard::from_bytes(&event.payload)
730            .map_err(|_| ProjectionError::DecodeFailed("CrossShellActivity payload"))?;
731        self.by_target_shell
732            .entry(notice.target_shell_id)
733            .or_default()
734            .push(notice);
735        self.cursor = Some(ProjectionCursor {
736            sequence: event.sequence,
737            tick: event.tick,
738        });
739        Ok(())
740    }
741
742    fn last_applied(&self) -> Option<(u64, Tick)> {
743        self.cursor.map(|c| (c.sequence, c.tick))
744    }
745}
746
747// ===================== Tests =====================
748
749#[cfg(test)]
750#[allow(clippy::unwrap_used, clippy::expect_used)]
751mod tests {
752    use std::sync::atomic::{AtomicUsize, Ordering};
753    use std::sync::Arc;
754
755    use super::*;
756    use arkhe_forge_core::actor::ActorKind;
757    use arkhe_forge_core::component::BoundedString;
758    use arkhe_kernel::abi::EntityId;
759    use bytes::Bytes;
760
761    fn sid(byte: u8) -> ShellId {
762        ShellId([byte; 16])
763    }
764
765    /// Counts `on_event` applications — observability for fan-out and
766    /// at-most-once assertions once the projection is boxed into a router.
767    struct CountingProjection {
768        observes: [TypeCode; 1],
769        cursor: Option<ProjectionCursor>,
770        applied: Arc<AtomicUsize>,
771    }
772
773    impl CountingProjection {
774        fn new(applied: Arc<AtomicUsize>) -> Self {
775            Self {
776                observes: [TypeCode(CrossShellActivity::TYPE_CODE)],
777                cursor: None,
778                applied,
779            }
780        }
781    }
782
783    impl Projection for CountingProjection {
784        fn observes(&self) -> &[TypeCode] {
785            &self.observes
786        }
787
788        fn on_event(
789            &mut self,
790            event: &EventRecord,
791            _ctx: &ProjectionContext<'_>,
792        ) -> Result<(), ProjectionError> {
793            self.applied.fetch_add(1, Ordering::SeqCst);
794            self.cursor = Some(ProjectionCursor {
795                sequence: event.sequence,
796                tick: event.tick,
797            });
798            Ok(())
799        }
800
801        fn last_applied(&self) -> Option<(u64, Tick)> {
802            self.cursor.map(|c| (c.sequence, c.tick))
803        }
804    }
805
806    /// Fails its first `on_event`, succeeds afterwards — drives the
807    /// partial-fan-out retry path.
808    struct FailOnceProjection {
809        observes: [TypeCode; 1],
810        cursor: Option<ProjectionCursor>,
811        failed: bool,
812    }
813
814    impl FailOnceProjection {
815        fn new() -> Self {
816            Self {
817                observes: [TypeCode(CrossShellActivity::TYPE_CODE)],
818                cursor: None,
819                failed: false,
820            }
821        }
822    }
823
824    impl Projection for FailOnceProjection {
825        fn observes(&self) -> &[TypeCode] {
826            &self.observes
827        }
828
829        fn on_event(
830            &mut self,
831            event: &EventRecord,
832            _ctx: &ProjectionContext<'_>,
833        ) -> Result<(), ProjectionError> {
834            if !self.failed {
835                self.failed = true;
836                return Err(ProjectionError::Storage("injected first-apply failure"));
837            }
838            self.cursor = Some(ProjectionCursor {
839                sequence: event.sequence,
840                tick: event.tick,
841            });
842            Ok(())
843        }
844
845        fn last_applied(&self) -> Option<(u64, Tick)> {
846            self.cursor.map(|c| (c.sequence, c.tick))
847        }
848    }
849
850    fn ent(v: u64) -> EntityId {
851        EntityId::new(v).unwrap()
852    }
853
854    fn make_cross_shell_event(seq: u64, tick: u64, target: ShellId) -> EventRecord {
855        let notice = CrossShellActivity {
856            schema_version: 1,
857            actor: ActorId::new(ent(1)),
858            target_shell_id: target,
859            record_shell_id: sid(0xAA),
860            detected_tick: Tick(tick),
861        };
862        EventRecord {
863            type_code: CrossShellActivity::TYPE_CODE,
864            sequence: seq,
865            tick: Tick(tick),
866            payload: Bytes::from(postcard::to_stdvec(&notice).unwrap()),
867        }
868    }
869
870    fn ctx(tick: u64) -> ProjectionContext<'static> {
871        ProjectionContext::new(Tick(tick), InstanceId::new(1).unwrap())
872    }
873
874    #[test]
875    fn router_defaults_to_passive() {
876        let r = ProjectionRouter::new();
877        assert_eq!(r.state(), ObserverState::Passive);
878    }
879
880    #[test]
881    fn router_promote_then_demote_then_drain() {
882        let mut r = ProjectionRouter::new();
883        r.promote_to_active().unwrap();
884        assert_eq!(r.state(), ObserverState::Active);
885        r.demote_to_passive().unwrap();
886        assert_eq!(r.state(), ObserverState::Passive);
887        r.begin_draining().unwrap();
888        assert_eq!(r.state(), ObserverState::Draining);
889        // Draining is terminal — EVERY transition out rejects through the
890        // single gate: promote, the Draining → Passive resurrection edge,
891        // and a repeated drain.
892        assert!(r.promote_to_active().is_err());
893        assert!(r.demote_to_passive().is_err());
894        assert!(r.begin_draining().is_err());
895        assert_eq!(r.state(), ObserverState::Draining);
896    }
897
898    #[test]
899    fn cross_shell_fanout_routes_to_target_shell_only() {
900        let mut r = ProjectionRouter::new();
901        r.promote_to_active().unwrap();
902        r.register(Box::new(CrossShellActivityFanout::new()));
903        let target = sid(0x33);
904        let ev = make_cross_shell_event(0, 100, target);
905        let applied = r.dispatch(&ev, &ctx(100)).unwrap();
906        assert_eq!(applied, 1);
907    }
908
909    /// #21 — the dispatch index must fan an event out to EVERY projection
910    /// that observes its TypeCode, in registration order. Two fanouts both
911    /// observing `CrossShellActivity` must each apply the same event.
912    #[test]
913    fn dispatcher_fans_out_to_all_matching_in_registration_order() {
914        let mut r = ProjectionRouter::new();
915        r.promote_to_active().unwrap();
916        r.register(Box::new(CrossShellActivityFanout::new()));
917        r.register(Box::new(CrossShellActivityFanout::new()));
918        let target = sid(0x44);
919        let ev = make_cross_shell_event(0, 100, target);
920        let applied = r.dispatch(&ev, &ctx(100)).unwrap();
921        assert_eq!(applied, 2, "both matching projections must apply the event");
922    }
923
924    /// #21 — a TypeCode with no registered observer is a cheap index miss
925    /// (no entry) and returns Ok(0) without scanning any projection.
926    #[test]
927    fn dispatcher_unobserved_typecode_is_index_miss() {
928        let mut r = ProjectionRouter::new();
929        r.promote_to_active().unwrap();
930        r.register(Box::new(CrossShellActivityFanout::new()));
931        let other_event = EventRecord {
932            type_code: 0x0003_0F02, // UserErasureScheduled — not observed
933            sequence: 0,
934            tick: Tick(1),
935            payload: Bytes::new(),
936        };
937        assert_eq!(r.dispatch(&other_event, &ctx(1)).unwrap(), 0);
938    }
939
940    #[test]
941    fn dispatcher_skips_projection_with_no_matching_observer() {
942        let mut r = ProjectionRouter::new();
943        r.promote_to_active().unwrap();
944        r.register(Box::new(CrossShellActivityFanout::new()));
945        let other_event = EventRecord {
946            type_code: 0x0003_0F02, // UserErasureScheduled
947            sequence: 0,
948            tick: Tick(1),
949            payload: Bytes::new(),
950        };
951        let applied = r.dispatch(&other_event, &ctx(1)).unwrap();
952        assert_eq!(applied, 0, "non-observed TypeCode must not hit the fanout");
953    }
954
955    #[test]
956    fn dispatcher_dedups_redelivered_event() {
957        let mut r = ProjectionRouter::new();
958        r.promote_to_active().unwrap();
959        r.register(Box::new(CrossShellActivityFanout::new()));
960        let target = sid(0x10);
961        let ev = make_cross_shell_event(0, 5, target);
962        r.dispatch(&ev, &ctx(5)).unwrap();
963        let applied_again = r.dispatch(&ev, &ctx(5)).unwrap();
964        assert_eq!(applied_again, 0, "redelivered (tick, sequence) must no-op");
965    }
966
967    #[test]
968    fn dispatcher_rejects_same_tick_gap() {
969        let mut r = ProjectionRouter::new();
970        r.promote_to_active().unwrap();
971        r.register(Box::new(CrossShellActivityFanout::new()));
972        let target = sid(0x10);
973        r.dispatch(&make_cross_shell_event(0, 5, target), &ctx(5))
974            .unwrap();
975        // Jump to sequence 2 within tick 5 — sequence 1 missing.
976        let err = r
977            .dispatch(&make_cross_shell_event(2, 5, target), &ctx(5))
978            .unwrap_err();
979        assert!(matches!(err, ProjectionError::SequenceGap { .. }));
980    }
981
982    #[test]
983    fn dispatcher_rejects_new_tick_missing_batch_head() {
984        let mut r = ProjectionRouter::new();
985        r.promote_to_active().unwrap();
986        r.register(Box::new(CrossShellActivityFanout::new()));
987        let target = sid(0x10);
988        r.dispatch(&make_cross_shell_event(0, 5, target), &ctx(5))
989            .unwrap();
990        // Tick 6 batch must start at sequence 0 — sequence 3 means the
991        // head of the new compute's batch is missing.
992        let err = r
993            .dispatch(&make_cross_shell_event(3, 6, target), &ctx(6))
994            .unwrap_err();
995        assert!(matches!(err, ProjectionError::SequenceGap { .. }));
996    }
997
998    #[test]
999    fn dispatcher_rejects_backward_sequence() {
1000        let mut r = ProjectionRouter::new();
1001        r.promote_to_active().unwrap();
1002        r.register(Box::new(CrossShellActivityFanout::new()));
1003        let target = sid(0x10);
1004        r.dispatch(&make_cross_shell_event(2, 5, target), &ctx(5))
1005            .unwrap();
1006        let err = r
1007            .dispatch(&make_cross_shell_event(1, 5, target), &ctx(5))
1008            .unwrap_err();
1009        assert!(matches!(err, ProjectionError::SequenceBackward { .. }));
1010    }
1011
1012    #[test]
1013    fn dispatcher_rejects_backward_tick() {
1014        let mut r = ProjectionRouter::new();
1015        r.promote_to_active().unwrap();
1016        r.register(Box::new(CrossShellActivityFanout::new()));
1017        let target = sid(0x10);
1018        r.dispatch(&make_cross_shell_event(0, 5, target), &ctx(5))
1019            .unwrap();
1020        let err = r
1021            .dispatch(&make_cross_shell_event(0, 4, target), &ctx(4))
1022            .unwrap_err();
1023        assert!(matches!(err, ProjectionError::SequenceBackward { .. }));
1024    }
1025
1026    /// Drivers must advance the tick between computes. If two computes
1027    /// share a tick, the second batch head `(tick, 0)` lands behind the
1028    /// cursor and must be rejected loudly — never silently dropped as a
1029    /// duplicate.
1030    #[test]
1031    fn same_tick_second_compute_head_rejected_loudly() {
1032        let mut r = ProjectionRouter::new();
1033        r.promote_to_active().unwrap();
1034        r.register(Box::new(CrossShellActivityFanout::new()));
1035        let target = sid(0x10);
1036        r.dispatch(&make_cross_shell_event(0, 5, target), &ctx(5))
1037            .unwrap();
1038        r.dispatch(&make_cross_shell_event(1, 5, target), &ctx(5))
1039            .unwrap();
1040        let err = r
1041            .dispatch(&make_cross_shell_event(0, 5, target), &ctx(5))
1042            .unwrap_err();
1043        assert!(matches!(err, ProjectionError::SequenceBackward { .. }));
1044    }
1045
1046    /// Same-tick compute collision with SINGLE-event batches: the second
1047    /// compute's head lands exactly on the cursor, so the equality path
1048    /// must compare event identity — a DIFFERENT event there is a
1049    /// `PositionConflict`, never a silent `Ok(0)` "redelivery".
1050    #[test]
1051    fn same_tick_single_event_batches_conflict_loudly() {
1052        let mut r = ProjectionRouter::new();
1053        r.promote_to_active().unwrap();
1054        r.register(Box::new(CrossShellActivityFanout::new()));
1055        r.dispatch(&make_cross_shell_event(0, 5, sid(0x10)), &ctx(5))
1056            .unwrap();
1057        // Distinct payload (different target shell) at the same (5, 0).
1058        let err = r
1059            .dispatch(&make_cross_shell_event(0, 5, sid(0x20)), &ctx(5))
1060            .unwrap_err();
1061        assert!(matches!(
1062            err,
1063            ProjectionError::PositionConflict {
1064                at: ProjectionCursor {
1065                    sequence: 0,
1066                    tick: Tick(5)
1067                }
1068            }
1069        ));
1070    }
1071
1072    /// A failed FIRST dispatch must not let a later position anchor the
1073    /// cursor silently past the lost event: only the exact retry may
1074    /// proceed.
1075    #[test]
1076    fn failed_first_dispatch_pins_the_anchor_until_retried() {
1077        let mut r = ProjectionRouter::new();
1078        r.promote_to_active().unwrap();
1079        r.register(Box::new(FailOnceProjection::new()));
1080        let ev = make_cross_shell_event(0, 5, sid(0x10));
1081        assert!(r.dispatch(&ev, &ctx(5)).is_err());
1082        // Skipping ahead past the failed event is a loud gap...
1083        let err = r
1084            .dispatch(&make_cross_shell_event(0, 6, sid(0x10)), &ctx(6))
1085            .unwrap_err();
1086        assert!(matches!(err, ProjectionError::SequenceGap { .. }));
1087        // ...while retrying the exact position succeeds and re-anchors.
1088        assert_eq!(r.dispatch(&ev, &ctx(5)).unwrap(), 1);
1089        assert_eq!(
1090            r.dispatch(&make_cross_shell_event(0, 6, sid(0x10)), &ctx(6))
1091                .unwrap(),
1092            1
1093        );
1094    }
1095
1096    /// The failed-position pin holds the event IDENTITY too: a
1097    /// DIFFERENT event arriving at the never-accepted position is a
1098    /// `PositionConflict`, never silently absorbed as the "retry" —
1099    /// otherwise two same-tick computes would conflate through the
1100    /// failure window.
1101    #[test]
1102    fn failed_first_dispatch_rejects_a_different_event_at_the_pinned_position() {
1103        let mut r = ProjectionRouter::new();
1104        r.promote_to_active().unwrap();
1105        r.register(Box::new(FailOnceProjection::new()));
1106        let ev = make_cross_shell_event(0, 5, sid(0x10));
1107        assert!(r.dispatch(&ev, &ctx(5)).is_err());
1108        // Distinct payload (different target shell) at the pinned (5, 0).
1109        let err = r
1110            .dispatch(&make_cross_shell_event(0, 5, sid(0x20)), &ctx(5))
1111            .unwrap_err();
1112        assert!(matches!(
1113            err,
1114            ProjectionError::PositionConflict {
1115                at: ProjectionCursor {
1116                    sequence: 0,
1117                    tick: Tick(5)
1118                }
1119            }
1120        ));
1121        // The genuine retry still proceeds.
1122        assert_eq!(r.dispatch(&ev, &ctx(5)).unwrap(), 1);
1123    }
1124
1125    /// The retry pin covers MID-STREAM failures, not only the first
1126    /// dispatch: after a successful anchor, a failed event must not be
1127    /// lost to a skip-ahead (the new-tick batch-head rule would
1128    /// otherwise accept it) nor replaced by a different same-position
1129    /// event.
1130    #[test]
1131    fn failed_mid_stream_dispatch_pins_the_retry() {
1132        struct FailSecondProjection {
1133            observes: [TypeCode; 1],
1134            cursor: Option<ProjectionCursor>,
1135            calls: usize,
1136        }
1137        impl Projection for FailSecondProjection {
1138            fn observes(&self) -> &[TypeCode] {
1139                &self.observes
1140            }
1141            fn on_event(
1142                &mut self,
1143                event: &EventRecord,
1144                _ctx: &ProjectionContext<'_>,
1145            ) -> Result<(), ProjectionError> {
1146                self.calls += 1;
1147                if self.calls == 2 {
1148                    return Err(ProjectionError::Storage("transient"));
1149                }
1150                self.cursor = Some(ProjectionCursor {
1151                    sequence: event.sequence,
1152                    tick: event.tick,
1153                });
1154                Ok(())
1155            }
1156            fn last_applied(&self) -> Option<(u64, Tick)> {
1157                self.cursor.map(|c| (c.sequence, c.tick))
1158            }
1159        }
1160
1161        let mut r = ProjectionRouter::new();
1162        r.promote_to_active().unwrap();
1163        r.register(Box::new(FailSecondProjection {
1164            observes: [TypeCode(CrossShellActivity::TYPE_CODE)],
1165            cursor: None,
1166            calls: 0,
1167        }));
1168        assert_eq!(
1169            r.dispatch(&make_cross_shell_event(0, 5, sid(0x10)), &ctx(5))
1170                .unwrap(),
1171            1
1172        );
1173        let failed = make_cross_shell_event(0, 6, sid(0x10));
1174        assert!(r.dispatch(&failed, &ctx(6)).is_err());
1175        // Skip-ahead past the failed event is a loud gap (the new-tick
1176        // batch-head rule must not absorb it)...
1177        let err = r
1178            .dispatch(&make_cross_shell_event(0, 7, sid(0x10)), &ctx(7))
1179            .unwrap_err();
1180        assert!(matches!(err, ProjectionError::SequenceGap { .. }));
1181        // ...a different event at the pinned position is a conflict...
1182        let err = r
1183            .dispatch(&make_cross_shell_event(0, 6, sid(0x20)), &ctx(6))
1184            .unwrap_err();
1185        assert!(matches!(err, ProjectionError::PositionConflict { .. }));
1186        // ...and the genuine retry proceeds, after which the stream
1187        // continues normally.
1188        assert_eq!(r.dispatch(&failed, &ctx(6)).unwrap(), 1);
1189        assert_eq!(
1190            r.dispatch(&make_cross_shell_event(0, 7, sid(0x10)), &ctx(7))
1191                .unwrap(),
1192            1
1193        );
1194    }
1195
1196    /// Two successive actions each get a fresh compute whose event sequence
1197    /// restarts at 0. The second `(tick+1, 0)` event is a distinct stream
1198    /// position — it must apply, not vanish as a "duplicate" of the first.
1199    #[test]
1200    fn successive_computes_restarting_sequence_zero_both_apply() {
1201        let applied = Arc::new(AtomicUsize::new(0));
1202        let mut r = ProjectionRouter::new();
1203        r.promote_to_active().unwrap();
1204        r.register(Box::new(CountingProjection::new(applied.clone())));
1205        assert_eq!(
1206            r.dispatch(&make_cross_shell_event(0, 100, sid(0x01)), &ctx(100))
1207                .unwrap(),
1208            1
1209        );
1210        assert_eq!(
1211            r.dispatch(&make_cross_shell_event(0, 101, sid(0x02)), &ctx(101))
1212                .unwrap(),
1213            1,
1214            "second compute's sequence-0 event must not be conflated"
1215        );
1216        assert_eq!(applied.load(Ordering::SeqCst), 2);
1217    }
1218
1219    /// Gap detection is stream-level: an event nobody observes still
1220    /// advances the cursor, so a subsequent same-tick jump is a gap and
1221    /// the contiguous successor applies.
1222    #[test]
1223    fn unobserved_events_advance_stream_cursor() {
1224        let mut r = ProjectionRouter::new();
1225        r.promote_to_active().unwrap();
1226        r.register(Box::new(CrossShellActivityFanout::new()));
1227        let unobserved = EventRecord {
1228            type_code: 0x0003_0F02, // UserErasureScheduled — not observed
1229            sequence: 0,
1230            tick: Tick(5),
1231            payload: Bytes::new(),
1232        };
1233        assert_eq!(r.dispatch(&unobserved, &ctx(5)).unwrap(), 0);
1234        let err = r
1235            .dispatch(&make_cross_shell_event(2, 5, sid(0x10)), &ctx(5))
1236            .unwrap_err();
1237        assert!(matches!(err, ProjectionError::SequenceGap { .. }));
1238        assert_eq!(
1239            r.dispatch(&make_cross_shell_event(1, 5, sid(0x10)), &ctx(5))
1240                .unwrap(),
1241            1
1242        );
1243    }
1244
1245    /// A projection carrying state from a previous run skips events at or
1246    /// before its `last_applied` when a rebuilt router replays the stream.
1247    #[test]
1248    fn rebuilt_router_skips_already_applied_events() {
1249        let applied = Arc::new(AtomicUsize::new(0));
1250        let mut p = CountingProjection::new(applied.clone());
1251        p.on_event(&make_cross_shell_event(0, 10, sid(0x01)), &ctx(10))
1252            .unwrap();
1253        assert_eq!(applied.load(Ordering::SeqCst), 1);
1254
1255        let mut r = ProjectionRouter::new();
1256        r.promote_to_active().unwrap();
1257        r.register(Box::new(p));
1258        assert_eq!(
1259            r.dispatch(&make_cross_shell_event(0, 10, sid(0x01)), &ctx(10))
1260                .unwrap(),
1261            0,
1262            "catch-up redelivery must not double-apply"
1263        );
1264        assert_eq!(applied.load(Ordering::SeqCst), 1);
1265        assert_eq!(
1266            r.dispatch(&make_cross_shell_event(0, 11, sid(0x01)), &ctx(11))
1267                .unwrap(),
1268            1
1269        );
1270        assert_eq!(applied.load(Ordering::SeqCst), 2);
1271    }
1272
1273    /// A projection failure mid-fan-out leaves the cursor unadvanced: the
1274    /// same event redispatches, projections that already applied it are
1275    /// skipped, and the failed projection gets its retry.
1276    #[test]
1277    fn failed_fanout_retry_does_not_double_apply() {
1278        let applied = Arc::new(AtomicUsize::new(0));
1279        let mut r = ProjectionRouter::new();
1280        r.promote_to_active().unwrap();
1281        r.register(Box::new(CountingProjection::new(applied.clone())));
1282        r.register(Box::new(FailOnceProjection::new()));
1283        let ev = make_cross_shell_event(0, 5, sid(0x10));
1284        assert!(r.dispatch(&ev, &ctx(5)).is_err());
1285        assert_eq!(applied.load(Ordering::SeqCst), 1);
1286        assert_eq!(
1287            r.dispatch(&ev, &ctx(5)).unwrap(),
1288            1,
1289            "retry applies only the previously failed projection"
1290        );
1291        assert_eq!(applied.load(Ordering::SeqCst), 1);
1292    }
1293
1294    #[test]
1295    fn draining_rejects_dispatch() {
1296        let mut r = ProjectionRouter::new();
1297        r.begin_draining().unwrap();
1298        let err = r
1299            .dispatch(&make_cross_shell_event(0, 1, sid(0)), &ctx(1))
1300            .unwrap_err();
1301        assert!(matches!(
1302            err,
1303            ProjectionError::NotActive {
1304                state: ObserverState::Draining
1305            }
1306        ));
1307    }
1308
1309    #[test]
1310    fn passive_rejects_dispatch() {
1311        // `ProjectionRouter::new()` starts in Passive — dispatch must reject
1312        // until the worker is promoted, otherwise an active-passive HA pair
1313        // could create split-brain rows in the PG-backed store.
1314        let mut r = ProjectionRouter::new();
1315        assert_eq!(r.state(), ObserverState::Passive);
1316        let err = r
1317            .dispatch(&make_cross_shell_event(0, 1, sid(0)), &ctx(1))
1318            .unwrap_err();
1319        assert!(matches!(
1320            err,
1321            ProjectionError::NotActive {
1322                state: ObserverState::Passive
1323            }
1324        ));
1325    }
1326
1327    #[test]
1328    fn demote_to_passive_blocks_subsequent_dispatch() {
1329        // After a successful Active dispatch, demoting back to Passive must
1330        // immediately stop accepting writes — covers the failover-back path.
1331        let mut r = ProjectionRouter::new();
1332        r.promote_to_active().unwrap();
1333        r.register(Box::new(CrossShellActivityFanout::new()));
1334        r.dispatch(&make_cross_shell_event(0, 5, sid(0x10)), &ctx(5))
1335            .unwrap();
1336        r.demote_to_passive().unwrap();
1337        let err = r
1338            .dispatch(&make_cross_shell_event(1, 6, sid(0x10)), &ctx(6))
1339            .unwrap_err();
1340        assert!(matches!(
1341            err,
1342            ProjectionError::NotActive {
1343                state: ObserverState::Passive
1344            }
1345        ));
1346    }
1347
1348    #[test]
1349    fn in_memory_store_roundtrip_actor() {
1350        let mut store = InMemoryProjectionStore::new();
1351        let row = ActorProjection {
1352            schema_version: 1,
1353            actor_id: ActorId::new(ent(42)),
1354            profile: ActorProfile {
1355                schema_version: 1,
1356                shell_id: sid(0x01),
1357                handle: BoundedString::<32>::new("alice").unwrap(),
1358                kind: ActorKind::Human,
1359                created_tick: Tick(1),
1360            },
1361            user_binding: None,
1362            cursor: None,
1363        };
1364        store.upsert_actor(&row).unwrap();
1365        let fetched = store.get_actor(ActorId::new(ent(42))).unwrap();
1366        assert_eq!(fetched, row);
1367    }
1368
1369    #[test]
1370    fn cross_shell_fanout_preserves_shell_partition() {
1371        let mut fanout = CrossShellActivityFanout::new();
1372        let shell_a = sid(0xAA);
1373        let shell_b = sid(0xBB);
1374        fanout
1375            .on_event(&make_cross_shell_event(0, 10, shell_a), &ctx(10))
1376            .unwrap();
1377        fanout
1378            .on_event(&make_cross_shell_event(1, 11, shell_b), &ctx(11))
1379            .unwrap();
1380        assert_eq!(fanout.notifications_for(&shell_a).len(), 1);
1381        assert_eq!(fanout.notifications_for(&shell_b).len(), 1);
1382        assert_eq!(fanout.last_applied(), Some((1, Tick(11))));
1383    }
1384
1385    #[test]
1386    fn projection_cursor_roundtrip() {
1387        let c = ProjectionCursor {
1388            sequence: 5,
1389            tick: Tick(10),
1390        };
1391        let bytes = postcard::to_stdvec(&c).unwrap();
1392        let back: ProjectionCursor = postcard::from_bytes(&bytes).unwrap();
1393        assert_eq!(c, back);
1394    }
1395
1396    /// Manifest auto-promote decision table.
1397    /// Run via a small helper that builds a `ManifestSnapshot` with a
1398    /// configurable `kms_auto_promote` string.
1399    #[test]
1400    fn auto_promote_policy_matrix() {
1401        use crate::hf2_kms::health::{Channel, MultiChannelHealth, Status};
1402        use crate::manifest::{
1403            AuditSection, FrontendSection, ManifestSnapshot, RuntimeSection, ShellSection,
1404        };
1405        use core::time::Duration;
1406
1407        fn manifest_with(policy: &str) -> ManifestSnapshot {
1408            ManifestSnapshot {
1409                schema_version: 1,
1410                shell: ShellSection {
1411                    shell_id: "test".to_string(),
1412                    display_name: "Test".to_string(),
1413                },
1414                runtime: RuntimeSection {
1415                    runtime_max: "0.15".to_string(),
1416                    runtime_current: "0.13".to_string(),
1417                },
1418                audit: AuditSection {
1419                    pii_cipher: "xchacha20-poly1305".to_string(),
1420                    dek_backend: "software-kek".to_string(),
1421                    kms_auto_promote: policy.to_string(),
1422                    signature_class: "ed25519".to_string(),
1423                    compliance_tier: 0,
1424                },
1425                frontend: FrontendSection::default(),
1426            }
1427        }
1428
1429        fn healthy_trio() -> MultiChannelHealth {
1430            let mut h = MultiChannelHealth::new(&[
1431                Channel::Default,
1432                Channel::DnsOverHttps,
1433                Channel::StaticIp,
1434            ]);
1435            for c in [Channel::Default, Channel::DnsOverHttps, Channel::StaticIp] {
1436                h.set_status(c, Status::Healthy);
1437            }
1438            h
1439        }
1440
1441        fn degraded_trio() -> MultiChannelHealth {
1442            // Only one channel healthy — below 2/3 quorum.
1443            let mut h = MultiChannelHealth::new(&[
1444                Channel::Default,
1445                Channel::DnsOverHttps,
1446                Channel::StaticIp,
1447            ]);
1448            h.set_status(Channel::Default, Status::Healthy);
1449            h.set_status(Channel::DnsOverHttps, Status::Failing);
1450            h.set_status(Channel::StaticIp, Status::Failing);
1451            h
1452        }
1453
1454        let r = ProjectionRouter::new();
1455        let healthy = healthy_trio();
1456        let degraded = degraded_trio();
1457
1458        // Manual policy → always Wait (operator drives the promotion).
1459        assert_eq!(
1460            r.evaluate_auto_promote(
1461                &manifest_with("manual"),
1462                Duration::from_secs(7200),
1463                &healthy,
1464                true,
1465            ),
1466            Some(PromotionDecision::Wait),
1467        );
1468
1469        // after_60min, short outage → Wait even with full health.
1470        assert_eq!(
1471            r.evaluate_auto_promote(
1472                &manifest_with("after_60min"),
1473                Duration::from_secs(59 * 60),
1474                &healthy,
1475                false,
1476            ),
1477            Some(PromotionDecision::Wait),
1478        );
1479
1480        // after_60min, outage cleared but health below quorum → Wait.
1481        assert_eq!(
1482            r.evaluate_auto_promote(
1483                &manifest_with("after_60min"),
1484                Duration::from_secs(60 * 60),
1485                &degraded,
1486                false,
1487            ),
1488            Some(PromotionDecision::Wait),
1489        );
1490
1491        // after_60min, outage cleared AND health quorum met → Promote.
1492        assert_eq!(
1493            r.evaluate_auto_promote(
1494                &manifest_with("after_60min"),
1495                Duration::from_secs(60 * 60),
1496                &healthy,
1497                false,
1498            ),
1499            Some(PromotionDecision::Promote),
1500        );
1501
1502        // threshold_hsm, shares not collected → Wait.
1503        assert_eq!(
1504            r.evaluate_auto_promote(
1505                &manifest_with("threshold_hsm"),
1506                Duration::from_secs(60 * 60),
1507                &degraded,
1508                false,
1509            ),
1510            Some(PromotionDecision::Wait),
1511        );
1512
1513        // threshold_hsm, shares collected → Promote (health is not gating here).
1514        assert_eq!(
1515            r.evaluate_auto_promote(
1516                &manifest_with("threshold_hsm"),
1517                Duration::from_secs(0),
1518                &degraded,
1519                true,
1520            ),
1521            Some(PromotionDecision::Promote),
1522        );
1523
1524        // Unknown policy string → None (conservative default — operator must act).
1525        assert!(r
1526            .evaluate_auto_promote(
1527                &manifest_with("unknown"),
1528                Duration::from_secs(86_400),
1529                &healthy,
1530                true,
1531            )
1532            .is_none());
1533    }
1534}