Skip to main content

liminal_server/server/connection/
services.rs

1use std::collections::HashMap;
2use std::path::{Path, PathBuf};
3use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
4use std::sync::{Arc, Mutex, OnceLock, RwLock};
5use std::time::Instant;
6
7use haematite::{CommitPolicy, Database, DatabaseConfig, EventStore, Retention};
8use liminal::channel::{
9    ChannelConfig, ChannelDelivery, ChannelHandle, ChannelMode, ChannelSupervisor, Schema,
10};
11use liminal::conversation::{ConversationSupervisor, ParticipantBehaviour};
12use liminal::durability::bridge::block_on;
13#[cfg(any(test, feature = "test-support"))]
14use liminal::durability::open_ephemeral;
15use liminal::durability::{
16    CountingStore, DedupCache, DedupDecision, DurableStore, EphemeralRoot, EphemeralSweepCounts,
17    HaematiteStore, ProcessingReceipt, StoreCountsHandle,
18};
19use liminal::protocol::{MessageEnvelope, ProtocolError, SchemaId as ProtocolSchemaId};
20use std::num::NonZeroU64;
21
22use super::channel_registry::{
23    ChannelAccessError, ChannelBuildError, ChannelConfigField, ChannelDescriptor, ChannelOrigin,
24    ChannelRegistration, ChannelRegistryError, ChannelState, ChannelStatus, MAX_CHANNELS_KEY,
25    Registered, STATE_ACTIVE, STATE_QUIESCED, UNRECORDED_QUIESCE_REASON,
26};
27use super::conversation::ConnectionConversation;
28use super::services_cluster::build_channel_cluster;
29use super::services_schema::{ChannelSchema, resolve_channel_schema, resolve_schema_bytes};
30use super::worker_front_door::WorkerFrontDoorServices;
31use crate::ServerError;
32use crate::config::types::{ClusterConfig, ParticipantConfig, ServerConfig, ServiceProfile};
33use crate::config::validation::required_persistence_path;
34use crate::health::reissue::OperatorCredentialReissuer;
35use crate::health::unloadable::UnloadableConversationRecord;
36use crate::server::participant::{
37    InstalledParticipantService, ParticipantWarm, ProductionParticipantHandler,
38};
39use crate::server::self_streams::events::PublishOrigin;
40use crate::server::self_streams::sink::OriginPublication;
41use crate::server::signals::ServerSignals;
42
43pub use super::services_cluster::ChannelCluster;
44
45#[path = "services_conversation.rs"]
46mod services_conversation;
47
48/// Registry of custom conversation responders, keyed by conversation subject.
49///
50/// A registered [`ParticipantBehaviour`] becomes the participant for any
51/// conversation opened on its subject; an unregistered subject has no local server.
52type ResponderRegistry = HashMap<String, Arc<dyn ParticipantBehaviour>>;
53
54/// Marker for resources retained by a connection process until unsubscribe.
55pub trait SubscriptionResource: std::fmt::Debug + Send {
56    /// Releases the library subscription resource.
57    ///
58    /// # Errors
59    /// Returns [`ServerError`] when the liminal library reports an unsubscribe failure.
60    fn unsubscribe(self: Box<Self>) -> Result<(), ServerError>;
61
62    /// Attempts to pull the next delivered envelope from the wrapped library
63    /// subscription without blocking.
64    ///
65    /// Returns `None` when the subscriber inbox is empty (or momentarily
66    /// unavailable): the connection process is the delivery pump, so a transient
67    /// empty read is simply "nothing to deliver this slice", never an error.
68    fn try_next(&mut self) -> Option<liminal::envelope::Envelope>;
69
70    /// Non-consuming availability query for the post-arm race barrier.
71    fn has_pending(&self) -> bool;
72
73    /// Whether an overflow has marked this subscription for shedding (§5). The
74    /// delivery pump sheds an overflowed subscription with a typed error frame.
75    /// Defaulted to `false`: a resource with no bounded inbox never overflows.
76    fn is_overflowed(&self) -> bool {
77        false
78    }
79
80    /// The library subscriber's pid, the id a recording publish names it by.
81    /// Defaulted to `None`: a resource with no library subscriber is never a
82    /// recipient.
83    fn subscriber_pid(&self) -> Option<u64> {
84        None
85    }
86
87    /// Reads a delivery and its durable position without acknowledging it.
88    ///
89    /// # Errors
90    /// The underlying subscription cannot be read.
91    fn try_next_positioned(
92        &mut self,
93    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
94        Ok(self.try_next().map(|envelope| (envelope, None)))
95    }
96
97    /// Checkpoints a delivery after its consumer settles it.
98    ///
99    /// # Errors
100    /// The resource has no durable acknowledgement path, or checkpointing failed.
101    fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
102        Err(ServerError::ListenerAccept {
103            message: format!("subscription cannot acknowledge durable position {position}"),
104        })
105    }
106
107    /// Reads the subscription-owned durable checkpoint.
108    ///
109    /// # Errors
110    /// The checkpoint cannot be read.
111    fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
112        Ok(None)
113    }
114}
115
116/// Library subscription resource owned by a single connection process.
117#[derive(Debug)]
118pub struct ConnectionSubscription {
119    id: u64,
120    /// Channel this subscription was opened on, carried for DIAGNOSTICS only —
121    /// the shed warning has to name the channel or an operator reading it learns
122    /// that something was shed without learning what. Empty when unset (the
123    /// scheduler-free unit fixtures, which construct through `new` and never log).
124    channel: String,
125    /// Client-chosen application stream the server delivers this subscription's
126    /// messages on (echoed on `SubscribeAck`, carried on every `Deliver`). Set by
127    /// the connection process from the `Subscribe` frame before the subscription is
128    /// stored; `0` only while momentarily unset during construction.
129    stream_id: u32,
130    selected_schema: ProtocolSchemaId,
131    resource: Box<dyn SubscriptionResource>,
132    issued: super::state::IssuedDeliveries,
133}
134
135impl ConnectionSubscription {
136    /// Creates an owned subscription resource for one connection process.
137    #[must_use]
138    pub fn new(
139        id: u64,
140        selected_schema: ProtocolSchemaId,
141        resource: Box<dyn SubscriptionResource>,
142    ) -> Self {
143        Self {
144            id,
145            channel: String::new(),
146            stream_id: 0,
147            selected_schema,
148            resource,
149            issued: super::state::IssuedDeliveries::new(0),
150        }
151    }
152
153    /// Records the channel this subscription was opened on (diagnostics).
154    ///
155    /// Additive setter rather than a fourth `new` parameter: `new` is public API
156    /// and its existing callers keep compiling byte-for-byte.
157    pub(super) fn set_channel(&mut self, channel: String) {
158        self.channel = channel;
159    }
160
161    /// The channel this subscription was opened on, or `""` when unset.
162    #[must_use]
163    pub(super) fn channel(&self) -> &str {
164        &self.channel
165    }
166
167    /// Returns the protocol subscription id.
168    #[must_use]
169    pub const fn id(&self) -> u64 {
170        self.id
171    }
172
173    /// Records the client-chosen delivery stream id for this subscription.
174    pub(super) const fn set_stream_id(&mut self, stream_id: u32) {
175        self.stream_id = stream_id;
176    }
177
178    /// Returns the client-chosen application stream id deliveries ride on.
179    #[must_use]
180    pub(super) const fn stream_id(&self) -> u32 {
181        self.stream_id
182    }
183
184    pub(super) fn set_pressure_ledger_depth(&mut self, limit: usize) {
185        self.issued = super::state::IssuedDeliveries::new(limit);
186    }
187
188    pub(super) fn issued_delivery(&mut self, sequence: u64) {
189        self.issued.record(sequence);
190    }
191
192    pub(super) fn match_pressure(&mut self, reference: &str, retain: bool) -> bool {
193        self.issued.matches(reference, retain)
194    }
195
196    /// Returns the schema selected for this subscription stream.
197    #[must_use]
198    pub const fn selected_schema(&self) -> ProtocolSchemaId {
199        self.selected_schema
200    }
201
202    /// The library subscriber's pid, when the subscription has one.
203    #[must_use]
204    pub fn subscriber_pid(&self) -> Option<u64> {
205        self.resource.subscriber_pid()
206    }
207
208    /// Attempts to pull the next delivered envelope without blocking.
209    pub(crate) fn try_next(&mut self) -> Option<liminal::envelope::Envelope> {
210        self.resource.try_next()
211    }
212
213    pub(crate) fn has_pending(&self) -> bool {
214        self.resource.has_pending()
215    }
216
217    /// Whether this subscription has been shed by an inbox overflow (§5).
218    pub(crate) fn is_overflowed(&self) -> bool {
219        self.resource.is_overflowed()
220    }
221
222    pub(super) fn unsubscribe(self) -> Result<(), ServerError> {
223        self.resource.unsubscribe()
224    }
225
226    pub(crate) fn try_next_positioned(
227        &mut self,
228    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
229        self.resource.try_next_positioned()
230    }
231
232    pub(crate) fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
233        self.resource.acknowledge_position(position)
234    }
235
236    pub(crate) fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
237        self.resource.consumer_position()
238    }
239}
240
241/// Outcome of a server publish.
242///
243/// Carries the assigned message id plus a genuine delivery ack (`delivered` = the
244/// message was accepted by at least one live subscriber on this publish, after any
245/// dedup-on-delivery suppression).
246#[derive(Clone, Copy, Debug, PartialEq, Eq)]
247pub struct PublishOutcome {
248    /// Transport-scoped message id assigned to the accepted publish.
249    ///
250    /// TRANSPORT-SCOPED, NOT A DURABLE POSITION (G1,
251    /// `docs/design/wire-durability-gaps-20260831.md`): this is a
252    /// process-local counter (`next_message_id`, reset to 1 on every
253    /// construction), so it restarts from 1 with the server and two servers
254    /// mint the same ids. It is monotonic only within one service instance's
255    /// lifetime and identifies the publish on THIS connection's wire, nothing
256    /// more. The durable position of a publish on a durable channel is the
257    /// store-assigned sequence the library now surfaces as
258    /// `ChannelDelivery::durable_position`; putting that on the wire ack is
259    /// the A1 resume lane's frame decision, not a reinterpretation of this
260    /// field.
261    pub message_id: u64,
262    /// Whether the message was genuinely delivered to a subscriber. `false` means
263    /// the publish was accepted but reached no subscriber (empty channel) or was
264    /// a duplicate suppressed by dedup-on-delivery.
265    pub delivered: bool,
266    /// Store-assigned position of the row this outcome answers for, when the
267    /// publish is known to have one. Unlike `message_id` this IS a durable
268    /// position; the connection process puts it on the wire ack only for a
269    /// connection that negotiated `PUBLISH_ACK_POSITION_VERSION`.
270    pub durable_position: Option<u64>,
271    /// Whom the publish reached, the console tap's word for it (SRV-012
272    /// R10): a subscriber, no subscriber, only subscribers that refused it,
273    /// or nobody because dedup suppressed it.
274    pub reach: Reach,
275}
276
277/// Whom a publish reached (SRV-012 R10).
278#[derive(Clone, Copy, Debug, PartialEq, Eq)]
279pub enum Reach {
280    /// At least one live subscriber took it.
281    Delivered,
282    /// The channel had no matching subscriber.
283    NoSubscriber,
284    /// Every matching subscriber refused it.
285    NoneAccepted,
286    /// A dedup-suppressed re-publish, sent to nobody.
287    Suppressed,
288}
289
290impl Reach {
291    /// The reach of a library delivery.
292    #[must_use]
293    pub const fn of(delivery: &ChannelDelivery) -> Self {
294        if delivery.is_delivered() {
295            Self::Delivered
296        } else if delivery.matched_count() == 0 {
297            Self::NoSubscriber
298        } else {
299            Self::NoneAccepted
300        }
301    }
302}
303
304/// Which channel operation is asking the roster for admission.
305///
306/// `Copy`, so the hot path allocates nothing to ask. The contrast is
307/// [`ServerError::UnsupportedOperation`]'s owned `operation: String`: that one is
308/// built on a refusal path and pays for its allocation once per refusal, while
309/// this one is consulted on every publish and every subscribe frame that reaches
310/// the connection process.
311#[derive(Clone, Copy, Debug, PartialEq, Eq)]
312pub enum ChannelOperation {
313    /// A `Publish` frame is asking.
314    Publish,
315    /// A `Subscribe` frame is asking.
316    Subscribe,
317}
318
319/// Operations that adapt wire frames to liminal library calls.
320pub trait ConnectionServices: std::fmt::Debug + Send + Sync {
321    /// Returns the complete participant service installed on this adapter.
322    ///
323    /// `None` keeps participant capability disabled even when the adapter owns a
324    /// durable store for unrelated channel traffic. The returned token is
325    /// server-sealed and atomically carries declared semantics plus durability,
326    /// making a handler-without-store activation impossible by construction.
327    fn participant_service(&self) -> Option<InstalledParticipantService> {
328        None
329    }
330
331    /// Delegates a publish request to the liminal library.
332    ///
333    /// `connection` is the publishing connection's id (its process pid, as
334    /// `apply_frame` knows it), carried as the publish origin so the console
335    /// tap attributes the message to it (SRV-012). `idempotency_key`, when `Some`, drives dedup-on-delivery: a re-publish with
336    /// the same key is delivered to subscribers at most once. The returned
337    /// [`PublishOutcome`] carries the genuine delivery ack.
338    ///
339    /// # Errors
340    /// Returns [`ServerError`] when the liminal publish operation fails.
341    fn publish(
342        &self,
343        connection: u64,
344        channel: &str,
345        envelope: &MessageEnvelope,
346        idempotency_key: Option<&str>,
347    ) -> Result<PublishOutcome, ServerError>;
348
349    /// Delegates a subscribe request to the liminal library.
350    ///
351    /// `install`, when `Some`, carries the connection's §5 shared inbox byte
352    /// budget, per-inbox fairness cap, and R3 wake notifier. The implementation
353    /// MUST install it on the subscription's inbox BEFORE the registration is
354    /// published to the channel actor (i.e. before any envelope can be
355    /// delivered), so no envelope is ever admitted uncharged, past the depth
356    /// cap, or without a wake. Implementations with no real inbox (test
357    /// stand-ins, capability-scoped profiles that refuse subscribe) may ignore
358    /// it.
359    ///
360    /// # Errors
361    /// Returns [`ServerError`] when the liminal subscribe operation fails.
362    fn subscribe(
363        &self,
364        channel: &str,
365        accepted_schemas: &[ProtocolSchemaId],
366        install: Option<liminal::channel::InboxInstall>,
367    ) -> Result<ConnectionSubscription, ServerError>;
368
369    /// Delegates unsubscribe to the liminal library.
370    ///
371    /// # Errors
372    /// Returns [`ServerError`] when the liminal unsubscribe operation fails.
373    fn unsubscribe(&self, subscription: ConnectionSubscription) -> Result<(), ServerError>;
374
375    /// Delegates conversation open to the liminal library.
376    ///
377    /// # Errors
378    /// Returns [`ServerError`] when the liminal conversation open operation fails.
379    fn open_conversation(
380        &self,
381        conversation_id: u64,
382        subject: &str,
383    ) -> Result<ConnectionConversation, ServerError>;
384
385    /// Delegates a conversation message to the liminal library.
386    ///
387    /// # Errors
388    /// Returns [`ServerError`] when the liminal conversation message operation fails.
389    fn conversation_message(
390        &self,
391        conversation: &ConnectionConversation,
392        envelope: &MessageEnvelope,
393        op_id: Option<u64>,
394    ) -> Result<(), ServerError>;
395
396    /// Delegates conversation close to the liminal library.
397    ///
398    /// # Errors
399    /// Returns [`ServerError`] when the liminal conversation close operation fails.
400    fn close_conversation(&self, conversation: ConnectionConversation) -> Result<(), ServerError>;
401
402    /// Flushes the shared durable store after channel publications have settled.
403    ///
404    /// # Errors
405    /// Returns [`ServerError`] when the shared store flush operation fails.
406    fn flush_durable_state(&self) -> Result<(), ServerError>;
407
408    /// Stops the participant warm and records the clean stop's capacity
409    /// snapshot (DUR-020 R3, R5). Called once, by the shutdown sequence,
410    /// before the final flush; an adapter with no participant does nothing.
411    fn stop_participant_warm(&self) {}
412
413    /// The listener is bound: whatever the start reads that no admission
414    /// needs before it is judged runs from here, behind the listening door.
415    /// An adapter with nothing to read does nothing. Answers the thread it
416    /// started, which the listener leaves to run and a test may join.
417    fn listening(self: Arc<Self>) -> Option<std::thread::JoinHandle<()>> {
418        None
419    }
420
421    /// Whether this adapter backs ordinary channel and conversation operations.
422    ///
423    /// The default is `true` — the full-service adapter serves publish, subscribe,
424    /// and conversation frames, so full mode is byte-for-byte unchanged. The
425    /// capability-scoped worker front door overrides this to `false`, letting
426    /// [`super::apply`] reject the channel/conversation frames it short-circuits on
427    /// empty connection state (`Unsubscribe`, `ConversationMessage`,
428    /// `ConversationClose`) with a typed error frame instead of silently swallowing
429    /// an operation for a resource that could never have been created in this
430    /// profile. Frames that always reach a service method (`Publish`, `Subscribe`,
431    /// `ConversationOpen`) are rejected by the front door's own method bodies and do
432    /// not consult this flag.
433    fn supports_channel_operations(&self) -> bool {
434        true
435    }
436
437    /// Whether `channel` admits `operation` right now.
438    ///
439    /// Consulted by the connection process BEFORE the operation is delegated, so
440    /// a roster refusal is typed at the moment of the decision, by the component
441    /// that made it, from the value it decided on. The alternative — classifying
442    /// an opaque failure afterwards by re-reading the roster in the error arm —
443    /// cannot tell "the roster refused this" from "something else failed while
444    /// the roster happened to change", and would put a confident wrong cause on
445    /// the wire.
446    ///
447    /// It does NOT replace the adapter's own inner check. Admission here is the
448    /// caller's guard; a service method is public and callable without a frame,
449    /// so it keeps its own.
450    ///
451    /// The default ADMITS. An adapter with no roster has nothing to say here and
452    /// its refusals travel as service errors exactly as they do today; only the
453    /// roster-owning adapter overrides this. A default body is also what keeps
454    /// this addition inside "minor": a method added without one breaks every
455    /// downstream implementor of a public trait.
456    ///
457    /// # Errors
458    /// Returns [`ChannelAccessError`] when the roster refuses the operation.
459    fn admit_channel(
460        &self,
461        operation: ChannelOperation,
462        channel: &str,
463    ) -> Result<(), ChannelAccessError> {
464        let _ = (operation, channel);
465        Ok(())
466    }
467}
468
469/// Default adapter from server wire frames to liminal channel/conversation APIs.
470#[derive(Debug)]
471pub struct LiminalConnectionServices {
472    /// The channel roster, keyed by channel name.
473    ///
474    /// Behind an [`RwLock`] over `Arc` values so a reader clones one pointer out
475    /// under the guard and works with it after the guard is released: no roster
476    /// read is ever held across a library call, and an entry handed out can
477    /// outlive a concurrent mutation of the map itself.
478    channels: RwLock<HashMap<String, Arc<ConfiguredChannel>>>,
479    export_subscriptions:
480        Mutex<Vec<std::sync::Weak<crate::link_bridge::subscriptions::ExportSubscriptions>>>,
481    /// The operator-declared bound on RUNTIME-registered channels
482    /// (`limits.max_channels`), carried verbatim from config.
483    ///
484    /// `None` is "the operator declared no bound", which refuses every runtime
485    /// registration rather than admitting an unbounded one — the roster's
486    /// aggregate size is otherwise unbounded the moment a registration API
487    /// exists. Boot-configured channels are never counted against it: they are
488    /// the bound the operator already wrote, in the file they wrote it in.
489    max_channels: Option<usize>,
490    cluster: ChannelCluster,
491    durable_store: Arc<dyn DurableStore>,
492    /// Reads the counters of every store these services built: the durable
493    /// store and, when it is a separate store, the ephemeral dedup store.
494    store_counts: StoreCountsHandle,
495    ephemeral_sweep: Option<EphemeralSweepCounts>,
496    signals: ServerSignals,
497    /// Complete participant service, installed only when semantic lifecycle
498    /// handling and its durable aggregate store are both ready.
499    participant_service: Option<InstalledParticipantService>,
500    /// The production handler's refused-load record, captured at construction.
501    ///
502    /// [`InstalledParticipantService`] holds its handler behind
503    /// `dyn ParticipantSemanticHandler`, so the concrete production handler is
504    /// no longer reachable once it is installed; the record is taken here, in
505    /// the one place the concrete handler exists, and carried out to the
506    /// startup path that publishes it onto the health endpoint. `None` is "no
507    /// participant is configured", which the operator surface reports as such
508    /// rather than as an empty refusal set.
509    unloadable_conversations: Option<UnloadableConversationRecord>,
510    /// The production handler as the A7 operator credential-re-issue authority
511    /// (§0.18), captured at construction for exactly the reason above: the
512    /// concrete handler is unreachable once erased behind
513    /// `dyn ParticipantSemanticHandler`, so the operator surface's handle on it
514    /// has to be taken in the one place it still exists. `None` is "no
515    /// participant is configured", which the route reports as such.
516    credential_reissuer: Option<Arc<dyn OperatorCredentialReissuer>>,
517    /// The participant warm's stop handle (DUR-020 R4, R5), captured at
518    /// construction for the same reason as the two above. The shutdown flush
519    /// stops the warm first, so no conversation begins restoring after the
520    /// server has begun to stop. `None` is "no participant is configured".
521    participant_warm: Option<ParticipantWarm>,
522    /// Dedup-on-delivery cache for publishes to **durable** channels. Keyed by
523    /// the per-message idempotency key carried on the publish frame; a duplicate
524    /// key is suppressed before fan-out so a subscriber receives it at most once.
525    ///
526    /// The namespace is stable across restarts by design (A1 §5): on a durable
527    /// channel the MESSAGE survives a crash too, so a surviving receipt is
528    /// truthful.
529    dedup: DedupCache,
530    /// Dedup-on-delivery cache for publishes to **ephemeral** channels
531    /// (A1 §5, graft §0.2 — the panel's sharpest crash hole).
532    ///
533    /// Identical machinery, different namespace: this one carries a per-boot
534    /// incarnation id, so a receipt and the message it vouches for share a
535    /// lifetime. Without it, `ephemeral + keyed publish -> Defer (buffered in
536    /// memory) -> complete_receipt (persisted) -> crash before the consumer
537    /// pops` loses the message while the receipt survives to suppress the
538    /// producer's retry — silent permanent loss of something the producer was
539    /// told was admitted. With it, the post-crash retry re-claims and
540    /// re-delivers, which is exactly ephemeral's at-least-once contract.
541    ///
542    /// This changes no dedup API. It changes only the namespace string the
543    /// server composes.
544    dedup_ephemeral: DedupCache,
545    /// Durable-channel idempotency keys whose publish outcome is not yet
546    /// known, held until their row is looked up by its key. A cache of the
547    /// durable in-flight claims, rebuilt from them when the services start.
548    held_keys: unresolved::HeldKeys,
549    /// The store history whose in-flight claims are still to be resolved:
550    /// some until the resolve has run, behind the listening door or at the
551    /// first keyed durable admission, whichever comes first (`unresolved_start.rs`).
552    claims_start: Mutex<Option<StoreHistory>>,
553    conversation_supervisor: Arc<ConversationSupervisor>,
554    /// Registered custom conversation responders, keyed by conversation subject.
555    ///
556    /// When a conversation is opened (`open_conversation`), the subject is looked
557    /// up here: a registered [`ParticipantBehaviour`] becomes the conversation's
558    /// participant; with no registration the local route refuses by name.
559    /// This is the seam aion #13 plugs a remote worker responder into. Interior
560    /// mutability is required because the services are shared behind `&self`.
561    responders: Mutex<ResponderRegistry>,
562    next_message_id: AtomicU64,
563    next_subscription_id: AtomicU64,
564    /// SRV-012: the console's held topology and tap.
565    console: Arc<crate::console::ConsoleHub>,
566}
567
568impl LiminalConnectionServices {
569    /// Builds library-backed services from validated server configuration.
570    ///
571    /// Durable-mode channels are backed by a shared haematite event store so
572    /// their publishes are persisted and survive the graceful-shutdown flush;
573    /// ephemeral channels carry no store.
574    ///
575    /// Full-only: a config selecting the worker-front-door profile is rejected at
576    /// entry (before any store is built), so this constructor can never build full
577    /// services for a profile that forbids them. Profile-aware callers go through
578    /// [`build_connection_services`] instead.
579    ///
580    /// # Errors
581    /// Returns [`ServerError`] when the config selects a non-full profile or a
582    /// configured channel cannot be initialized.
583    pub fn from_config(config: &ServerConfig) -> Result<Self, ServerError> {
584        require_full_profile(config)?;
585        let path = required_persistence_path(config)?;
586        let history = StoreHistory::of(path);
587        let subsystems = RootedProductionSubsystems::new(path)?;
588        let store = subsystems.durable_store(path)?;
589        Self::from_config_with_store_via(config, store, history, &subsystems)
590    }
591
592    /// Builds services over a caller-provided durable store.
593    ///
594    /// Used by tests that need to inspect persisted state through the same store
595    /// handle the durable channels write to.
596    ///
597    /// Full-only: rejects a worker-front-door profile at entry, exactly like
598    /// [`Self::from_config`].
599    ///
600    /// # Errors
601    /// Returns [`ServerError`] when the config selects a non-full profile or a
602    /// configured channel cannot be initialized.
603    pub fn from_config_with_store(
604        config: &ServerConfig,
605        durable_store: Arc<dyn DurableStore>,
606    ) -> Result<Self, ServerError> {
607        require_full_profile(config)?;
608        #[cfg(any(test, feature = "test-support"))]
609        if config.persistence_path.is_none() {
610            return Self::from_config_with_store_via(
611                config,
612                durable_store,
613                StoreHistory::Recorded,
614                &ProductionSubsystems,
615            );
616        }
617        let subsystems = RootedProductionSubsystems::new(required_persistence_path(config)?)?;
618        Self::from_config_with_store_via(config, durable_store, StoreHistory::Recorded, &subsystems)
619    }
620
621    /// [`Self::from_config_with_store`] with the subsystem factory injected.
622    ///
623    /// The channel supervisor and conversation supervisor are constructed ONLY
624    /// through `subsystems` — there is no direct constructor call in this body —
625    /// so a factory that records as a side effect of constructing cannot have its
626    /// recording omitted (§9 D2 seam census, record-by-construction). No profile
627    /// check here: the caller (the public wrapper or the profile dispatch in
628    /// [`build_connection_services`]) has already established the full profile.
629    /// `history` says whether `durable_store` can already hold in-flight
630    /// claims for the services to look up before they judge a publish.
631    fn from_config_with_store_via(
632        config: &ServerConfig,
633        durable_store: Arc<dyn DurableStore>,
634        history: StoreHistory,
635        subsystems: &dyn SubsystemFactory,
636    ) -> Result<Self, ServerError> {
637        // Build ONE shared channel supervisor for the whole server. When a
638        // [cluster] section is present it is distribution-enabled, so every
639        // channel actor and subscriber shares the clustered scheduler the cluster
640        // attaches its process-group transport to (SRV-005, Constraint B).
641        let cluster = subsystems.channel_cluster(config.cluster.as_ref())?;
642        let mut channels = HashMap::new();
643        for channel in &config.channels {
644            // Resolve the channel's real JSON Schema (loaded from `schema_ref`
645            // during config validation) or the permissive empty schema when the
646            // channel declared none. The protocol schema id advertised at
647            // subscribe time is derived from the SAME schema bytes so an SDK
648            // deriving ids from schema bytes converges on it.
649            let resolved = resolve_channel_schema(channel);
650            let configured = build_configured_channel(
651                &channel.name,
652                resolved,
653                channel.durable,
654                ChannelOrigin::BootConfigured,
655                &durable_store,
656                cluster.supervisor(),
657            )
658            .map_err(|error| ServerError::ConfigValidation {
659                message: error.boot_message(&channel.name),
660            })?;
661            channels.insert(channel.name.clone(), Arc::new(configured));
662        }
663        let conversation_supervisor = subsystems.conversation_supervisor()?;
664        let dedup = history.delivery_dedup(Arc::clone(&durable_store));
665        // A1 §5: the ephemeral namespace is scoped to THIS incarnation, so an
666        // ephemeral receipt cannot outlive the in-memory message it vouches for.
667        // No earlier process used it, so it starts empty.
668        let ephemeral_store = ephemeral_dedup_store(&durable_store, history, subsystems);
669        let store_counts = combined_store_counts(&durable_store, &ephemeral_store);
670        let dedup_ephemeral = DedupCache::new_empty(ephemeral_store, ephemeral_dedup_namespace());
671        // Production participant activation (LP gap closure, Part B): the
672        // deployment's [participant] section installs the ONE production
673        // semantic handler, sealed together with the same durable store the
674        // conversation logs live in, under the configured wire-frame limit.
675        // No section, no service — the capability bit stays off and the
676        // connection path is byte-identical to the pre-activation build.
677        let signals = ServerSignals::default();
678        let installed_participant = config
679            .participant
680            .as_ref()
681            .map(|participant| install_participant(&durable_store, participant, &signals))
682            .transpose()?;
683        let device = config
684            .cluster
685            .as_ref()
686            .map_or("local", |cluster| cluster.node_name.as_str());
687        let console = Arc::new(console_hub(
688            device,
689            &channels,
690            config.persistence_path.is_some(),
691            config.auth.as_ref().and_then(|auth| auth.lys.as_ref()),
692        )?);
693        signals.install_console(Arc::clone(&console))?;
694        cluster.observer().set_console(Arc::clone(&console))?;
695        let (participant_service, unloadable_conversations, credential_reissuer, participant_warm) =
696            installed_participant.map_or((None, None, None, None), |installed| {
697                (
698                    Some(installed.service),
699                    Some(installed.unloadable_conversations),
700                    Some(installed.credential_reissuer),
701                    Some(installed.warm),
702                )
703            });
704        let services = Self {
705            channels: RwLock::new(channels),
706            export_subscriptions: Mutex::new(Vec::new()),
707            max_channels: config.limits.max_channels,
708            cluster,
709            durable_store,
710            store_counts,
711            ephemeral_sweep: subsystems.ephemeral_sweep_counts(),
712            signals,
713            participant_service,
714            unloadable_conversations,
715            credential_reissuer,
716            participant_warm,
717            dedup,
718            dedup_ephemeral,
719            held_keys: unresolved::HeldKeys::default(),
720            // Every durable in-flight claim is looked up by its key before the
721            // services judge a keyed durable publish: behind the listening
722            // door, or by that publish itself when it comes first.
723            claims_start: Mutex::new(Some(history)),
724            conversation_supervisor,
725            responders: Mutex::new(HashMap::new()),
726            next_message_id: AtomicU64::new(1),
727            next_subscription_id: AtomicU64::new(1),
728            console,
729        };
730        Ok(services)
731    }
732
733    /// Builds services with no configured channels.
734    ///
735    /// # Errors
736    /// Returns [`ServerError`] when the conversation supervisor scheduler cannot start.
737    #[cfg(any(test, feature = "test-support"))]
738    pub fn empty() -> Result<Self, ServerError> {
739        let conversation_supervisor = ProductionSubsystems.conversation_supervisor()?;
740        let store =
741            open_ephemeral(DEFAULT_SHARD_COUNT).map_err(|error| ServerError::ConfigValidation {
742                message: format!("failed to open test durable store: {error}"),
743            })?;
744        let durable_store: Arc<dyn DurableStore> = Arc::new(CountingStore::new(store));
745        let store_counts = combined_store_counts(&durable_store, &durable_store);
746        let dedup = StoreHistory::Fresh.delivery_dedup(Arc::clone(&durable_store));
747        // A1 §5: the ephemeral namespace is scoped to THIS incarnation, so an
748        // ephemeral receipt cannot outlive the in-memory message it vouches for.
749        let dedup_ephemeral =
750            DedupCache::new_empty(Arc::clone(&durable_store), ephemeral_dedup_namespace());
751        let signals = ServerSignals::default();
752        let console = Arc::new(console_hub("local", &HashMap::new(), false, None)?);
753        signals.install_console(Arc::clone(&console))?;
754        let cluster = build_channel_cluster(None)?;
755        cluster.observer().set_console(Arc::clone(&console))?;
756        Ok(Self {
757            channels: RwLock::new(HashMap::new()),
758            export_subscriptions: Mutex::new(Vec::new()),
759            // No config, so no declared bound: this builder serves tests and
760            // callers with no channels at all, and a registration against it
761            // refuses `CapNotConfigured` exactly as an undeclared deployment's
762            // would.
763            max_channels: None,
764            cluster,
765            durable_store,
766            store_counts,
767            ephemeral_sweep: None,
768            signals,
769            participant_service: None,
770            unloadable_conversations: None,
771            credential_reissuer: None,
772            participant_warm: None,
773            dedup,
774            dedup_ephemeral,
775            held_keys: unresolved::HeldKeys::default(),
776            claims_start: Mutex::new(None),
777            conversation_supervisor,
778            responders: Mutex::new(HashMap::new()),
779            next_message_id: AtomicU64::new(1),
780            next_subscription_id: AtomicU64::new(1),
781            console,
782        })
783    }
784
785    /// The retained counts from the root's one startup sweep.
786    #[must_use]
787    pub const fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
788        self.ephemeral_sweep
789    }
790
791    /// The shared channel supervisor + cluster resolver backing this service.
792    ///
793    /// The server runtime uses this to attach the cluster to the channel
794    /// supervisor's clustered scheduler (SRV-005).
795    #[must_use]
796    pub const fn channel_cluster(&self) -> &ChannelCluster {
797        &self.cluster
798    }
799
800    /// Returns the shared durable store backing this service's durable channels.
801    #[must_use]
802    pub fn durable_store(&self) -> Arc<dyn DurableStore> {
803        Arc::clone(&self.durable_store)
804    }
805
806    /// The console's held topology and tap (SRV-012).
807    #[must_use]
808    pub fn console(&self) -> &crate::console::ConsoleHub {
809        &self.console
810    }
811
812    /// Applies a topology change; a poisoned model is counted by the hub
813    /// (`topology_emit_failures`) and logged, never recovered.
814    fn console_changed(&self, change: crate::console::TopologyChange) {
815        self.signals.topology(change);
816    }
817
818    /// Conversation subjects with a registered responder, sorted. Read from
819    /// the in-memory registry; touches no store.
820    ///
821    /// # Errors
822    /// Returns [`ServerError`] when the responder registry lock is poisoned.
823    pub fn responder_subjects(&self) -> Result<Vec<String>, ServerError> {
824        let responders = self.lock_responders()?;
825        let mut subjects: Vec<String> = responders.keys().cloned().collect();
826        drop(responders);
827        subjects.sort();
828        Ok(subjects)
829    }
830
831    /// Restores every participant conversation the warm has not yet reached,
832    /// on the caller's thread, and answers how many refused (DUR-020 R4).
833    /// The store-count journeys call it so their windows do not race the
834    /// warm thread; the server itself never waits for the warm.
835    ///
836    /// # Errors
837    /// Returns [`ServerError`] when the pending-restore lock is poisoned.
838    pub fn finish_participant_warm(&self) -> Result<usize, ServerError> {
839        self.participant_warm.as_ref().map_or(Ok(0), |warm| {
840            warm.finish()
841                .map_err(|error| ServerError::ParticipantStartupRestore {
842                    message: error.to_string(),
843                })
844        })
845    }
846
847    /// Reads the store work these services have done since they were built,
848    /// across every store they built.
849    ///
850    /// Empty (every snapshot zero) when the durable store was supplied by the
851    /// caller and does not count.
852    #[must_use]
853    pub fn store_counts(&self) -> StoreCountsHandle {
854        self.store_counts.clone()
855    }
856
857    /// The production participant handler's refused-load record, when a
858    /// participant is configured.
859    ///
860    /// The server's startup path publishes this onto the health endpoint so
861    /// `GET /unloadable-conversations` answers from the same record the
862    /// handler writes. `None` means no participant is configured at all, which
863    /// the surface reports as a distinct state from "nothing was refused".
864    #[must_use]
865    pub fn unloadable_conversation_record(&self) -> Option<UnloadableConversationRecord> {
866        self.unloadable_conversations.clone()
867    }
868
869    /// The A7 operator credential-re-issue authority, when a participant is
870    /// configured (§0.18).
871    ///
872    /// The startup path publishes this onto the health endpoint so
873    /// `POST /operator/credential-reissue` reaches the SAME serialized
874    /// participant-state point every wire operation does. `None` means no
875    /// participant is configured, which the route reports as a distinct state
876    /// from an unknown identity.
877    #[must_use]
878    pub fn credential_reissuer(&self) -> Option<Arc<dyn OperatorCredentialReissuer>> {
879        self.credential_reissuer.clone()
880    }
881
882    /// Installs a complete participant bundle in full-service supervisor tests.
883    ///
884    /// Production full services intentionally stay disabled until a concrete
885    /// lifecycle handler exists. This consuming test builder exercises the real
886    /// supervisor activation path without allowing an already-shared adapter to
887    /// change capability posture.
888    #[cfg(test)]
889    #[must_use]
890    pub(crate) fn with_participant_service(
891        mut self,
892        participant_service: InstalledParticipantService,
893    ) -> Self {
894        self.participant_service = Some(participant_service);
895        self
896    }
897
898    /// Returns the conversation supervisor backing supervised conversations.
899    ///
900    /// Tests use this to reach the underlying beamr scheduler so they can spawn
901    /// or terminate participant processes and exercise crash detection.
902    #[must_use]
903    pub fn conversation_supervisor(&self) -> Arc<ConversationSupervisor> {
904        Arc::clone(&self.conversation_supervisor)
905    }
906
907    /// Registers a conversation responder for a routing `subject`.
908    ///
909    /// A later conversation on this exact subject runs the registered behaviour
910    /// as a supervised linked participant. A registration replaces the previous
911    /// behaviour and returns it to the caller.
912    ///
913    /// # Errors
914    /// Returns [`ServerError`] when the responder registry lock is poisoned.
915    pub fn register_responder(
916        &self,
917        subject: impl Into<String>,
918        behaviour: Arc<dyn ParticipantBehaviour>,
919    ) -> Result<Option<Arc<dyn ParticipantBehaviour>>, ServerError> {
920        let subject = subject.into();
921        let mut responders = self.lock_responders()?;
922        let previous = responders.insert(subject.clone(), behaviour);
923        drop(responders);
924        self.console_changed(crate::console::TopologyChange::Service {
925            subject,
926            present: true,
927        });
928        Ok(previous)
929    }
930
931    /// Removes the custom responder registered for `subject`, if any.
932    ///
933    /// After removal local calls on the subject are refused with `no_server`.
934    /// Returns the removed behaviour when one was registered.
935    ///
936    /// # Errors
937    /// Returns [`ServerError`] when the responder registry lock is poisoned.
938    pub fn unregister_responder(
939        &self,
940        subject: &str,
941    ) -> Result<Option<Arc<dyn ParticipantBehaviour>>, ServerError> {
942        let mut responders = self.lock_responders()?;
943        let removed = responders.remove(subject);
944        drop(responders);
945        if removed.is_some() {
946            self.console_changed(crate::console::TopologyChange::Service {
947                subject: subject.to_owned(),
948                present: false,
949            });
950        }
951        Ok(removed)
952    }
953
954    /// Locks the responder registry, mapping a poisoned lock to a [`ServerError`]
955    /// rather than panicking (the workspace denies `unwrap`/`expect`/`panic`).
956    fn lock_responders(&self) -> Result<std::sync::MutexGuard<'_, ResponderRegistry>, ServerError> {
957        self.responders
958            .lock()
959            .map_err(|_poisoned| ServerError::ListenerAccept {
960                message: "responder registry lock poisoned".to_owned(),
961            })
962    }
963
964    /// Takes the channel roster's read guard, mapping a poisoned lock to a
965    /// [`ServerError`] rather than panicking (the workspace denies
966    /// `unwrap`/`expect`/`panic`), exactly as [`Self::lock_responders`] does for
967    /// the responder registry.
968    ///
969    /// Every caller clones the `Arc` it needs out of the returned guard and drops
970    /// the guard before doing anything else: the roster lock is never held across
971    /// a call into the liminal library.
972    fn read_channels(
973        &self,
974    ) -> Result<std::sync::RwLockReadGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>, ServerError>
975    {
976        self.channels
977            .read()
978            .map_err(|_poisoned| ServerError::ListenerAccept {
979                message: "channel roster lock poisoned".to_owned(),
980            })
981    }
982
983    /// Subscribes to a configured channel and returns the raw library
984    /// subscription handle so a test can drain the subscriber inbox directly and
985    /// observe exactly which messages reached a subscriber.
986    #[cfg(test)]
987    pub(in crate::server) fn channel_handle_for_test(
988        &self,
989        channel: &str,
990    ) -> Result<ChannelHandle, ServerError> {
991        self.read_channels()?
992            .get(channel)
993            .map(|configured| configured.handle.clone())
994            .ok_or_else(|| ServerError::ListenerAccept {
995                message: format!("channel '{channel}' is not configured"),
996            })
997    }
998
999    #[cfg(test)]
1000    pub(crate) fn subscribe_handle_for_test(
1001        &self,
1002        channel: &str,
1003    ) -> Result<liminal::channel::SubscriptionHandle, ServerError> {
1004        self.subscribe_handle_for_test_with_install(channel, None)
1005    }
1006
1007    /// [`Self::subscribe_handle_for_test`] carrying an explicit install, so a
1008    /// test can put a subscriber under a REAL §5 regime (byte budget, fairness
1009    /// cap) and observe what a publish the bus dropped does to the dedup
1010    /// receipt.
1011    #[cfg(test)]
1012    pub(crate) fn subscribe_handle_for_test_with_install(
1013        &self,
1014        channel: &str,
1015        install: Option<liminal::channel::InboxInstall>,
1016    ) -> Result<liminal::channel::SubscriptionHandle, ServerError> {
1017        let channels = self.read_channels()?;
1018        let configured =
1019            channels
1020                .get(channel)
1021                .map(Arc::clone)
1022                .ok_or_else(|| ServerError::ListenerAccept {
1023                    message: format!("channel '{channel}' is not configured"),
1024                })?;
1025        drop(channels);
1026        install
1027            .map_or_else(
1028                || configured.handle.subscribe(),
1029                |install| configured.handle.subscribe_with_install(install),
1030            )
1031            .map_err(|error| ServerError::ListenerAccept {
1032                message: format!("liminal subscribe failed for channel '{channel}': {error}"),
1033            })
1034    }
1035
1036    /// Claims the delivery right for an idempotency key.
1037    ///
1038    /// Returns `Ok(true)` when this is the first publish for the key (the caller
1039    /// may deliver), and `Ok(false)` when the key was already claimed/completed (a
1040    /// duplicate the caller must suppress). The dedup cache is driven synchronously
1041    /// over the in-memory haematite store via the durable bridge.
1042    fn claim_delivery(cache: &DedupCache, key: &str) -> Result<bool, ServerError> {
1043        let decision = block_on(cache.claim_or_get(key, dedup_timestamp_millis()))
1044            .map_err(|error| ServerError::ListenerAccept {
1045                message: format!("dedup bridge failed for key '{key}': {error}"),
1046            })?
1047            .map_err(|error| ServerError::ListenerAccept {
1048                message: format!("dedup claim failed for key '{key}': {error}"),
1049            })?;
1050        Ok(matches!(decision, DedupDecision::Claimed))
1051    }
1052
1053    /// The dedup namespace a publish to `channel` belongs in (A1 §5).
1054    ///
1055    /// Durability decides, because durability is what determines whether a
1056    /// receipt can outlive the message it vouches for. A durable channel's
1057    /// message survives a crash, so its receipt may; an ephemeral channel's
1058    /// does not, so its receipt must not.
1059    const fn dedup_for(&self, channel: &ConfiguredChannel) -> &DedupCache {
1060        match channel.handle.config().mode {
1061            ChannelMode::Durable => &self.dedup,
1062            ChannelMode::Ephemeral => &self.dedup_ephemeral,
1063        }
1064    }
1065
1066    /// Releases a dangling in-flight dedup claim after a failed delivery.
1067    ///
1068    /// Best-effort: a release failure cannot mask the original publish error, so
1069    /// this returns nothing and logs at `error` level instead of surfacing. It is
1070    /// never silent — the leak (a permanently suppressed key) must be observable.
1071    /// `release_claim` itself never clobbers a stored receipt, so calling it on the
1072    /// failure path is safe even if a concurrent completion raced ahead.
1073    fn release_claim(cache: &DedupCache, key: &str) {
1074        match block_on(cache.release_claim(key)) {
1075            Ok(Ok(())) => {}
1076            Ok(Err(error)) => {
1077                tracing::error!(
1078                    idempotency_key = key,
1079                    %error,
1080                    "failed to release dedup claim after publish failure; key may stay suppressed"
1081                );
1082            }
1083            Err(error) => {
1084                tracing::error!(
1085                    idempotency_key = key,
1086                    %error,
1087                    "dedup release bridge failed after publish failure; key may stay suppressed"
1088                );
1089            }
1090        }
1091    }
1092
1093    /// The next connection-level message id.
1094    fn next_message_id(&self) -> u64 {
1095        self.next_message_id.fetch_add(1, Ordering::Relaxed)
1096    }
1097
1098    /// The outcome of a publish the channel took, recorded as accepted with
1099    /// its genuine subscriber deliveries. The delivered count (0 for an empty
1100    /// channel) is the same signal the delivery ack is derived from.
1101    fn delivered(&self, delivery: &ChannelDelivery) -> PublishOutcome {
1102        crate::metrics::publish_accepted();
1103        let delivered_count = u64::try_from(delivery.delivered_count()).unwrap_or(u64::MAX);
1104        crate::metrics::deliveries_recorded(delivered_count);
1105        PublishOutcome {
1106            message_id: self.next_message_id(),
1107            delivered: delivery.is_delivered(),
1108            durable_position: delivery.durable_position(),
1109            reach: Reach::of(delivery),
1110        }
1111    }
1112}
1113
1114#[path = "deferred_store.rs"]
1115mod deferred_store;
1116#[path = "unresolved.rs"]
1117mod unresolved;
1118#[path = "unresolved_start.rs"]
1119mod unresolved_start;
1120#[cfg(test)]
1121#[path = "unresolved_start_tests.rs"]
1122mod unresolved_start_tests;
1123
1124use deferred_store::{DeferredStore, ThrowawayStoreOpener};
1125use unresolved_start::StoreHistory;
1126
1127/// The runtime channel-registration surface.
1128///
1129/// # Reachability: a test seam today, not a production capability
1130///
1131/// Production channels come exclusively from boot configuration; no production
1132/// code path calls `register_channel` (caller census at `c381f01`, G2 in
1133/// `docs/design/wire-durability-gaps-20260831.md` — the only callers are
1134/// `services_registry_tests.rs` and `apply_admission_tests.rs`). The method is
1135/// kept because its semantics — cap gating, typed identity refusals,
1136/// idempotent re-registration — are the contract a future runtime admission
1137/// path must satisfy, and the tests below pin that contract now. Wiring an
1138/// actual runtime admission route (an operator endpoint, a wire frame) is a
1139/// product decision that has NOT been taken; nothing here implies it. Reading
1140/// this method as an existing operator capability is the false-affordance
1141/// mistake this paragraph exists to prevent.
1142///
1143/// Inherent methods, not trait methods, and deliberately so: this is
1144/// authority-moving vocabulary that belongs to the ONE adapter owning a channel
1145/// roster. Putting `register`/`quiesce` on the public [`ConnectionServices`]
1146/// trait would break every external implementor and would hand
1147/// register/quiesce words to a profile that serves no channels at all.
1148///
1149/// The same seam as the existing runtime-mutation API on this type
1150/// (`register_responder`/`unregister_responder`): `&self`, interior mutability,
1151/// typed `Result`.
1152impl LiminalConnectionServices {
1153    /// Registers `spec` on the live roster.
1154    ///
1155    /// Test seam today: see the reachability note on this `impl` block — no
1156    /// production path reaches this method.
1157    ///
1158    /// Idempotent when an entry of that name already has an IDENTICAL
1159    /// configuration — mode, protocol schema id, and schema document, all three
1160    /// — and refuses typed, naming the first differing field, otherwise. An
1161    /// identical registration against a boot-configured entry answers
1162    /// [`Registered::AlreadyIdentical`] and leaves its origin alone: flipping it
1163    /// would make the entry lie about its restart fate and would move it into
1164    /// the counted population without a channel having been created.
1165    ///
1166    /// The cap is consulted first, in two steps with different reach. An absent
1167    /// `limits.max_channels` refuses EVERY call, identical or not, before the
1168    /// roster is read at all: a deployment that has declared no bound has not
1169    /// said what it admits, and unbounded-by-default is not a bound. A REACHED
1170    /// cap refuses only a call that would create an entry — it gates the insert,
1171    /// which is the population it bounds. An identical re-registration inserts
1172    /// nothing and so is answered `AlreadyIdentical` even at a full roster;
1173    /// refusing it would break idempotency at exactly the boundary a projector
1174    /// re-projecting its record crosses.
1175    ///
1176    /// # Errors
1177    /// Returns [`ChannelRegistryError`] when no cap is configured, the cap is
1178    /// reached, the name exists with a different configuration, the schema bytes
1179    /// do not parse or compile, durable initialization over the shared store
1180    /// fails, or the roster lock is poisoned.
1181    pub fn register_channel(
1182        &self,
1183        spec: &ChannelRegistration,
1184    ) -> Result<Registered, ChannelRegistryError> {
1185        // The cap is consulted before the roster is touched: an undeclared bound
1186        // refuses every runtime registration, identical or not, because the
1187        // deployment has not said what it admits.
1188        let Some(limit) = self.max_channels else {
1189            return Err(ChannelRegistryError::CapNotConfigured {
1190                cap: MAX_CHANNELS_KEY,
1191            });
1192        };
1193        // Schema resolution is pure (a JSON parse and a digest), so it runs with
1194        // no lock held and its result serves BOTH the identity comparison and
1195        // the construction below.
1196        let resolved = resolve_schema_bytes(spec.schema_bytes.as_deref()).map_err(|error| {
1197            ChannelRegistryError::SchemaRejected {
1198                name: spec.name.clone(),
1199                message: error.to_string(),
1200            }
1201        })?;
1202
1203        // Fast paths, under a READ lock: an already-identical registration
1204        // builds nothing, and a full roster refuses before paying for a
1205        // construction it would discard. Neither is the authority — the write
1206        // lock below re-decides both, because the roster can move in between.
1207        let (existing, registered_count) = {
1208            let channels = self.read_roster()?;
1209            let existing = channels.get(&spec.name).map(Arc::clone);
1210            let registered_count = runtime_registered_count(&channels);
1211            drop(channels);
1212            (existing, registered_count)
1213        };
1214        if let Some(existing) = existing {
1215            return compare_registration(&existing, spec, &resolved);
1216        }
1217        if registered_count >= limit {
1218            return Err(ChannelRegistryError::CapReached {
1219                cap: MAX_CHANNELS_KEY,
1220                limit,
1221            });
1222        }
1223
1224        // Construction runs with NO lock held. A durable channel recovers its
1225        // per-partition sequence counters from the store here, which is O(stream
1226        // length) in store reads; holding the roster lock across it would put a
1227        // slow store walk on every connection's publish and subscribe path.
1228        let configured = build_configured_channel(
1229            &spec.name,
1230            resolved,
1231            spec.durable,
1232            ChannelOrigin::RuntimeRegistered,
1233            &self.durable_store,
1234            self.cluster.supervisor(),
1235        )
1236        .map_err(|error| error.into_registry_error(&spec.name))?;
1237
1238        // The authoritative decision: identity, cap, and insert under ONE write
1239        // lock, so the count that admitted the entry and the insert that added
1240        // it cannot be separated by a concurrent registration.
1241        let mut exporters = self.export_subscriptions.lock().map_err(|_| {
1242            ChannelRegistryError::RosterUnavailable {
1243                message: "export subscription registry lock poisoned".to_owned(),
1244            }
1245        })?;
1246        exporters.retain(|exporter| exporter.strong_count() > 0);
1247        let mut channels = self.write_roster()?;
1248        let raced = channels.get(&spec.name).map(Arc::clone);
1249        let registered_count = runtime_registered_count(&channels);
1250        if raced.is_none() && registered_count < limit {
1251            let mut prepared = Vec::new();
1252            for exporter in exporters.iter().filter_map(std::sync::Weak::upgrade) {
1253                if let Some(binding) = exporter.binding(&spec.name) {
1254                    let refused = |error: ServerError| ChannelRegistryError::RosterUnavailable {
1255                        message: format!(
1256                            "export '{}' cannot subscribe to '{}': {error}",
1257                            exporter.link(),
1258                            spec.name
1259                        ),
1260                    };
1261                    let subscription = self
1262                        .build_subscription(
1263                            &spec.name,
1264                            &configured.handle,
1265                            configured.protocol_schema,
1266                            Some(binding.install),
1267                            Some(&binding.consumer),
1268                        )
1269                        .map_err(refused)?;
1270                    let subscription =
1271                        crate::link_bridge::subscriptions::PreparedSubscription::new(
1272                            binding.prefix,
1273                            spec.name.clone(),
1274                            subscription,
1275                        )
1276                        .map_err(refused)?;
1277                    prepared.push((exporter, subscription));
1278                }
1279            }
1280            // Bind every exporter before a publish can find the new roster entry.
1281            for (exporter, subscription) in prepared {
1282                exporter.attach(subscription);
1283            }
1284            channels.insert(spec.name.clone(), Arc::new(configured));
1285            drop(channels);
1286            drop(exporters);
1287            self.console_changed(crate::console::TopologyChange::Channel {
1288                name: spec.name.clone(),
1289                life: line_life(spec.durable),
1290                state: crate::console::records::LineState::Live,
1291            });
1292            return Ok(Registered::Created);
1293        }
1294        drop(channels);
1295        drop(exporters);
1296        if let Some(raced) = raced {
1297            // A racer registered this name while the channel above was being
1298            // built. The built entry is dropped unused: it owns no actor (the
1299            // actor is spawned lazily on first use) and its durable
1300            // construction only READ the store, so discarding it changes
1301            // nothing an observer could see.
1302            return compare_registration(&raced, spec, &schema_of(&configured));
1303        }
1304        Err(ChannelRegistryError::CapReached {
1305            cap: MAX_CHANNELS_KEY,
1306            limit,
1307        })
1308    }
1309
1310    /// Moves `name` from active to quiesced with a named `reason`. ONE-WAY.
1311    ///
1312    /// New publishes and new subscribes are refused afterwards, carrying the
1313    /// reason. Existing subscriptions are UNTOUCHED: nothing revokes a
1314    /// subscription handle, the actor's subscriber list is not walked, no EXIT
1315    /// is sent, and the channel actor keeps running. Quiesce is a roster-level
1316    /// admission decision, not an actor command.
1317    ///
1318    /// Re-quiescing under the IDENTICAL reason is `Ok(())`; a DIFFERENT reason
1319    /// refuses, carrying the reason already on record.
1320    ///
1321    /// The return does NOT mean "no new subscriber can appear". A subscribe that
1322    /// has already passed admission completes and gets its stream — the
1323    /// linearisation point is the admission read, not the subscribe's
1324    /// completion. A consumer that needs "nobody is attached" must observe
1325    /// attachment directly.
1326    ///
1327    /// # Errors
1328    /// Returns [`ChannelRegistryError`] when the name is not registered, is
1329    /// already quiesced under a different reason, or the roster lock is
1330    /// poisoned.
1331    pub fn quiesce_channel(
1332        &self,
1333        name: &str,
1334        reason: impl Into<String>,
1335    ) -> Result<(), ChannelRegistryError> {
1336        let reason = reason.into();
1337        // A READ lock: the state machine lives on the ENTRY, not in the map, so
1338        // the operation the design most wants to be safe never blocks a reader.
1339        let configured = {
1340            let channels = self.read_roster()?;
1341            channels.get(name).map(Arc::clone).ok_or_else(|| {
1342                ChannelRegistryError::NotRegistered {
1343                    name: name.to_owned(),
1344                }
1345            })?
1346        };
1347        configured.quiesce(name, &reason)?;
1348        self.console_changed(crate::console::TopologyChange::Channel {
1349            name: name.to_owned(),
1350            life: line_life(configured.durable),
1351            state: crate::console::records::LineState::Quiesced,
1352        });
1353        Ok(())
1354    }
1355
1356    /// Cheap typed probe: one roster read plus one atomic load.
1357    ///
1358    /// Touches no actor and therefore CANNOT spawn one. Every handle accessor
1359    /// that could answer a question about a channel's activity routes through
1360    /// the lazy-spawn path, so a probe built on one would materialise the actor
1361    /// of the idle channel it was asked about — turning a read into a side
1362    /// effect. This reads the roster entry's own recorded fields and nothing
1363    /// else.
1364    ///
1365    /// # Errors
1366    /// Returns [`ChannelRegistryError::RosterUnavailable`] only.
1367    pub fn channel_status(&self, name: &str) -> Result<ChannelStatus, ChannelRegistryError> {
1368        let configured = {
1369            let channels = self.read_roster()?;
1370            channels.get(name).map(Arc::clone)
1371        };
1372        Ok(
1373            configured.map_or(ChannelStatus::NotRegistered, |configured| {
1374                status_of(&configured)
1375            }),
1376        )
1377    }
1378
1379    /// Every roster entry with its status, sorted by name, from ONE roster
1380    /// read (SRV-012 R7: the console's topology snapshot).
1381    ///
1382    /// # Errors
1383    /// Returns [`ChannelRegistryError::RosterUnavailable`] when the roster
1384    /// lock is poisoned.
1385    pub fn channel_census(&self) -> Result<Vec<(String, ChannelStatus)>, ChannelRegistryError> {
1386        let mut census: Vec<(String, ChannelStatus)> = self
1387            .read_roster()?
1388            .iter()
1389            .map(|(name, configured)| (name.clone(), status_of(configured)))
1390            .collect();
1391        census.sort_by(|left, right| left.0.cmp(&right.0));
1392        Ok(census)
1393    }
1394
1395    /// The whole roster: one minimal descriptor per entry, sorted by name.
1396    ///
1397    /// The census companion to [`Self::channel_status`]. A by-name probe answers
1398    /// about a name the caller already suspects; only an enumeration can reveal
1399    /// a name the caller does not know to ask about, and a verification sweep
1400    /// with no population denominator is an instrument shape this estate
1401    /// forbids. Touches no actor, under the same constraint as the probe.
1402    ///
1403    /// # Errors
1404    /// Returns [`ChannelRegistryError::RosterUnavailable`] only.
1405    pub fn registered_channels(&self) -> Result<Vec<ChannelDescriptor>, ChannelRegistryError> {
1406        let entries: Vec<(String, Arc<ConfiguredChannel>)> = {
1407            let channels = self.read_roster()?;
1408            channels
1409                .iter()
1410                .map(|(name, configured)| (name.clone(), Arc::clone(configured)))
1411                .collect()
1412        };
1413        let mut descriptors: Vec<ChannelDescriptor> = entries
1414            .into_iter()
1415            .map(|(name, configured)| ChannelDescriptor {
1416                name,
1417                origin: configured.origin,
1418                state: configured.state(),
1419            })
1420            .collect();
1421        descriptors.sort_by(|left, right| left.name.cmp(&right.name));
1422        Ok(descriptors)
1423    }
1424
1425    /// The roster admission funnel: the ONE place a channel operation's
1426    /// permission is decided.
1427    ///
1428    /// Returns the admitted entry, so the caller works from the value the
1429    /// decision was made on rather than reading the roster a second time and
1430    /// risking a different answer. This read is the LINEARISATION POINT for the
1431    /// quiesce race: everything the caller does with the returned entry happens
1432    /// outside the lock and is not re-checked, which is exactly why a quiesce
1433    /// that commits after this returns does not stop the operation it admitted.
1434    ///
1435    /// # Errors
1436    /// Returns [`ChannelAccessError`] when the channel is absent, quiesced, or
1437    /// the roster lock is poisoned.
1438    fn admit_channel(&self, channel: &str) -> Result<Arc<ConfiguredChannel>, ChannelAccessError> {
1439        let configured = {
1440            let channels = self.channels.read().map_err(|_poisoned| {
1441                ChannelAccessError::RosterUnavailable {
1442                    message: ROSTER_POISONED.to_owned(),
1443                }
1444            })?;
1445            channels.get(channel).map(Arc::clone)
1446        };
1447        let configured = configured.ok_or_else(|| ChannelAccessError::NotRegistered {
1448            name: channel.to_owned(),
1449        })?;
1450        if configured.state.load(Ordering::Acquire) == STATE_QUIESCED {
1451            return Err(ChannelAccessError::Quiesced {
1452                name: channel.to_owned(),
1453                reason: configured.recorded_quiesce_reason(),
1454            });
1455        }
1456        Ok(configured)
1457    }
1458
1459    /// Takes the roster's READ guard for the registration surface, mapping a
1460    /// poisoned lock to a typed [`ChannelRegistryError`].
1461    ///
1462    /// Separate from [`Self::read_channels`] because the two surfaces answer
1463    /// different callers with different error types; recovering one from the
1464    /// other would mean inspecting a message.
1465    fn read_roster(
1466        &self,
1467    ) -> Result<
1468        std::sync::RwLockReadGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>,
1469        ChannelRegistryError,
1470    > {
1471        #[cfg(test)]
1472        ROSTER_READS.with(|reads| reads.set(reads.get() + 1));
1473        self.channels
1474            .read()
1475            .map_err(|_poisoned| ChannelRegistryError::RosterUnavailable {
1476                message: ROSTER_POISONED.to_owned(),
1477            })
1478    }
1479
1480    /// Takes the roster's WRITE guard — held only for the check-and-insert that
1481    /// must be atomic, never across a call into the liminal library.
1482    fn write_roster(
1483        &self,
1484    ) -> Result<
1485        std::sync::RwLockWriteGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>,
1486        ChannelRegistryError,
1487    > {
1488        self.channels
1489            .write()
1490            .map_err(|_poisoned| ChannelRegistryError::RosterUnavailable {
1491                message: ROSTER_POISONED.to_owned(),
1492            })
1493    }
1494
1495    /// Gives back the tap slot a subscribe to `channel` reserved at
1496    /// admission, taking its tap channel off the roster when no tap stands
1497    /// behind it (SRV-012 R6).
1498    fn give_back_tap(&self, channel: &str) {
1499        self.console.release_tap(channel, |subject| {
1500            crate::console::ServerPublisher::retire_console_channel(self, subject)
1501        });
1502    }
1503
1504    /// The trait's subscribe, from the console guard to the subscription;
1505    /// [`ConnectionServices::subscribe`] gives back the tap reservation when
1506    /// this fails.
1507    fn open_subscription(
1508        &self,
1509        channel: &str,
1510        accepted_schemas: &[ProtocolSchemaId],
1511        install: Option<liminal::channel::InboxInstall>,
1512        consumer: Option<&str>,
1513    ) -> Result<ConnectionSubscription, ServerError> {
1514        // The frame path refused this already at admission; this guard is for
1515        // a caller holding the trait directly.
1516        refuse_console_tap_target(channel).map_err(|error| access_to_server_error(&error))?;
1517        // As in `publish`: the admission funnel clones the entry's `Arc` out
1518        // under the read guard and the guard is released before schema
1519        // negotiation or the actor round-trip below. That release is what makes
1520        // this the linearisation point — a quiesce landing after it does not
1521        // stop the subscription this call is already building.
1522        let configured = self
1523            .admit_channel(channel)
1524            .map_err(|error| access_to_server_error(&error))?;
1525        let selected_schema = if accepted_schemas.is_empty() {
1526            configured.protocol_schema
1527        } else {
1528            liminal::protocol::negotiate_schema(configured.protocol_schema, accepted_schemas)
1529                .map_err(|error| server_error_from_protocol(&error))?
1530        };
1531        self.build_subscription(
1532            channel,
1533            &configured.handle,
1534            selected_schema,
1535            install,
1536            consumer,
1537        )
1538    }
1539
1540    fn build_subscription(
1541        &self,
1542        channel: &str,
1543        handle: &ChannelHandle,
1544        selected_schema: ProtocolSchemaId,
1545        install: Option<liminal::channel::InboxInstall>,
1546        consumer: Option<&str>,
1547    ) -> Result<ConnectionSubscription, ServerError> {
1548        // `subscribe_with_install` installs the §5 budget/fairness cap and the R3
1549        // wake notifier on the inbox at construction — strictly before the
1550        // registration is published to the channel actor — so there is no window
1551        // in which a publish can land uncharged or without a wake.
1552        let subscription = match (consumer, install) {
1553            (Some(consumer), Some(install)) => handle.subscribe_consumer(consumer, install),
1554            (Some(_), None) => {
1555                return Err(ServerError::ListenerAccept {
1556                    message: "a durable consumer requires a bounded inbox install".to_owned(),
1557                });
1558            }
1559            (None, Some(install)) => handle.subscribe_with_install(install),
1560            (None, None) => handle.subscribe(),
1561        }
1562        .map_err(|error| ServerError::ListenerAccept {
1563            message: format!("liminal subscribe failed for channel '{channel}': {error}"),
1564        })?;
1565        let id = self.next_subscription_id.fetch_add(1, Ordering::Relaxed);
1566        let mut connection_subscription = ConnectionSubscription::new(
1567            id,
1568            selected_schema,
1569            Box::new(LiminalSubscriptionResource { subscription }),
1570        );
1571        connection_subscription.set_channel(channel.to_owned());
1572        // SRV-012 R6: the console tap attaches and detaches through the
1573        // supervisor's observer, which hears every subscribe and every
1574        // departure once (ServerObserver).
1575        Ok(connection_subscription)
1576    }
1577
1578    pub(crate) fn subscribe_consumer(
1579        &self,
1580        channel: &str,
1581        consumer: &str,
1582        install: liminal::channel::InboxInstall,
1583    ) -> Result<ConnectionSubscription, ServerError> {
1584        self.open_subscription(channel, &[], Some(install), Some(consumer))
1585    }
1586
1587    pub(crate) fn install_exporter(
1588        &self,
1589        exporter: &Arc<crate::link_bridge::subscriptions::ExportSubscriptions>,
1590    ) -> Result<(), ServerError> {
1591        let mut exporters =
1592            self.export_subscriptions
1593                .lock()
1594                .map_err(|_| ServerError::ListenerAccept {
1595                    message: "export subscription registry lock poisoned".to_owned(),
1596                })?;
1597        exporters.retain(|exporter| exporter.strong_count() > 0);
1598        if exporters
1599            .iter()
1600            .filter_map(std::sync::Weak::upgrade)
1601            .any(|existing| existing.link() == exporter.link())
1602        {
1603            return Err(ServerError::ListenerAccept {
1604                message: format!(
1605                    "link '{}' already owns an export subscription",
1606                    exporter.link()
1607                ),
1608            });
1609        }
1610        let mut prepared = Vec::new();
1611        for channel in self
1612            .registered_channels()
1613            .map_err(|error| ServerError::ListenerAccept {
1614                message: error.to_string(),
1615            })?
1616        {
1617            if matches!(channel.state, ChannelState::Active) {
1618                if let Some(binding) = exporter.binding(&channel.name) {
1619                    let subscription =
1620                        self.subscribe_consumer(&channel.name, &binding.consumer, binding.install)?;
1621                    prepared.push(
1622                        crate::link_bridge::subscriptions::PreparedSubscription::new(
1623                            binding.prefix,
1624                            channel.name,
1625                            subscription,
1626                        )?,
1627                    );
1628                }
1629            }
1630        }
1631        for subscription in prepared {
1632            exporter.attach(subscription);
1633        }
1634        exporters.push(Arc::downgrade(exporter));
1635        drop(exporters);
1636        Ok(())
1637    }
1638
1639    /// Refuses a subscribe that would start one more live tap than
1640    /// [`crate::console::MAX_LIVE_TAPS`] (SRV-012 R6); a subscriber to a tap
1641    /// already live is not a new tap.
1642    ///
1643    /// A poisoned tap table does not refuse the subscribe (R8: a console
1644    /// bookkeeping failure never fails the client operation it watches); the
1645    /// attach that follows meets the poison and counts it.
1646    ///
1647    /// # Errors
1648    /// Returns [`ChannelAccessError::ConsoleTapLimit`] naming the channel.
1649    fn refuse_console_tap_limit(&self, channel: &str) -> Result<(), ChannelAccessError> {
1650        match self.console.reserve_tap(channel) {
1651            Ok(false) => Err(ChannelAccessError::ConsoleTapLimit {
1652                name: channel.to_owned(),
1653                limit: crate::console::MAX_LIVE_TAPS,
1654            }),
1655            Ok(true) | Err(_) => Ok(()),
1656        }
1657    }
1658}
1659
1660impl crate::console::ServerPublisher for LiminalConnectionServices {
1661    /// The server's own record on the one publish path, with a server origin:
1662    /// attributed to `liminal.<device>`, and never counted as a connection's
1663    /// work by the cost tick.
1664    fn publish_console(
1665        &self,
1666        subject: &str,
1667        payload: &[u8],
1668    ) -> Result<(), crate::console::ConsoleError> {
1669        let envelope = MessageEnvelope::new(
1670            ProtocolSchemaId::new([0_u8; ProtocolSchemaId::WIRE_LEN]),
1671            liminal::protocol::CausalContext::independent(),
1672            payload.to_vec(),
1673        );
1674        self.publish_with_origin(
1675            subject,
1676            &envelope,
1677            None,
1678            PublishOrigin::Server {
1679                device: self.console.device(),
1680            },
1681        )
1682        .map(|_published| ())
1683        .map_err(|source| crate::console::ConsoleError::Server {
1684            subject: subject.to_owned(),
1685            source,
1686        })
1687    }
1688
1689    /// Takes a `console.tap.<target>` channel off the roster (SRV-012 R6).
1690    /// Only a console subject is ever removed here.
1691    fn retire_console_channel(&self, subject: &str) -> Result<(), crate::console::ConsoleError> {
1692        if !super::channel_registry::is_console_subject(subject) {
1693            return Ok(());
1694        }
1695        let mut channels =
1696            self.write_roster()
1697                .map_err(|error| crate::console::ConsoleError::Server {
1698                    subject: subject.to_owned(),
1699                    source: ServerError::ListenerAccept {
1700                        message: error.to_string(),
1701                    },
1702                })?;
1703        channels.remove(subject);
1704        drop(channels);
1705        Ok(())
1706    }
1707}
1708
1709/// The production participant service and the operator surfaces taken from
1710/// its one live handler before it is erased behind
1711/// `dyn ParticipantSemanticHandler`.
1712struct InstalledParticipant {
1713    service: InstalledParticipantService,
1714    unloadable_conversations: UnloadableConversationRecord,
1715    credential_reissuer: Arc<dyn OperatorCredentialReissuer>,
1716    warm: ParticipantWarm,
1717}
1718
1719/// Production participant activation (LP gap closure, Part B): the
1720/// deployment's [participant] section installs the ONE production semantic
1721/// handler, sealed together with the same durable store the conversation logs
1722/// live in, under the configured wire-frame limit.
1723///
1724/// # Errors
1725/// Returns [`ServerError::ParticipantStartupRestore`] when the handler cannot
1726/// be built or wired, and [`ServerError::ConfigValidation`] when the
1727/// wire-frame limit is below the protocol's minimum participant frame.
1728fn install_participant(
1729    durable_store: &Arc<dyn DurableStore>,
1730    participant: &ParticipantConfig,
1731    signals: &ServerSignals,
1732) -> Result<InstalledParticipant, ServerError> {
1733    let handler = ProductionParticipantHandler::new(Arc::clone(durable_store), *participant)
1734        .map_err(|error| ServerError::ParticipantStartupRestore {
1735            message: error.to_string(),
1736        })?;
1737    // Taken BEFORE the handler is erased behind
1738    // `dyn ParticipantSemanticHandler`. Construction restores no
1739    // conversation (DUR-020), so the record fills as conversations restore,
1740    // on first touch or in the warm, and is complete once the warm has
1741    // finished.
1742    handler.install_signals(signals.clone()).map_err(|error| {
1743        ServerError::ParticipantStartupRestore {
1744            message: error.to_string(),
1745        }
1746    })?;
1747    let unloadable_conversations = handler.unloadable_record();
1748    let handler = Arc::new(handler);
1749    let warm = handler
1750        .install_warm()
1751        .map_err(|error| ServerError::ParticipantStartupRestore {
1752            message: error.to_string(),
1753        })?;
1754    // A7 (§0.18): taken here for the same reason, and from the same one live
1755    // handler, so the operator surface and the participant wire cannot end up
1756    // talking to two different owners.
1757    let credential_reissuer: Arc<dyn OperatorCredentialReissuer> =
1758        Arc::<ProductionParticipantHandler>::clone(&handler);
1759    let service = InstalledParticipantService::new(
1760        handler,
1761        Arc::clone(durable_store),
1762        participant.wire_frame_limit,
1763    )
1764    .map_err(|error| ServerError::ConfigValidation {
1765        message: format!(
1766            "participant.wire_frame_limit: {} is below the protocol's minimum \
1767             complete participant frame ({error:?})",
1768            participant.wire_frame_limit
1769        ),
1770    })?;
1771    Ok(InstalledParticipant {
1772        service,
1773        unloadable_conversations,
1774        credential_reissuer,
1775        warm,
1776    })
1777}
1778
1779/// The console hub for `device`, its model built from the roster as it
1780/// stands (responders, connections and peers arrive as changes).
1781///
1782/// # Errors
1783/// Returns [`ServerError::ListenerAccept`] when the device is blank or the
1784/// wall clock cannot be read.
1785fn console_hub(
1786    device: &str,
1787    channels: &HashMap<String, Arc<ConfiguredChannel>>,
1788    depot: bool,
1789    lys: Option<&crate::config::types::LysConfig>,
1790) -> Result<crate::console::ConsoleHub, ServerError> {
1791    let this = crate::console::ThisDevice {
1792        id: device.to_owned(),
1793        liminal: env!("CARGO_PKG_VERSION").to_owned(),
1794        started: Instant::now(),
1795        depot,
1796    };
1797    let lines = channels.iter().map(|(name, configured)| {
1798        let state = match configured.state() {
1799            ChannelState::Active => crate::console::records::LineState::Live,
1800            ChannelState::Quiesced { .. } => crate::console::records::LineState::Quiesced,
1801        };
1802        (name.clone(), line_life(configured.durable), state)
1803    });
1804    let mut model = crate::console::TopologyModel::new(this, [], lines, [], []);
1805    if let Some(machine) = lys.and_then(|lys| lys.machine.as_ref()) {
1806        model = model.with_machine(machine);
1807    }
1808    crate::console::wall::WallBase::now()
1809        .and_then(|wall| crate::console::ConsoleHub::new(device, model, wall))
1810        .map_err(|error| ServerError::ListenerAccept {
1811            message: format!("console hub: {error}"),
1812        })
1813}
1814
1815/// A roster entry's status as [`LiminalConnectionServices::channel_status`]
1816/// answers it.
1817fn status_of(configured: &ConfiguredChannel) -> ChannelStatus {
1818    let mode = configured.handle.config().mode;
1819    match configured.state() {
1820        ChannelState::Active => ChannelStatus::Active {
1821            origin: configured.origin,
1822            mode,
1823            schema: configured.protocol_schema,
1824        },
1825        ChannelState::Quiesced { reason } => ChannelStatus::Quiesced {
1826            reason,
1827            origin: configured.origin,
1828            mode,
1829        },
1830    }
1831}
1832
1833#[cfg(test)]
1834thread_local! {
1835    /// Roster read locks taken on this thread. Test-only.
1836    static ROSTER_READS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
1837}
1838
1839/// Roster read locks taken on the calling thread so far. Test-only.
1840#[cfg(test)]
1841pub(crate) fn roster_reads() -> u64 {
1842    ROSTER_READS.with(std::cell::Cell::get)
1843}
1844
1845/// A channel's line life: durable channels keep, ephemeral ones pass.
1846const fn line_life(durable: bool) -> crate::console::records::LineLife {
1847    if durable {
1848        crate::console::records::LineLife::Kept
1849    } else {
1850        crate::console::records::LineLife::Passing
1851    }
1852}
1853
1854/// The diagnostic carried when the roster lock is poisoned. One string, shared
1855/// by every surface that reports it, so the wording cannot drift between them.
1856const ROSTER_POISONED: &str = "channel roster lock poisoned";
1857
1858/// How many roster entries the registration cap counts.
1859///
1860/// Runtime-registered entries ONLY. Boot-configured channels are the operator's
1861/// own authored bound — they are in the file the operator wrote — and counting
1862/// them would make one number mean two different things depending on how the
1863/// deployment was configured. The origin never flips, so this population is
1864/// well defined over time.
1865fn runtime_registered_count(channels: &HashMap<String, Arc<ConfiguredChannel>>) -> usize {
1866    channels
1867        .values()
1868        .filter(|configured| configured.origin == ChannelOrigin::RuntimeRegistered)
1869        .count()
1870}
1871
1872/// The three-field identity comparison behind idempotent-if-identical
1873/// registration.
1874///
1875/// All three must match; the FIRST mismatch is reported by name. The name itself
1876/// is not compared — it is the roster key, a precondition of the comparison
1877/// rather than a member of it — and neither are the supervisor or the durable
1878/// store, which are one server-wide instance each and cannot differ between two
1879/// registrations in one process.
1880///
1881/// The schema is compared as BOTH its protocol id and its parsed document, on
1882/// purpose. The id is a 64-bit non-cryptographic digest, so the document guards
1883/// against a collision accepting a different schema as identical; and the two
1884/// fail in opposite directions, because two byte sequences that parse to the
1885/// same document but differ in whitespace produce different ids — which must
1886/// refuse, since the id is what every future subscriber negotiates.
1887///
1888/// The channel's own `Schema` is not compared: it is not `PartialEq`, and its
1889/// identifier is a fresh value per construction, so comparing it would refuse
1890/// every idempotent re-registration.
1891fn compare_registration(
1892    existing: &ConfiguredChannel,
1893    spec: &ChannelRegistration,
1894    resolved: &ChannelSchema,
1895) -> Result<Registered, ChannelRegistryError> {
1896    let requested_mode = if spec.durable {
1897        ChannelMode::Durable
1898    } else {
1899        ChannelMode::Ephemeral
1900    };
1901    let config = existing.handle.config();
1902    let mismatch = if config.mode == requested_mode {
1903        if existing.protocol_schema == resolved.protocol_id {
1904            if *config.schema.definition() == resolved.document {
1905                None
1906            } else {
1907                Some(ChannelConfigField::SchemaDocument)
1908            }
1909        } else {
1910            Some(ChannelConfigField::SchemaId)
1911        }
1912    } else {
1913        Some(ChannelConfigField::Mode)
1914    };
1915    mismatch.map_or(Ok(Registered::AlreadyIdentical), |field| {
1916        Err(ChannelRegistryError::AlreadyRegistered {
1917            name: spec.name.clone(),
1918            field,
1919        })
1920    })
1921}
1922
1923/// Recovers the resolved schema of an already-built entry, so the write-lock
1924/// re-check compares the SAME three fields against the same values the fast path
1925/// would have.
1926fn schema_of(configured: &ConfiguredChannel) -> ChannelSchema {
1927    ChannelSchema {
1928        document: configured.handle.config().schema.definition().clone(),
1929        protocol_id: configured.protocol_schema,
1930    }
1931}
1932
1933/// Builds one roster entry: the channel's validation engine, its library handle
1934/// (durable or ephemeral), and the protocol schema id advertised at subscribe
1935/// time.
1936///
1937/// This is the SOLE place a [`ConfiguredChannel`] is constructed. Keeping
1938/// construction in one function is what makes "every channel on this server is
1939/// the same kind of object" structural rather than a promise: there is no second
1940/// body a channel could be built by, so no channel can drift into a different
1941/// shape. The boot loop in [`LiminalConnectionServices::from_config_with_store_via`]
1942/// is its caller.
1943///
1944/// `resolved` is consumed because its JSON Schema document is moved into the
1945/// channel's [`Schema`]; the protocol id is carried onto the entry unchanged.
1946/// `origin` is stamped here and never written again.
1947///
1948/// The failure is typed ([`ChannelBuildError`]) rather than a [`ServerError`] so
1949/// BOTH callers can render it without inspecting a message: the boot loop maps
1950/// it back to the exact `ConfigValidation` strings it has always produced, and
1951/// `register_channel` maps it to the registry's own typed variants.
1952///
1953/// # Errors
1954/// Returns [`ChannelBuildError`] when the JSON Schema document does not compile
1955/// or durable initialization over the shared store fails.
1956fn build_configured_channel(
1957    name: &str,
1958    resolved: ChannelSchema,
1959    durable: bool,
1960    origin: ChannelOrigin,
1961    durable_store: &Arc<dyn DurableStore>,
1962    supervisor: &ChannelSupervisor,
1963) -> Result<ConfiguredChannel, ChannelBuildError> {
1964    let schema =
1965        Schema::new(resolved.document).map_err(|error| ChannelBuildError::SchemaRejected {
1966            message: error.to_string(),
1967        })?;
1968    let channel_config = if durable {
1969        ChannelConfig::new(name.to_owned(), schema, ChannelMode::Durable)
1970    } else {
1971        ChannelConfig::new(name.to_owned(), schema, ChannelMode::Ephemeral)
1972    };
1973    let handle = if durable {
1974        ChannelHandle::new_durable_with_supervisor(
1975            channel_config,
1976            Arc::clone(durable_store),
1977            supervisor.clone(),
1978        )
1979        .map_err(|error| ChannelBuildError::DurableInitFailed {
1980            message: error.to_string(),
1981        })?
1982    } else {
1983        ChannelHandle::with_supervisor(channel_config, supervisor.clone())
1984    };
1985    Ok(ConfiguredChannel {
1986        handle,
1987        protocol_schema: resolved.protocol_id,
1988        durable,
1989        origin,
1990        state: AtomicU8::new(STATE_ACTIVE),
1991        quiesce_reason: OnceLock::new(),
1992    })
1993}
1994
1995/// Returns the current epoch-millis timestamp used as the dedup entry anchor.
1996///
1997/// A clock error before the Unix epoch yields `0`: the timestamp is only a TTL
1998/// anchor for the in-memory cache and a zero anchor never breaks the at-most-once
1999/// claim semantics, so this avoids surfacing a clock fault on the publish path.
2000fn dedup_timestamp_millis() -> u64 {
2001    use std::time::{SystemTime, UNIX_EPOCH};
2002    SystemTime::now()
2003        .duration_since(UNIX_EPOCH)
2004        .ok()
2005        .and_then(|duration| u64::try_from(duration.as_millis()).ok())
2006        .unwrap_or(0)
2007}
2008
2009/// Default shard count for an on-disk durable store.
2010///
2011/// Haematite routes keys across this many single-threaded shard actors; a small
2012/// power of two gives parallelism across cursors/streams without spawning an
2013/// actor per core. The value is fixed (haematite has no silent default) and not
2014/// yet surfaced in server config.
2015const DEFAULT_SHARD_COUNT: usize = 8;
2016
2017/// Shard count of the throwaway store an incarnation's ephemeral dedup
2018/// namespace lives in: it holds one namespace, and dies with the process.
2019const THROWAWAY_SHARD_COUNT: usize = 1;
2020
2021/// Namespace prefix for the dedup-on-delivery cache streams. Keeps delivery dedup
2022/// keys from colliding with any other haematite streams in the shared store.
2023///
2024/// This is the **durable** namespace (A1 §5): stable across restarts, because a
2025/// durable channel's message outlives a restart too.
2026const DELIVERY_DEDUP_NAMESPACE: &str = "liminal:delivery-dedup";
2027
2028/// The **ephemeral** dedup namespace for this server incarnation (A1 §5,
2029/// graft §0.2).
2030///
2031/// Scoped to the process incarnation so an ephemeral channel's receipt shares a
2032/// lifetime with the in-memory message it vouches for. Composed from the
2033/// process id and the boot instant, which is enough to make two incarnations
2034/// disagree; nothing reads the components back, so no meaning is attached to
2035/// them beyond distinctness.
2036///
2037/// The namespace lives in [`ephemeral_dedup_store`], never in a store that
2038/// outlives the process, so no incarnation's records are left behind for a
2039/// later one to read past.
2040fn ephemeral_dedup_namespace() -> String {
2041    use std::time::{SystemTime, UNIX_EPOCH};
2042    let boot_nanos = SystemTime::now()
2043        .duration_since(UNIX_EPOCH)
2044        .map_or(0, |since| since.as_nanos());
2045    format!(
2046        "{DELIVERY_DEDUP_NAMESPACE}:incarnation:{}-{boot_nanos}",
2047        std::process::id()
2048    )
2049}
2050
2051/// One reader over the counters of `durable_store` and, when it is a separate
2052/// store, `ephemeral_store`; a store that is the same `Arc` is read once.
2053fn combined_store_counts(
2054    durable_store: &Arc<dyn DurableStore>,
2055    ephemeral_store: &Arc<dyn DurableStore>,
2056) -> StoreCountsHandle {
2057    let durable = durable_store.store_counts().unwrap_or_default();
2058    if Arc::ptr_eq(durable_store, ephemeral_store) {
2059        return durable;
2060    }
2061    durable.with(&ephemeral_store.store_counts().unwrap_or_default())
2062}
2063
2064/// The store an incarnation's ephemeral dedup namespace is written to: the
2065/// services' own store when it was created empty for this process and dies
2066/// with it, otherwise a store of its own that does, opened by the namespace's
2067/// first write rather than at start.
2068///
2069/// Written into a persistent store, every incarnation's namespace would stay
2070/// there for good, under the durable namespace's prefix, with nothing that
2071/// ever reads it again to prune it.
2072fn ephemeral_dedup_store(
2073    durable_store: &Arc<dyn DurableStore>,
2074    history: StoreHistory,
2075    subsystems: &dyn SubsystemFactory,
2076) -> Arc<dyn DurableStore> {
2077    match (history, durable_store) {
2078        #[cfg(any(test, feature = "test-support"))]
2079        (StoreHistory::Fresh, store) => Arc::clone(store),
2080        (StoreHistory::Created | StoreHistory::Recorded, _) => Arc::new(CountingStore::new(
2081            DeferredStore::new(subsystems.throwaway_store()),
2082        )),
2083    }
2084}
2085
2086/// Beamr-scheduler-owning subsystems the full-service construction path builds
2087/// beyond the connection supervisor's own scheduler. The §9 D2 seam census counts
2088/// these: the worker-front-door profile must construct NONE of them. Test-gated:
2089/// this is the recording vocabulary of the gate's instrument, not production state.
2090#[cfg(test)]
2091#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
2092pub(super) enum SchedulerSubsystem {
2093    /// The shared channel supervisor (its own beamr scheduler).
2094    ChannelSupervisor,
2095    /// The conversation supervisor (its own beamr scheduler).
2096    ConversationSupervisor,
2097    /// The haematite store's database (its shard-actor scheduler).
2098    HaematiteStore,
2099}
2100
2101/// Constructor seam for every scheduler-owning subsystem (`SchedulerSubsystem`)
2102/// the profile-aware construction path can create.
2103///
2104/// These methods are the ONLY route through which [`build_connection_services`]
2105/// and the [`LiminalConnectionServices`] config constructors reach
2106/// `build_channel_cluster`, `ConversationSupervisor::new`, and the durable-store
2107/// constructors — no direct constructor call exists in those bodies. The §9 D2
2108/// gate therefore injects a factory that records as a side effect of
2109/// constructing (the D3 store-seam ownership move applied to schedulers): a
2110/// recording cannot be omitted without also failing to construct the subsystem,
2111/// which closes the "hand-placed census call beside the constructor" gap where a
2112/// future subsystem could be built without its courtesy call.
2113pub(super) trait SubsystemFactory {
2114    /// Constructs the shared channel supervisor + cluster resolver (a beamr
2115    /// scheduler).
2116    ///
2117    /// # Errors
2118    /// Returns [`ServerError`] when the channel supervisor scheduler cannot start.
2119    fn channel_cluster(
2120        &self,
2121        cluster_config: Option<&ClusterConfig>,
2122    ) -> Result<ChannelCluster, ServerError>;
2123
2124    /// Constructs the conversation supervisor (a beamr scheduler).
2125    ///
2126    /// # Errors
2127    /// Returns [`ServerError`] when the conversation supervisor scheduler cannot
2128    /// start.
2129    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError>;
2130
2131    /// Constructs the persistent durable store under the explicit directory.
2132    ///
2133    /// # Errors
2134    /// Returns [`ServerError`] when the store cannot be opened.
2135    fn durable_store(&self, persistence_path: &Path) -> Result<Arc<dyn DurableStore>, ServerError>;
2136
2137    /// The opener of the throwaway store an incarnation's ephemeral dedup
2138    /// namespace lives in when the durable store outlives the process. The
2139    /// store is opened by the namespace's first write, never by this call.
2140    fn throwaway_store(&self) -> ThrowawayStoreOpener;
2141
2142    fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
2143        None
2144    }
2145}
2146
2147/// Test construction factory for callers that own their fixture directories.
2148#[cfg(any(test, feature = "test-support"))]
2149pub(super) struct ProductionSubsystems;
2150
2151#[cfg(any(test, feature = "test-support"))]
2152impl SubsystemFactory for ProductionSubsystems {
2153    fn channel_cluster(
2154        &self,
2155        cluster_config: Option<&ClusterConfig>,
2156    ) -> Result<ChannelCluster, ServerError> {
2157        build_channel_cluster(cluster_config)
2158    }
2159
2160    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2161        Ok(Arc::new(ConversationSupervisor::new().map_err(
2162            |error| ServerError::ConfigValidation {
2163                message: format!("failed to start conversation supervisor: {error}"),
2164            },
2165        )?))
2166    }
2167
2168    fn durable_store(&self, persistence_path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2169        build_durable_store(persistence_path)
2170    }
2171
2172    fn throwaway_store(&self) -> ThrowawayStoreOpener {
2173        Box::new(|| open_ephemeral(THROWAWAY_SHARD_COUNT))
2174    }
2175}
2176
2177struct RootedProductionSubsystems {
2178    root: Arc<EphemeralRoot>,
2179}
2180
2181impl RootedProductionSubsystems {
2182    fn new(path: &Path) -> Result<Self, ServerError> {
2183        let root = EphemeralRoot::open(path).map_err(|error| ServerError::ConfigValidation {
2184            message: error.to_string(),
2185        })?;
2186        Ok(Self { root })
2187    }
2188}
2189
2190impl SubsystemFactory for RootedProductionSubsystems {
2191    fn channel_cluster(
2192        &self,
2193        config: Option<&ClusterConfig>,
2194    ) -> Result<ChannelCluster, ServerError> {
2195        build_channel_cluster(config)
2196    }
2197
2198    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2199        Ok(Arc::new(ConversationSupervisor::new().map_err(
2200            |error| ServerError::ConfigValidation {
2201                message: format!("failed to start conversation supervisor: {error}"),
2202            },
2203        )?))
2204    }
2205
2206    fn durable_store(&self, path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2207        build_durable_store(path)
2208    }
2209
2210    fn throwaway_store(&self) -> ThrowawayStoreOpener {
2211        let root = Arc::clone(&self.root);
2212        Box::new(move || root.open_store(THROWAWAY_SHARD_COUNT))
2213    }
2214
2215    fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
2216        Some(self.root.sweep_counts())
2217    }
2218}
2219
2220/// Test-only recording implementation of [`SubsystemFactory`] for the §9 D2
2221/// construction gate. Lives here (not in a test module's private scope) so the
2222/// supervisor-level gate test reuses the same instrument.
2223#[cfg(test)]
2224#[allow(clippy::expect_used)]
2225pub(super) mod subsystem_census {
2226    use std::path::{Path, PathBuf};
2227    use std::sync::{Arc, Mutex};
2228
2229    use liminal::conversation::ConversationSupervisor;
2230    use liminal::durability::{DurableStore, open_ephemeral_rooted};
2231
2232    use super::{
2233        ChannelCluster, ProductionSubsystems, SchedulerSubsystem, SubsystemFactory,
2234        THROWAWAY_SHARD_COUNT, ThrowawayStoreOpener, build_durable_store,
2235    };
2236    use crate::ServerError;
2237    use crate::config::types::ClusterConfig;
2238
2239    /// Records each [`SchedulerSubsystem`] AS A SIDE EFFECT of constructing it,
2240    /// then hands back the production-constructed subsystem (with ephemeral
2241    /// stores rooted in an isolated directory, so the fs half of the gate is a
2242    /// real negative assertion). Because [`SubsystemFactory`] is the only route
2243    /// the profile-aware construction path has to these constructors, a
2244    /// recording cannot be omitted without also failing to construct — the
2245    /// record-by-construction guarantee.
2246    pub struct RecordingSubsystems {
2247        census: Mutex<Vec<SchedulerSubsystem>>,
2248        ephemeral_root: PathBuf,
2249    }
2250
2251    impl RecordingSubsystems {
2252        /// A recording factory whose ephemeral stores live under `ephemeral_root`.
2253        pub fn rooted(ephemeral_root: &Path) -> Self {
2254            Self {
2255                census: Mutex::new(Vec::new()),
2256                ephemeral_root: ephemeral_root.to_path_buf(),
2257            }
2258        }
2259
2260        /// The recorded construction census, sorted for order-independent
2261        /// comparison.
2262        pub fn recorded(&self) -> Vec<SchedulerSubsystem> {
2263            let mut recorded = self
2264                .census
2265                .lock()
2266                .expect("subsystem census lock is never poisoned in tests")
2267                .clone();
2268            recorded.sort();
2269            recorded
2270        }
2271
2272        fn record(&self, subsystem: SchedulerSubsystem) {
2273            self.census
2274                .lock()
2275                .expect("subsystem census lock is never poisoned in tests")
2276                .push(subsystem);
2277        }
2278    }
2279
2280    impl SubsystemFactory for RecordingSubsystems {
2281        fn channel_cluster(
2282            &self,
2283            cluster_config: Option<&ClusterConfig>,
2284        ) -> Result<ChannelCluster, ServerError> {
2285            let cluster = ProductionSubsystems.channel_cluster(cluster_config)?;
2286            self.record(SchedulerSubsystem::ChannelSupervisor);
2287            Ok(cluster)
2288        }
2289
2290        fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2291            let supervisor = ProductionSubsystems.conversation_supervisor()?;
2292            self.record(SchedulerSubsystem::ConversationSupervisor);
2293            Ok(supervisor)
2294        }
2295
2296        fn durable_store(
2297            &self,
2298            persistence_path: &Path,
2299        ) -> Result<Arc<dyn DurableStore>, ServerError> {
2300            let store = build_durable_store(persistence_path)?;
2301            self.record(SchedulerSubsystem::HaematiteStore);
2302            Ok(store)
2303        }
2304
2305        fn throwaway_store(&self) -> ThrowawayStoreOpener {
2306            let root = self.ephemeral_root.clone();
2307            Box::new(move || open_ephemeral_rooted(&root, THROWAWAY_SHARD_COUNT))
2308        }
2309    }
2310}
2311
2312/// Full-only constructor guard: rejects a config whose profile is not `Full`.
2313///
2314/// [`LiminalConnectionServices`]' config-based constructors call this at entry so
2315/// the full service stack can never be built for a worker-front-door config —
2316/// profile enforcement holds on every public construction path, not only the
2317/// file-loading pipeline.
2318fn require_full_profile(config: &ServerConfig) -> Result<(), ServerError> {
2319    match config.services.profile()? {
2320        ServiceProfile::Full => Ok(()),
2321        ServiceProfile::WorkerFrontDoor => Err(ServerError::ConfigValidation {
2322            message: format!(
2323                "services.profile: \"{}\" cannot construct the full LiminalConnectionServices; \
2324                 build profile-selected services via build_connection_services",
2325                ServiceProfile::WORKER_FRONT_DOOR
2326            ),
2327        }),
2328    }
2329}
2330
2331/// Builds the connection-services adapter selected by `config`'s service profile.
2332///
2333/// `Full` builds [`LiminalConnectionServices`] (channels, conversations, durable
2334/// store, dedup cache) exactly as today. `WorkerFrontDoor` builds
2335/// [`WorkerFrontDoorServices`], which constructs none of that machinery. This is the
2336/// single profile-dispatch authority: [`super::supervisor::ConnectionSupervisor`]'s
2337/// config constructor and the standalone runtime's worker arm both route through it
2338/// (the runtime's full arm stays on the explicit
2339/// [`LiminalConnectionServices::from_config`] path because it also needs the shared
2340/// channel cluster, which the trait object does not expose — that path is guarded
2341/// full-only at entry).
2342///
2343/// # Errors
2344/// Returns [`ServerError`] when the selected adapter cannot be constructed, the
2345/// configured profile value is not recognised, or the worker-front-door profile is
2346/// combined with full-only config fields.
2347pub fn build_connection_services(
2348    config: &ServerConfig,
2349) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2350    match config.services.profile()? {
2351        ServiceProfile::Full => {
2352            let subsystems = RootedProductionSubsystems::new(required_persistence_path(config)?)?;
2353            build_connection_services_via(config, &subsystems)
2354        }
2355        ServiceProfile::WorkerFrontDoor => build_worker_front_door(config),
2356    }
2357}
2358
2359/// [`build_connection_services`] with the subsystem factory injected — the §9 D2
2360/// gate seam, both halves at once.
2361///
2362/// Every scheduler-owning subsystem the `Full` branch creates is constructed
2363/// through `subsystems` and nowhere else, so a recording factory observes exactly
2364/// what was built (thread half, record-by-construction); the gate's factory also
2365/// roots its ephemeral stores in an isolated directory, so "the root stays empty
2366/// on the worker branch" is a real negative assertion (fs half, the D3 pattern).
2367/// The `WorkerFrontDoor` branch never touches the factory — a regression that gave
2368/// the front door any subsystem would both record in the census and land a store
2369/// directory in the injected root.
2370///
2371/// The worker branch re-runs the cross-field checks here, not only in file-loading
2372/// validation, so a directly-constructed config cannot smuggle full-only machinery
2373/// past the profile.
2374pub(super) fn build_connection_services_via(
2375    config: &ServerConfig,
2376    subsystems: &dyn SubsystemFactory,
2377) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2378    match config.services.profile()? {
2379        ServiceProfile::Full => {
2380            let path = required_persistence_path(config)?;
2381            let history = StoreHistory::of(path);
2382            let store = subsystems.durable_store(path)?;
2383            Ok(Arc::new(
2384                LiminalConnectionServices::from_config_with_store_via(
2385                    config, store, history, subsystems,
2386                )?,
2387            ))
2388        }
2389        ServiceProfile::WorkerFrontDoor => build_worker_front_door(config),
2390    }
2391}
2392
2393fn build_worker_front_door(
2394    config: &ServerConfig,
2395) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2396    let errors = crate::config::validation::worker_front_door_field_errors(config);
2397    if !errors.is_empty() {
2398        return Err(ServerError::ConfigValidation {
2399            message: errors.join("; "),
2400        });
2401    }
2402    Ok(Arc::new(WorkerFrontDoorServices::new()))
2403}
2404
2405/// Opens the persistent durable store under the configured directory.
2406pub(crate) fn open_persistent_store(
2407    persistence_path: &Path,
2408) -> Result<Arc<dyn DurableStore>, ServerError> {
2409    build_durable_store(persistence_path)
2410}
2411
2412/// Every append is permanent through its WAL frames and one device flush
2413/// before it returns under either commit policy; what the policy decides is
2414/// when the shard folds its staged appends into a new tree version, the fold
2415/// that writes the leaf and every internal node above it as fresh files. Per
2416/// append, one keyed publish left 8 new node files (82,137 bytes) on the J3
2417/// store (NODE-FILES-PER-POST, check 15 on Dean at 68a08b35, 10 October
2418/// 2026), and the live spine reached 2,856,796 files that way. Buffered at
2419/// 1 MiB, Waffles' number (10 October 2026 11:19): the threshold haematite's
2420/// own `LeafLog` runs on, and the most an unclean restart replays from staged
2421/// frames per shard. The store's own reads merge the buffer, so a publish
2422/// reads back at once; a clean stop commits what is staged; no policy carries
2423/// a timer.
2424const SPINE_COMMIT_THRESHOLD_BYTES: NonZeroU64 = match NonZeroU64::new(1 << 20) {
2425    Some(threshold) => threshold,
2426    None => unreachable!(),
2427};
2428
2429/// The commit policy the spine's event store opens under.
2430const SPINE_COMMIT_POLICY: CommitPolicy = CommitPolicy::buffered(SPINE_COMMIT_THRESHOLD_BYTES);
2431
2432fn build_durable_store(path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2433    let data_dir = durable_data_dir(path);
2434    let database = open_or_create_database(&data_dir)?;
2435    let event_store = EventStore::with_commit_policy(database, SPINE_COMMIT_POLICY);
2436    Ok(Arc::new(CountingStore::new(HaematiteStore::new(Arc::new(
2437        event_store,
2438    )))))
2439}
2440
2441/// The directory the persistent database under `persistence_path` lives in.
2442pub(super) fn durable_data_dir(persistence_path: &Path) -> PathBuf {
2443    persistence_path.join("durability")
2444}
2445
2446/// Whether a haematite database already exists at `data_dir`: its
2447/// `config.json` is written when it is created.
2448pub(super) fn database_exists(data_dir: &Path) -> bool {
2449    database_config_file(data_dir).exists()
2450}
2451
2452/// What creating a haematite database leaves in its directory. A shard's
2453/// directory appears beside them the first time anything opens the shard, so
2454/// a database holding nothing else has never been read or written.
2455const UNTOUCHED_DATABASE_ENTRIES: [&str; 2] = ["config.json", "writer.lock"];
2456
2457/// Whether the database at `data_dir` holds only what its creation wrote.
2458/// Anything this cannot read, or does not recognise, answers `false`: the
2459/// database is then taken to hold what an earlier server wrote.
2460pub(super) fn database_untouched(data_dir: &Path) -> bool {
2461    std::fs::read_dir(data_dir).is_ok_and(|entries| {
2462        entries.into_iter().all(|entry| {
2463            entry.is_ok_and(|entry| {
2464                entry
2465                    .file_name()
2466                    .to_str()
2467                    .is_some_and(|name| UNTOUCHED_DATABASE_ENTRIES.contains(&name))
2468            })
2469        })
2470    })
2471}
2472
2473fn database_config_file(data_dir: &Path) -> PathBuf {
2474    data_dir.join("config.json")
2475}
2476
2477/// Every commit writes new copies of the nodes it changes; the durable store
2478/// deletes the copies its committed root no longer reaches while it runs, so it
2479/// holds its live records and not every version ever written. A store created
2480/// under `KeepAll` is moved onto this retention by its next open.
2481pub(super) const DURABLE_RETENTION: Retention = Retention::reclaim(0);
2482
2483/// Opens an existing haematite database at `data_dir`, or creates one.
2484fn open_or_create_database(data_dir: &Path) -> Result<Database, ServerError> {
2485    let config_file = database_config_file(data_dir);
2486    let result = if config_file.exists() {
2487        adopt_missing_config_keys(data_dir, &config_file)?;
2488        Database::open_with_retention(
2489            data_dir,
2490            haematite::RuntimeOptions {
2491                long_wait_interval: std::time::Duration::from_secs(5),
2492                // DUR-021 R1: no pass starts and no step runs until the
2493                // listeners are bound and runtime.rs releases it.
2494                reclaim_start: haematite::ReclaimStart::Held,
2495            },
2496            DURABLE_RETENTION,
2497        )
2498    } else {
2499        Database::create(
2500            DatabaseConfig {
2501                data_dir: data_dir.to_path_buf(),
2502                shard_count: DEFAULT_SHARD_COUNT,
2503                distributed: None,
2504                executor_threads: None,
2505                // haematite 0.8.3 requires this field and refuses `None` at
2506                // validation; `Unlimited` is the crate's explicit spelling of the
2507                // pre-budget (0.8.1) behaviour — the behaviour-preserving choice
2508                // for a dependency hop. A real byte ceiling is a deployment
2509                // decision for its own measured lane. A store whose `config.json`
2510                // predates the field is migrated by the open arm above
2511                // (`adopt_missing_config_keys`) — same value, written once, on
2512                // upgrade.
2513                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
2514                retention: DURABLE_RETENTION,
2515            },
2516            haematite::RuntimeOptions {
2517                long_wait_interval: std::time::Duration::from_secs(5),
2518                // DUR-021 R1: no pass starts and no step runs until the
2519                // listeners are bound and runtime.rs releases it.
2520                reclaim_start: haematite::ReclaimStart::Held,
2521            },
2522        )
2523    };
2524    let database = result.map_err(|error| ServerError::ConfigValidation {
2525        message: format!(
2526            "failed to open durable store at {}: {error}",
2527            data_dir.display()
2528        ),
2529    })?;
2530    liminal::durability::record_store_opened();
2531    super::reclaim_placement::arm_watch(&database);
2532    Ok(database)
2533}
2534
2535/// The stored keys this crate's create arm always writes, with the value it
2536/// writes, for a `config.json` written before haematite had the key.
2537///
2538/// - `node_cache_budget`: haematite 0.8.3 refuses a store without it
2539///   (`MissingNodeCacheBudget`); `"unlimited"` is the pre-budget behaviour.
2540/// - `retention`: `"keep_all"` is the 0.14.1 behaviour, written in so the
2541///   store's stored retention is explicit before the open. The open itself is
2542///   [`Database::open_with_retention`] under [`DURABLE_RETENTION`], which then
2543///   records reclaim in place of it, so an adopted store reclaims from that
2544///   same open onward (spine reclaim, Hermes's ruling of 6 October 2026:
2545///   `past_roots` 0).
2546const ADOPTED_CONFIG_KEYS: [(&str, &str); 2] = [
2547    ("node_cache_budget", "unlimited"),
2548    ("retention", "keep_all"),
2549];
2550
2551/// Writes the values of [`ADOPTED_CONFIG_KEYS`] into a `config.json` that
2552/// lacks any of them, so a store created before haematite had the key opens on
2553/// upgrade with no operator hand. The open that follows then moves its
2554/// retention onto [`DURABLE_RETENTION`].
2555///
2556/// This crate IS the caller that owns these decisions, and it already makes
2557/// them on every create, so answering them here, once, on upgrade, changes
2558/// nothing about how the store runs and takes the file off the operator's
2559/// desk (the store migrates itself or it doesn't ship).
2560///
2561/// Touches the file in exactly one case: a readable JSON object missing one of
2562/// those keys. Anything else — unreadable bytes, a non-object, an explicit
2563/// value however wrong — is left byte-identical so [`Database::open`] surfaces
2564/// haematite's own sharpest refusal instead of this function guessing over
2565/// it. The write is atomic and durable (tmp file beside the target, fsync,
2566/// rename, directory fsync): a crash mid-migration leaves the original bytes,
2567/// and the next boot migrates again.
2568fn adopt_missing_config_keys(data_dir: &Path, config_file: &Path) -> Result<(), ServerError> {
2569    let Ok(bytes) = std::fs::read(config_file) else {
2570        return Ok(());
2571    };
2572    let Ok(mut config) = serde_json::from_slice::<serde_json::Value>(&bytes) else {
2573        return Ok(());
2574    };
2575    let Some(object) = config.as_object_mut() else {
2576        return Ok(());
2577    };
2578    let mut adopted = Vec::new();
2579    for (key, value) in ADOPTED_CONFIG_KEYS {
2580        if !object.contains_key(key) {
2581            object.insert(key.to_owned(), serde_json::Value::String(value.to_owned()));
2582            adopted.push(key);
2583        }
2584    }
2585    if adopted.is_empty() {
2586        return Ok(());
2587    }
2588    let keys = adopted.join(", ");
2589    let migrated =
2590        serde_json::to_vec_pretty(&config).map_err(|error| ServerError::ConfigValidation {
2591            message: format!(
2592                "durable store config at {} predates `{keys}` and its migration could not be \
2593                 serialized: {error}",
2594                config_file.display()
2595            ),
2596        })?;
2597    let write_error = |step: &str, error: std::io::Error| ServerError::ConfigValidation {
2598        message: format!(
2599            "durable store config at {} predates `{keys}` and migrating it failed at {step}: \
2600             {error}",
2601            config_file.display()
2602        ),
2603    };
2604    let tmp_file = data_dir.join("config.json.migrate-tmp");
2605    let mut tmp = std::fs::File::create(&tmp_file)
2606        .map_err(|error| write_error("creating the temporary file", error))?;
2607    std::io::Write::write_all(&mut tmp, &migrated)
2608        .map_err(|error| write_error("writing the temporary file", error))?;
2609    tmp.sync_all()
2610        .map_err(|error| write_error("syncing the temporary file", error))?;
2611    drop(tmp);
2612    std::fs::rename(&tmp_file, config_file)
2613        .map_err(|error| write_error("renaming over config.json", error))?;
2614    std::fs::File::open(data_dir)
2615        .and_then(|dir| dir.sync_all())
2616        .map_err(|error| write_error("syncing the data directory", error))?;
2617    tracing::warn!(
2618        config = %config_file.display(),
2619        keys = %keys,
2620        "durable store config predates these keys -- wrote the create-arm values (the \
2621         behaviour-preserving ones) so the store opens with no operator action"
2622    );
2623    Ok(())
2624}
2625
2626pub(super) fn reserved_namespace_refusal(channel: &str) -> Option<ServerError> {
2627    if channel.starts_with("liminal.") {
2628        crate::metrics::self_stream_reserved_publication();
2629        Some(ServerError::ReservedNamespace {
2630            channel: channel.to_owned(),
2631        })
2632    } else {
2633        None
2634    }
2635}
2636
2637impl LiminalConnectionServices {
2638    pub(in crate::server) const fn signals(&self) -> &ServerSignals {
2639        &self.signals
2640    }
2641
2642    pub(in crate::server) fn self_stream_head(&self, name: &str) -> Result<u64, ServerError> {
2643        let channels = self.read_channels()?;
2644        let channel = channels
2645            .get(name)
2646            .ok_or_else(|| ServerError::ListenerAccept {
2647                message: format!("self stream '{name}' is absent after append"),
2648            })?;
2649        let heads =
2650            channel
2651                .handle
2652                .durable_positions()
2653                .map_err(|error| ServerError::ListenerAccept {
2654                    message: format!("self stream '{name}' head is unavailable: {error}"),
2655                })?;
2656        drop(channels);
2657        match heads.as_slice() {
2658            [head] => Ok(*head),
2659            _ => Err(ServerError::ListenerAccept {
2660                message: format!("self stream '{name}' must have exactly one durable partition"),
2661            }),
2662        }
2663    }
2664
2665    pub(in crate::server) fn publish_self_stream_batch(
2666        &self,
2667        payloads: &[Vec<u8>],
2668        publisher: &str,
2669    ) -> Result<u64, ServerError> {
2670        if publisher.trim().is_empty() || payloads.is_empty() {
2671            return Err(ServerError::ConfigValidation {
2672                message: "self-stream batch requires a publisher and rows".to_owned(),
2673            });
2674        }
2675        let configured = self
2676            .admit_channel("liminal.events")
2677            .map_err(|error| access_to_server_error(&error))?;
2678        let deliveries = configured
2679            .handle
2680            .publish_batch_with_delivery(
2681                payloads,
2682                &liminal::envelope::PublisherId::new(publisher),
2683                None,
2684            )
2685            .map_err(|error| match error {
2686                liminal::error::LiminalError::PublishUnresolved { append } => {
2687                    ServerError::PublishUnresolved {
2688                        channel: "liminal.events".to_owned(),
2689                        idempotency_key: None,
2690                        sequence: Some(append.sequence()),
2691                        message: append.to_string(),
2692                    }
2693                }
2694                error => ServerError::ListenerAccept {
2695                    message: format!("self-stream receipt batch failed: {error}"),
2696                },
2697            })?;
2698        if deliveries.len() != payloads.len()
2699            || deliveries
2700                .iter()
2701                .any(|delivery| delivery.durable_position().is_none())
2702        {
2703            return Err(ServerError::ListenerAccept {
2704                message: "self-stream receipt batch was refused before append".to_owned(),
2705            });
2706        }
2707        for pair in deliveries.windows(2) {
2708            if pair[0]
2709                .durable_position()
2710                .and_then(|position| position.checked_add(1))
2711                != pair[1].durable_position()
2712            {
2713                return Err(ServerError::ListenerAccept {
2714                    message: "self-stream receipt batch positions are not contiguous".to_owned(),
2715                });
2716            }
2717        }
2718        for delivery in &deliveries {
2719            self.delivered(delivery);
2720        }
2721        deliveries
2722            .last()
2723            .and_then(ChannelDelivery::durable_position)
2724            .and_then(|position| position.checked_add(1))
2725            .ok_or_else(|| ServerError::ListenerAccept {
2726                message: "self-stream receipt batch head is unavailable".to_owned(),
2727            })
2728    }
2729
2730    pub(in crate::server) fn checkpoint_self_stream_heads(
2731        &self,
2732        heads: [u64; 2],
2733    ) -> Result<(), ServerError> {
2734        for (name, head) in ["liminal.events:0", "liminal.cost:0"]
2735            .into_iter()
2736            .zip(heads)
2737        {
2738            block_on(liminal::durability::head_checkpoint::advance_head(
2739                self.durable_store.as_ref(),
2740                name,
2741                head,
2742            ))
2743            .map_err(|error| ServerError::ShutdownFlush {
2744                message: format!("self-stream head checkpoint bridge for '{name}' failed: {error}"),
2745            })?
2746            .map_err(|error| ServerError::ShutdownFlush {
2747                message: format!("self-stream head checkpoint for '{name}' failed: {error}"),
2748            })?;
2749        }
2750        Ok(())
2751    }
2752
2753    pub(in crate::server) fn install_self_stream_channels(&self) -> Result<(), ServerError> {
2754        let names = ["liminal.cost", "liminal.events"];
2755        {
2756            let channels = self.read_channels()?;
2757            for name in names {
2758                if channels.contains_key(name) {
2759                    return Err(ServerError::ConfigValidation {
2760                        message: format!("self-stream channel '{name}' is already configured"),
2761                    });
2762                }
2763            }
2764        }
2765        let mut built = Vec::with_capacity(names.len());
2766        for name in names {
2767            let schema =
2768                resolve_schema_bytes(None).map_err(|error| ServerError::ConfigValidation {
2769                    message: format!("self-stream channel '{name}' schema failed: {error}"),
2770                })?;
2771            let channel = build_configured_channel(
2772                name,
2773                schema,
2774                true,
2775                ChannelOrigin::BootConfigured,
2776                &self.durable_store,
2777                self.cluster.supervisor(),
2778            )
2779            .map_err(|error| ServerError::ConfigValidation {
2780                message: error.boot_message(name),
2781            })?;
2782            built.push((name.to_owned(), Arc::new(channel)));
2783        }
2784        let mut channels =
2785            self.channels
2786                .write()
2787                .map_err(|error| ServerError::ConfigValidation {
2788                    message: format!("self-stream channel roster lock poisoned: {error}"),
2789                })?;
2790        for (name, _) in &built {
2791            if channels.contains_key(name) {
2792                return Err(ServerError::ConfigValidation {
2793                    message: format!("self-stream channel '{name}' was concurrently configured"),
2794                });
2795            }
2796        }
2797        channels.extend(built);
2798        drop(channels);
2799        // SRV-012 R7: every roster channel but `console.*` is a line, and
2800        // these join after the hub's model was built from the roster.
2801        for name in names {
2802            self.console_changed(crate::console::TopologyChange::Channel {
2803                name: name.to_owned(),
2804                life: line_life(true),
2805                state: crate::console::records::LineState::Live,
2806            });
2807        }
2808        Ok(())
2809    }
2810
2811    /// SRV-012 R4: registers `console.topology` as an ephemeral,
2812    /// server-owned channel (never a store write). A channel of that name
2813    /// already configured (ephemeral, since config refuses a durable one) is
2814    /// kept as it is.
2815    pub(in crate::server) fn install_console_channels(&self) -> Result<(), ServerError> {
2816        self.register_console_channel(crate::console::TOPOLOGY_SUBJECT)
2817            .map_err(|error| ServerError::ConfigValidation {
2818                message: format!("console channel registration failed: {error}"),
2819            })
2820    }
2821
2822    /// Puts the ephemeral console channel `name` on the roster unless it is
2823    /// there.
2824    fn register_console_channel(&self, name: &str) -> Result<(), ChannelRegistryError> {
2825        if self.read_roster()?.contains_key(name) {
2826            return Ok(());
2827        }
2828        let schema = resolve_schema_bytes(None).map_err(|error| {
2829            ChannelRegistryError::RosterUnavailable {
2830                message: format!("console channel '{name}' schema failed: {error}"),
2831            }
2832        })?;
2833        let channel = build_configured_channel(
2834            name,
2835            schema,
2836            false,
2837            ChannelOrigin::BootConfigured,
2838            &self.durable_store,
2839            self.cluster.supervisor(),
2840        )
2841        .map_err(|error| error.into_registry_error(name))?;
2842        self.write_roster()?
2843            .entry(name.to_owned())
2844            .or_insert_with(|| Arc::new(channel));
2845        Ok(())
2846    }
2847
2848    /// Publishes on an ordinary channel for an application embedding the server.
2849    ///
2850    /// The live and durable publisher identity is liminal.<host>. Reserved self
2851    /// streams and console channels retain the same refusal as connection publishes.
2852    /// An idempotency key has the same delivery semantics as a connection publish.
2853    ///
2854    /// # Errors
2855    ///
2856    /// Returns a configuration error for an empty or whitespace-only host name,
2857    /// a namespace refusal for a reserved channel, or the existing admission,
2858    /// durability, and publication error.
2859    pub fn publish_embedded(
2860        &self,
2861        host: &str,
2862        channel: &str,
2863        envelope: &MessageEnvelope,
2864        idempotency_key: Option<&str>,
2865    ) -> Result<PublishOutcome, ServerError> {
2866        self.publish_with_origin(
2867            channel,
2868            envelope,
2869            idempotency_key,
2870            PublishOrigin::Host { name: host },
2871        )
2872        .map(|published| published.outcome)
2873    }
2874
2875    pub(in crate::server) fn publish_with_origin(
2876        &self,
2877        channel: &str,
2878        envelope: &MessageEnvelope,
2879        idempotency_key: Option<&str>,
2880        origin: PublishOrigin<'_>,
2881    ) -> Result<OriginPublication, ServerError> {
2882        validate_publication_origin(channel, idempotency_key, origin)?;
2883        // SRV-012 to_stations: only while a console tap is live does the
2884        // fan-out record whom the message reached, in a buffer the tap lends
2885        // and takes back; otherwise the publish is the plain one.
2886        let mut recipients = self.signals.lend_recipients();
2887        // Lane brief ruling 6: a tapped message is first seen as the server
2888        // receives it, before admission; the clock is read only while a tap
2889        // lent a record.
2890        let received_at = recipients.is_some().then(Instant::now);
2891        let result = (|| {
2892            // The roster read holds its guard only long enough to clone the entry's
2893            // `Arc` out; everything below — the dedup bridge and the publish into the
2894            // channel actor — runs with no roster lock held.
2895            //
2896            // This inner admission STAYS even once the connection process consults
2897            // the roster ahead of delegating: this method is public and callable
2898            // without going through a frame at all, and a guard that only exists in
2899            // the caller is not a guard.
2900            let configured = self
2901                .admit_channel(channel)
2902                .map_err(|error| access_to_server_error(&error))?;
2903
2904            // Dedup-on-delivery: a publish carrying an idempotency key is delivered to
2905            // subscribers AT MOST ONCE across re-publishes of the same key. Only a
2906            // fresh `Claimed` decision proceeds to fan-out; a `Completed`/`InFlight`
2907            // decision is a duplicate and is suppressed (no second delivery), which is
2908            // the at-most-once guarantee the aion outbox relies on.
2909            // A1 §5: the namespace is chosen by the channel's DURABILITY, so an
2910            // ephemeral receipt cannot outlive the message it vouches for.
2911            // A keyed publish to a durable channel claims its key WITH the channel
2912            // as evidence and holds the claim across an unresolved append
2913            // (`unresolved.rs`); the ephemeral namespace keeps the claim below.
2914            if let (Some(key), ChannelMode::Durable) =
2915                (idempotency_key, configured.handle.config().mode)
2916            {
2917                return self
2918                    .publish_durable_keyed(
2919                        &configured,
2920                        channel,
2921                        (envelope, origin),
2922                        key,
2923                        recipients.as_mut(),
2924                    )
2925                    .map(OriginPublication::connection);
2926            }
2927            let dedup = self.dedup_for(&configured);
2928            if let Some(key) = idempotency_key {
2929                if !Self::claim_delivery(dedup, key)? {
2930                    // A dedup-suppressed re-publish is still an accepted publish (it is
2931                    // assigned a message id), but it reaches no subscriber, so it counts
2932                    // toward publishes and not deliveries.
2933                    crate::metrics::publish_accepted();
2934                    return Ok(OriginPublication::connection(PublishOutcome {
2935                        message_id: self.next_message_id(),
2936                        delivered: false,
2937                        durable_position: None,
2938                        reach: Reach::Suppressed,
2939                    }));
2940                }
2941            }
2942
2943            let delivery = fan_out(&configured, envelope, origin, recipients.as_mut());
2944            let delivery = match delivery {
2945                Ok(delivery) => delivery,
2946                // An unkeyed durable publish whose append outcome is not known is
2947                // answered as unresolved, never as a definite failure a producer
2948                // would retry into a second row.
2949                Err(liminal::error::LiminalError::PublishUnresolved { append }) => {
2950                    return Err(ServerError::PublishUnresolved {
2951                        channel: channel.to_owned(),
2952                        idempotency_key: None,
2953                        sequence: Some(append.sequence()),
2954                        message: append.to_string(),
2955                    });
2956                }
2957                Err(error) => {
2958                    // The claim above appended an `InFlight` entry but the delivery
2959                    // failed before `complete_receipt` could run. Release the claim so
2960                    // the key is re-claimable; otherwise every re-publish would see
2961                    // `InFlight` and be suppressed forever. Best-effort: surface the
2962                    // ORIGINAL publish error regardless, but never swallow a release
2963                    // failure silently (it leaves the leak intact).
2964                    if let Some(key) = idempotency_key {
2965                        Self::release_claim(dedup, key);
2966                    }
2967                    return Err(ServerError::ListenerAccept {
2968                        message: format!("liminal publish failed for channel '{channel}': {error}"),
2969                    });
2970                }
2971            };
2972
2973            // A1 §5, the two-way dedup mapping. Every publish outcome maps to
2974            // EXACTLY ONE of complete/release — the same shape as the C1 contract:
2975            //
2976            // * **Defer (or Accept) => `complete_receipt`.** The message is
2977            //   admitted and bus-held, so the key is done: a producer retrying it
2978            //   hits `Completed` and there is no duplicate delivery and no InFlight
2979            //   leak. This is the path that makes "re-publishing with the same key
2980            //   is safe and is suppressed while the bus owns the message" true.
2981            // * **Reject => `release_claim`.** Aggregation makes a producer-visible
2982            //   Reject mean delivered-to-zero, so the correct dedup state is "as if
2983            //   never published": tombstone it so a deliberate retry can re-claim.
2984            //   `release_claim` never clobbers a receipt, so the at-most-once guard
2985            //   survives a concurrent completion racing ahead.
2986            //
2987            // Mixed accept/reject reports Defer and so completes — consistent with
2988            // "at least one delivery happened".
2989            if let Some(key) = idempotency_key {
2990                if delivery.is_admitted() {
2991                    block_on(dedup.complete_receipt(key, ProcessingReceipt::new(Vec::new())))
2992                        .map_err(|error| ServerError::ListenerAccept {
2993                            message: format!(
2994                                "dedup receipt bridge failed for key '{key}': {error}"
2995                            ),
2996                        })?
2997                        .map_err(|error| ServerError::ListenerAccept {
2998                            message: format!("dedup receipt write failed for key '{key}': {error}"),
2999                        })?;
3000                } else {
3001                    Self::release_claim(dedup, key);
3002                }
3003            }
3004
3005            let self_row_at = delivery.durable_position().filter(|_| origin.is_server());
3006            Ok(OriginPublication {
3007                outcome: self.delivered(&delivery),
3008                self_row_appended: self_row_at.is_some(),
3009                self_row_at,
3010            })
3011        })();
3012        self.signals.published(
3013            channel,
3014            origin,
3015            envelope,
3016            &result,
3017            recipients.as_ref().zip(received_at),
3018        );
3019        if let Some(buffer) = recipients {
3020            self.signals.give_back(buffer);
3021        }
3022        result
3023    }
3024}
3025
3026/// The library publish of an unkeyed message, recording what it saw in
3027/// `recipients` when the tap lent a record.
3028fn fan_out(
3029    configured: &ConfiguredChannel,
3030    envelope: &MessageEnvelope,
3031    origin: PublishOrigin<'_>,
3032    recipients: Option<&mut liminal::channel::PublishRecord>,
3033) -> Result<liminal::channel::ChannelDelivery, liminal::error::LiminalError> {
3034    let publisher = || publisher_for_origin(origin);
3035    recipients.map_or_else(
3036        || {
3037            configured
3038                .handle
3039                .publish_with_delivery(&envelope.payload, publisher(), None)
3040        },
3041        |recipients| {
3042            configured.handle.publish_recording(
3043                &envelope.payload,
3044                publisher(),
3045                None,
3046                None,
3047                recipients,
3048            )
3049        },
3050    )
3051}
3052
3053fn validate_publication_origin(
3054    channel: &str,
3055    idempotency_key: Option<&str>,
3056    origin: PublishOrigin<'_>,
3057) -> Result<(), ServerError> {
3058    if !origin.is_server() {
3059        refuse_console_namespace(channel).map_err(|error| access_to_server_error(&error))?;
3060        if let Some(error) = reserved_namespace_refusal(channel) {
3061            return Err(error);
3062        }
3063    }
3064    if let PublishOrigin::Host { name } = origin
3065        && name.trim().is_empty()
3066    {
3067        return Err(ServerError::ConfigValidation {
3068            message: "embedded publication requires a nonempty host name".to_owned(),
3069        });
3070    }
3071    if let PublishOrigin::Server { device } = origin {
3072        if device.trim().is_empty() || idempotency_key.is_some() {
3073            return Err(ServerError::ConfigValidation {
3074                message: "server-origin publication requires a device and no idempotency key"
3075                    .to_owned(),
3076            });
3077        }
3078    }
3079    Ok(())
3080}
3081
3082fn publisher_for_origin(origin: PublishOrigin<'_>) -> liminal::envelope::PublisherId {
3083    match origin {
3084        PublishOrigin::Connection { .. } => liminal::envelope::PublisherId::default(),
3085        PublishOrigin::Host { name } => {
3086            liminal::envelope::PublisherId::new(format!("liminal.{name}"))
3087        }
3088        PublishOrigin::Server { device } => {
3089            liminal::envelope::PublisherId::new(format!("liminal.{device}"))
3090        }
3091    }
3092}
3093
3094impl ConnectionServices for LiminalConnectionServices {
3095    fn participant_service(&self) -> Option<InstalledParticipantService> {
3096        self.participant_service.clone()
3097    }
3098
3099    /// The roster's admission decision, exposed across the trait boundary as the
3100    /// permission alone.
3101    ///
3102    /// Two methods share the name `admit_channel` on this type: this trait
3103    /// method, and the private inherent funnel it delegates to. They are the
3104    /// same decision at two different boundaries — the funnel returns the ENTRY
3105    /// the decision was made on, because its in-crate callers go on to publish
3106    /// or subscribe through it, and this one returns `()`, because a caller
3107    /// outside the crate is asking whether it MAY, not for the thing itself.
3108    /// Handing the entry out here would make [`ConfiguredChannel`] public
3109    /// surface and hand a frame-level caller a channel handle it has no business
3110    /// holding. Rust resolves an inherent method ahead of a trait method of the
3111    /// same name, so the funnel's existing callers — and the `Self::` call below
3112    /// — reach the funnel, not this method; the trait form is reached only
3113    /// through a `dyn ConnectionServices`, which is exactly the caller it exists
3114    /// for. `an_admitted_operation_the_service_then_refuses_stays_undifferentiated`
3115    /// is the instrument that would catch this delegation turning into a
3116    /// recursion.
3117    ///
3118    /// `operation` is where the answers part: absence and quiesce refuse
3119    /// publishes and subscribes alike, but the console namespace refuses only
3120    /// a connection's publish (a console reads `console.*` as an ordinary
3121    /// subscriber, and only the server may write there), and a tap target
3122    /// refuses only a subscribe. Both are checked before the roster, so a
3123    /// client is told the same thing whether or not the subject is registered
3124    /// yet.
3125    fn admit_channel(
3126        &self,
3127        operation: ChannelOperation,
3128        channel: &str,
3129    ) -> Result<(), ChannelAccessError> {
3130        match operation {
3131            ChannelOperation::Publish => refuse_console_namespace(channel)?,
3132            ChannelOperation::Subscribe => {
3133                refuse_console_tap_target(channel)?;
3134                self.refuse_console_tap_limit(channel)?;
3135                // SRV-012 R4/R6: a tap channel exists while it has a
3136                // subscriber, and only while the console streams run.
3137                let admitted = if self.signals.console_enabled()
3138                    && super::channel_registry::console_tap_target(channel).is_some()
3139                {
3140                    self.register_console_channel(channel).map_err(|error| {
3141                        ChannelAccessError::RosterUnavailable {
3142                            message: error.to_string(),
3143                        }
3144                    })
3145                } else {
3146                    Ok(())
3147                }
3148                .and_then(|()| Self::admit_channel(self, channel).map(|_| ()));
3149                if admitted.is_err() {
3150                    self.give_back_tap(channel);
3151                }
3152                return admitted;
3153            }
3154        }
3155        Self::admit_channel(self, channel)?;
3156        Ok(())
3157    }
3158
3159    fn publish(
3160        &self,
3161        connection: u64,
3162        channel: &str,
3163        envelope: &MessageEnvelope,
3164        idempotency_key: Option<&str>,
3165    ) -> Result<PublishOutcome, ServerError> {
3166        // The frame path refused this already at admission. This guard is for
3167        // a caller holding the trait directly: a guard that only exists in the
3168        // caller is not a guard.
3169        refuse_console_namespace(channel).map_err(|error| access_to_server_error(&error))?;
3170        self.publish_with_origin(
3171            channel,
3172            envelope,
3173            idempotency_key,
3174            PublishOrigin::Connection { id: connection },
3175        )
3176        .map(|published| published.outcome)
3177    }
3178
3179    fn subscribe(
3180        &self,
3181        channel: &str,
3182        accepted_schemas: &[ProtocolSchemaId],
3183        install: Option<liminal::channel::InboxInstall>,
3184    ) -> Result<ConnectionSubscription, ServerError> {
3185        let subscribed = self.open_subscription(channel, accepted_schemas, install, None);
3186        if subscribed.is_err() {
3187            // SRV-012 R6: a tap slot reserved at admission for a subscribe
3188            // that did not happen is given back, and a tap channel it
3189            // registered with no tap behind it leaves the roster.
3190            self.give_back_tap(channel);
3191        }
3192        subscribed
3193    }
3194
3195    fn unsubscribe(&self, subscription: ConnectionSubscription) -> Result<(), ServerError> {
3196        subscription.unsubscribe()
3197    }
3198
3199    fn open_conversation(
3200        &self,
3201        conversation_id: u64,
3202        subject: &str,
3203    ) -> Result<ConnectionConversation, ServerError> {
3204        self.open_routed_conversation(conversation_id, subject)
3205    }
3206
3207    fn conversation_message(
3208        &self,
3209        conversation: &ConnectionConversation,
3210        envelope: &MessageEnvelope,
3211        op_id: Option<u64>,
3212    ) -> Result<(), ServerError> {
3213        self.routed_conversation_message(conversation, envelope, op_id)
3214    }
3215
3216    fn close_conversation(&self, conversation: ConnectionConversation) -> Result<(), ServerError> {
3217        conversation.close()
3218    }
3219
3220    fn listening(self: Arc<Self>) -> Option<std::thread::JoinHandle<()>> {
3221        self.resolve_claims_behind_listener()
3222    }
3223
3224    fn stop_participant_warm(&self) {
3225        // DUR-020 R5: the warm stops at its next conversation boundary and
3226        // the stop does not wait for it; whatever it had not restored
3227        // restores on the next start. Then the clean stop records the
3228        // capacity snapshot (R3), and the final flush makes it durable.
3229        if let Some(warm) = &self.participant_warm {
3230            warm.stop();
3231            warm.record_clean_stop();
3232        }
3233    }
3234
3235    fn flush_durable_state(&self) -> Result<(), ServerError> {
3236        // A flush observer publishes to a durable channel, so no channel append lock can be held.
3237        // Separate ephemeral dedup state dies with this incarnation; shared dedup state is covered here.
3238        self.signals.with_flush_barrier(|| {
3239            block_on(self.durable_store.flush())
3240                .map_err(|error| ServerError::ShutdownFlush {
3241                    message: format!("shared durable store flush bridge failed: {error}"),
3242                })?
3243                .map_err(|error| ServerError::ShutdownFlush {
3244                    message: format!("shared durable store flush failed: {error}"),
3245                })?;
3246            Ok(())
3247        })?;
3248        Ok(())
3249    }
3250}
3251
3252/// One roster entry: a channel's library handle, the protocol schema id
3253/// advertised for it, where it came from, and its admission state.
3254#[derive(Debug)]
3255pub(super) struct ConfiguredChannel {
3256    handle: ChannelHandle,
3257    protocol_schema: ProtocolSchemaId,
3258    /// Durable (stored) or ephemeral, as built; the console's line life.
3259    durable: bool,
3260    /// Boot-configured or runtime-registered. Written once at construction and
3261    /// NEVER flipped: it is the only field that predicts what a restart does to
3262    /// this entry, and it defines the population the registration cap counts.
3263    origin: ChannelOrigin,
3264    /// [`STATE_ACTIVE`] or [`STATE_QUIESCED`]. Written at most once, by one
3265    /// `compare_exchange`, so the transition is one-way and exactly one caller
3266    /// can perform it.
3267    state: AtomicU8,
3268    /// The quiesce cause, set STRICTLY BEFORE `state` flips so any reader that
3269    /// observes [`STATE_QUIESCED`] can read it. Written once (the `OnceLock`
3270    /// enforces that structurally), which is why a re-quiesce under a different
3271    /// reason must refuse rather than silently keep or replace one of them.
3272    quiesce_reason: OnceLock<String>,
3273}
3274
3275impl ConfiguredChannel {
3276    /// The entry's admission state as a value.
3277    ///
3278    /// One `Acquire` load and, when quiesced, one read of the already-written
3279    /// reason. Touches no actor: the whole point of recording state on the entry
3280    /// is that observing it cannot spawn the thing being observed.
3281    fn state(&self) -> ChannelState {
3282        if self.state.load(Ordering::Acquire) == STATE_QUIESCED {
3283            ChannelState::Quiesced {
3284                reason: self.recorded_quiesce_reason(),
3285            }
3286        } else {
3287            ChannelState::Active
3288        }
3289    }
3290
3291    /// The recorded quiesce reason, for a caller that has already observed
3292    /// [`STATE_QUIESCED`] with an `Acquire` load.
3293    fn recorded_quiesce_reason(&self) -> String {
3294        self.quiesce_reason
3295            .get()
3296            .map_or_else(|| UNRECORDED_QUIESCE_REASON.to_owned(), Clone::clone)
3297    }
3298
3299    /// Moves this entry from active to quiesced, recording `reason` first.
3300    ///
3301    /// The reason is written to the `OnceLock` BEFORE the `Release`
3302    /// `compare_exchange` that flips the state, so no reader can observe
3303    /// `QUIESCED` without being able to read why. Re-quiescing under the
3304    /// IDENTICAL reason is `Ok(())` — the caller's intent already holds — and a
3305    /// DIFFERENT reason refuses, because the recorded one cannot be replaced
3306    /// without losing a cause and cannot be kept without lying to the caller.
3307    fn quiesce(&self, name: &str, reason: &str) -> Result<(), ChannelRegistryError> {
3308        // Whoever wins the `OnceLock` writes the cause on record; every other
3309        // caller — a re-quiesce or a concurrent racer — is judged against THAT
3310        // reason, never against whether it happened to perform the flip itself.
3311        // Reporting success for having won the CAS would tell a racer whose
3312        // reason lost that its reason took effect.
3313        let recorded = match self.quiesce_reason.set(reason.to_owned()) {
3314            Ok(()) => reason.to_owned(),
3315            Err(_rejected) => self.recorded_quiesce_reason(),
3316        };
3317        // The reason is now readable, so the flip may become visible. The CAS
3318        // result is deliberately unused: the state is `QUIESCED` afterwards
3319        // whether this call or a racer performed the transition, and the answer
3320        // to the caller is governed by the recorded reason above.
3321        let _flipped_here = self.state.compare_exchange(
3322            STATE_ACTIVE,
3323            STATE_QUIESCED,
3324            Ordering::Release,
3325            Ordering::Acquire,
3326        );
3327        if recorded == reason {
3328            return Ok(());
3329        }
3330        Err(ChannelRegistryError::AlreadyQuiesced {
3331            name: name.to_owned(),
3332            reason: recorded,
3333        })
3334    }
3335}
3336
3337#[derive(Debug)]
3338struct LiminalSubscriptionResource {
3339    subscription: liminal::channel::SubscriptionHandle,
3340}
3341
3342impl SubscriptionResource for LiminalSubscriptionResource {
3343    fn try_next_positioned(
3344        &mut self,
3345    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
3346        self.subscription
3347            .try_next_positioned()
3348            .map_err(|error| subscription_failure(&error))
3349    }
3350
3351    fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
3352        self.subscription
3353            .acknowledge_position(position)
3354            .map_err(|error| subscription_failure(&error))
3355    }
3356
3357    fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
3358        self.subscription
3359            .consumer_position()
3360            .map_err(|error| subscription_failure(&error))
3361    }
3362    fn unsubscribe(self: Box<Self>) -> Result<(), ServerError> {
3363        drop(self.subscription);
3364        Ok(())
3365    }
3366
3367    fn is_overflowed(&self) -> bool {
3368        self.subscription.is_overflowed()
3369    }
3370
3371    fn subscriber_pid(&self) -> Option<u64> {
3372        Some(self.subscription.subscriber_pid())
3373    }
3374
3375    fn has_pending(&self) -> bool {
3376        self.subscription.has_pending()
3377    }
3378
3379    fn try_next(&mut self) -> Option<liminal::envelope::Envelope> {
3380        match self.subscription.try_next() {
3381            Ok(envelope) => envelope,
3382            Err(error) => {
3383                tracing::error!(
3384                    %error,
3385                    "subscription could not receive its next envelope"
3386                );
3387                None
3388            }
3389        }
3390    }
3391}
3392
3393/// Preserves the library's subscription failure at the connection boundary.
3394fn subscription_failure(error: &liminal::error::LiminalError) -> ServerError {
3395    ServerError::ListenerAccept {
3396        message: format!("liminal subscription: {error}"),
3397    }
3398}
3399
3400/// Renders an admission refusal as the service error the trait's callers expect.
3401///
3402/// The two refusals that already existed keep their EXACT bytes: an absent
3403/// channel is still `channel '<name>' is not configured` and a poisoned roster
3404/// is still `channel roster lock poisoned`, so nothing that reads these messages
3405/// today sees a change. Discrimination lives in the reason code
3406/// ([`ChannelAccessError::reason_code`]), which the connection process carries to
3407/// the wire — this rendering is the degraded path for a caller that reaches the
3408/// service directly and has no code to carry.
3409///
3410/// `Quiesced` has no predecessor string to preserve; it carries the refusal's own
3411/// message, reason included.
3412///
3413/// The connection process renders its admission refusals through this SAME
3414/// function rather than through [`ChannelAccessError`]'s own `Display`. Two
3415/// consequences, both wanted. The bytes on the wire for an absent channel stay
3416/// exactly what they have always been, which is the whole of the semver promise
3417/// this lane makes. And an operation refused AT admission and one refused after
3418/// admission by the service produce the identical message, differing only in the
3419/// reason code — so the code really is the only discriminator, instead of the
3420/// message quietly becoming a second one.
3421pub(super) fn access_to_server_error(error: &ChannelAccessError) -> ServerError {
3422    let message = match error {
3423        ChannelAccessError::NotRegistered { name } => {
3424            format!("channel '{name}' is not configured")
3425        }
3426        ChannelAccessError::RosterUnavailable { message } => message.clone(),
3427        ChannelAccessError::Quiesced { .. }
3428        | ChannelAccessError::ConsoleNamespace { .. }
3429        | ChannelAccessError::ConsoleTapTarget { .. }
3430        | ChannelAccessError::ConsoleTapLimit { .. } => error.to_string(),
3431    };
3432    ServerError::ListenerAccept { message }
3433}
3434
3435/// Refuses a subscribe to `console.tap.<target>` whose target is empty or
3436/// under `console.`: the tap never taps the console's own records.
3437///
3438/// # Errors
3439/// Returns [`ChannelAccessError::ConsoleTapTarget`] naming the channel.
3440fn refuse_console_tap_target(channel: &str) -> Result<(), ChannelAccessError> {
3441    use super::channel_registry::{console_tap_target, is_console_subject};
3442    match console_tap_target(channel) {
3443        Some(target) if target.is_empty() || is_console_subject(target) => {
3444            Err(ChannelAccessError::ConsoleTapTarget {
3445                name: channel.to_owned(),
3446            })
3447        }
3448        _ => Ok(()),
3449    }
3450}
3451
3452/// Refuses a connection's publish under `console.`, the server's own namespace.
3453///
3454/// # Errors
3455/// Returns [`ChannelAccessError::ConsoleNamespace`] naming the channel.
3456fn refuse_console_namespace(channel: &str) -> Result<(), ChannelAccessError> {
3457    if super::channel_registry::is_console_subject(channel) {
3458        return Err(ChannelAccessError::ConsoleNamespace {
3459            name: channel.to_owned(),
3460        });
3461    }
3462    Ok(())
3463}
3464
3465pub(super) fn server_error_from_protocol(error: &ProtocolError) -> ServerError {
3466    ServerError::ListenerAccept {
3467        message: format!("protocol operation failed: {error}"),
3468    }
3469}
3470
3471/// The three pinned registration tests. A CHILD module of `services` because
3472/// test 2 must hold the exact entry the admission funnel handed out.
3473#[cfg(test)]
3474#[path = "services_registry_tests.rs"]
3475mod services_registry_tests;
3476
3477#[cfg(test)]
3478#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
3479mod durable_store_tests {
3480    use liminal::durability::open_ephemeral_rooted;
3481
3482    use super::subsystem_census::RecordingSubsystems;
3483    use super::{
3484        ConnectionServices, DEFAULT_SHARD_COUNT, LiminalConnectionServices, SchedulerSubsystem,
3485        build_connection_services, build_connection_services_via, build_durable_store,
3486    };
3487    use crate::ServerError;
3488    use crate::config::types::{LimitsConfig, ServerConfig, ServicesConfig};
3489
3490    /// Counts directory entries under `root`, for the empty/one-dir assertions
3491    /// on an injected ephemeral root.
3492    fn entry_count(root: &std::path::Path) -> usize {
3493        std::fs::read_dir(root)
3494            .expect("ephemeral root is readable")
3495            .count()
3496    }
3497
3498    /// A minimal channel-free config with the given service `profile`. No channels,
3499    /// routing, persistence, or cluster — the shape both profiles accept (the full
3500    /// profile simply builds an empty channel set; the worker-front-door profile
3501    /// requires exactly this shape).
3502    fn config_with_profile(profile: &str) -> ServerConfig {
3503        ServerConfig {
3504            listen_address: "127.0.0.1:0".parse().expect("valid socket addr"),
3505            health_listen_address: "127.0.0.1:1".parse().expect("valid socket addr"),
3506            drain_timeout_ms: 30_000,
3507            channels: Vec::new(),
3508            routing_rules: Vec::new(),
3509            persistence_path: None,
3510            cluster: None,
3511            auth: None,
3512            services: ServicesConfig {
3513                profile: profile.to_owned(),
3514                ..ServicesConfig::default()
3515            },
3516            limits: LimitsConfig::default(),
3517            participant: None,
3518            websocket: None,
3519        }
3520    }
3521
3522    /// §9 D2 front-door construction gate (fs half): building the worker-front-door
3523    /// services creates NO haematite store and NO temp dir, while the full profile
3524    /// over an equally-rooted factory DOES create exactly one store directory.
3525    ///
3526    /// The injected root is the only place an ephemeral store directory can appear
3527    /// (the recording factory roots its stores there), so "the root stays empty on
3528    /// the front-door branch" is a real negative assertion: a regression that gave
3529    /// the front door a store would land its directory here and fail this test. The
3530    /// full-profile arm is the positive control proving the seam genuinely
3531    /// constructs a store when the profile asks for one.
3532    #[test]
3533    fn worker_front_door_builds_no_store_and_no_temp_dir() {
3534        let front_door_root = tempfile::tempdir().expect("test can create an ephemeral root");
3535        let full_root = tempfile::tempdir().expect("test can create an ephemeral root");
3536
3537        let front_door_subsystems = RecordingSubsystems::rooted(front_door_root.path());
3538        let front_door: std::sync::Arc<dyn ConnectionServices> = build_connection_services_via(
3539            &config_with_profile("worker-front-door"),
3540            &front_door_subsystems,
3541        )
3542        .expect("worker-front-door services build");
3543        assert!(
3544            !front_door.supports_channel_operations(),
3545            "the worker front door serves no channel operations"
3546        );
3547        assert_eq!(
3548            entry_count(front_door_root.path()),
3549            0,
3550            "the worker front door creates no ephemeral store directory (no haematite, no temp dir)"
3551        );
3552
3553        let full_subsystems = RecordingSubsystems::rooted(full_root.path());
3554        let mut config = config_with_profile("full");
3555        config.persistence_path = Some(full_root.path().to_path_buf());
3556        let full =
3557            build_connection_services_via(&config, &full_subsystems).expect("full services build");
3558        assert!(
3559            full.supports_channel_operations(),
3560            "full mode serves channel operations"
3561        );
3562        assert_eq!(
3563            entry_count(full_root.path()),
3564            1,
3565            "full mode builds its persistent database in the explicit directory"
3566        );
3567
3568        drop(front_door);
3569        drop(full);
3570    }
3571
3572    /// §9 D2 front-door construction gate (thread half — record-by-construction
3573    /// census): the worker profile constructs NO channel-supervisor,
3574    /// conversation-supervisor, or haematite scheduler, while the SAME instrument
3575    /// over the full profile records all three — the positive control proving the
3576    /// census detects the extra schedulers, so an empty census on the worker branch
3577    /// is a real observation, not a decoration.
3578    ///
3579    /// The instrument's boundary: recording happens INSIDE the [`SubsystemFactory`]
3580    /// methods that are the profile-aware path's only route to these constructors,
3581    /// so a recording cannot be silently omitted — a future subsystem added to this
3582    /// path either goes through the factory (and is recorded by construction) or
3583    /// bypasses it, which is a code-review-visible structural violation of the
3584    /// factory seam, not a silently-missing side call. The connection supervisor's
3585    /// own scheduler is the shared baseline of both profiles and is asserted at the
3586    /// supervisor level (`supervisor::tests`); an OS-level thread census upgrades
3587    /// this when the beamr composition lane's scheduler-inventory API (currently on
3588    /// their branch, not yet consumable from liminal) lands.
3589    #[test]
3590    fn worker_profile_census_is_empty_and_full_profile_records_all_schedulers() {
3591        let worker_root = tempfile::tempdir().expect("test can create an ephemeral root");
3592        let full_root = tempfile::tempdir().expect("test can create an ephemeral root");
3593
3594        let worker_subsystems = RecordingSubsystems::rooted(worker_root.path());
3595        let front_door = build_connection_services_via(
3596            &config_with_profile("worker-front-door"),
3597            &worker_subsystems,
3598        )
3599        .expect("worker-front-door services build");
3600        assert_eq!(
3601            worker_subsystems.recorded(),
3602            Vec::<SchedulerSubsystem>::new(),
3603            "the worker front door constructs no scheduler-owning subsystem"
3604        );
3605
3606        let full_subsystems = RecordingSubsystems::rooted(full_root.path());
3607        let mut config = config_with_profile("full");
3608        config.persistence_path = Some(full_root.path().to_path_buf());
3609        let full =
3610            build_connection_services_via(&config, &full_subsystems).expect("full services build");
3611        assert_eq!(
3612            full_subsystems.recorded(),
3613            vec![
3614                SchedulerSubsystem::ChannelSupervisor,
3615                SchedulerSubsystem::ConversationSupervisor,
3616                SchedulerSubsystem::HaematiteStore,
3617            ],
3618            "the full profile constructs every scheduler-owning subsystem, once each — \
3619             the positive control proving the census instrument detects them"
3620        );
3621
3622        drop(front_door);
3623        drop(full);
3624    }
3625
3626    /// MAJOR-1 regression: the full-only constructors reject a worker-front-door
3627    /// config with a typed `ConfigValidation` error AT ENTRY — no full service can
3628    /// be created through any public config-based constructor under that profile.
3629    #[test]
3630    fn full_only_constructors_reject_worker_profile() {
3631        let config = config_with_profile("worker-front-door");
3632
3633        let from_config = LiminalConnectionServices::from_config(&config);
3634        assert!(
3635            matches!(from_config, Err(ServerError::ConfigValidation { .. })),
3636            "from_config must reject a worker-front-door profile with ConfigValidation, got {from_config:?}"
3637        );
3638
3639        let root = tempfile::tempdir().expect("test can create an ephemeral root");
3640        let store = open_ephemeral_rooted(root.path(), DEFAULT_SHARD_COUNT)
3641            .expect("test store for the rejection check builds");
3642        let from_config_with_store =
3643            LiminalConnectionServices::from_config_with_store(&config, std::sync::Arc::new(store));
3644        assert!(
3645            matches!(
3646                from_config_with_store,
3647                Err(ServerError::ConfigValidation { .. })
3648            ),
3649            "from_config_with_store must reject a worker-front-door profile with ConfigValidation"
3650        );
3651    }
3652
3653    /// MAJOR-1 regression: the profile-aware factory itself re-runs the
3654    /// worker-front-door cross-field checks, so a directly-constructed config (one
3655    /// that never passed file-loading validation) combining the worker profile with
3656    /// full-only machinery is refused with the same typed `ConfigValidation` errors.
3657    #[test]
3658    fn build_connection_services_rejects_worker_profile_with_full_only_fields() {
3659        let mut config = config_with_profile("worker-front-door");
3660        config.channels = vec![crate::config::types::ChannelDef {
3661            name: "orders".to_owned(),
3662            schema_ref: None,
3663            durable: false,
3664            loaded_schema: None,
3665        }];
3666        config.persistence_path = Some(std::path::PathBuf::from("/tmp"));
3667
3668        let result = build_connection_services(&config);
3669        let Err(ServerError::ConfigValidation { message }) = result else {
3670            panic!("expected ConfigValidation for worker profile with full-only fields");
3671        };
3672        assert!(message.contains("builds no channels"), "got: {message}");
3673        assert!(
3674            message.contains("builds no durable store"),
3675            "got: {message}"
3676        );
3677    }
3678
3679    /// The required-path constructor creates the persistent database in its
3680    /// configured directory without populating the separate throwaway root.
3681    #[test]
3682    fn persistent_store_uses_configured_path_and_creates_no_temp_dir() {
3683        let home = tempfile::tempdir().expect("test can create a temp dir");
3684        let ephemeral_root = tempfile::tempdir().expect("test can create an ephemeral root");
3685
3686        let store = build_durable_store(home.path()).expect("persistent store builds");
3687
3688        assert!(
3689            home.path().join("durability").join("config.json").exists(),
3690            "the persistent database is created under the configured path"
3691        );
3692        assert_eq!(
3693            entry_count(ephemeral_root.path()),
3694            0,
3695            "the persistent branch creates no ephemeral guard directory"
3696        );
3697
3698        drop(store);
3699    }
3700
3701    #[test]
3702    fn missing_path_refuses_before_any_store_is_opened() -> Result<(), Box<dyn std::error::Error>> {
3703        let root = tempfile::tempdir()?;
3704        let factory = RecordingSubsystems::rooted(root.path());
3705        let result = build_connection_services_via(&config_with_profile("full"), &factory);
3706        assert!(
3707            matches!(result, Err(ServerError::ConfigValidation { message }) if message.contains("no store is opened"))
3708        );
3709        assert_eq!(entry_count(root.path()), 0);
3710        assert!(factory.recorded().is_empty());
3711        Ok(())
3712    }
3713
3714    /// The store migrates itself: a `config.json` written before haematite
3715    /// required `node_cache_budget` opens on the new server with no operator
3716    /// hand, and the file afterwards carries the key — spelled exactly as the
3717    /// create arm spells it. The pre-field store is fabricated by creating a
3718    /// current one and deleting the key, which is byte-faithful to what a
3719    /// 0.8.1-era store looks like to the parser: the key absent, everything
3720    /// else in place.
3721    #[test]
3722    fn a_store_whose_config_predates_the_budget_field_opens_with_no_operator_hand() {
3723        let home = tempfile::tempdir().expect("test can create a temp dir");
3724        let store = build_durable_store(home.path()).expect("persistent store builds");
3725        drop(store);
3726
3727        let config_file = home.path().join("durability").join("config.json");
3728        let bytes = std::fs::read(&config_file).expect("config.json is readable");
3729        let mut config: serde_json::Value =
3730            serde_json::from_slice(&bytes).expect("config.json parses");
3731        config
3732            .as_object_mut()
3733            .expect("config.json is an object")
3734            .remove("node_cache_budget")
3735            .expect("the created config carries the budget to remove");
3736        std::fs::write(
3737            &config_file,
3738            serde_json::to_vec_pretty(&config).expect("serializes"),
3739        )
3740        .expect("test can age the config");
3741
3742        let reopened = build_durable_store(home.path())
3743            .expect("a pre-budget store opens with no operator action");
3744        drop(reopened);
3745
3746        let after = std::fs::read(&config_file).expect("config.json is readable after");
3747        let migrated: serde_json::Value =
3748            serde_json::from_slice(&after).expect("migrated config parses");
3749        assert_eq!(
3750            migrated.get("node_cache_budget"),
3751            Some(&serde_json::Value::String("unlimited".to_owned())),
3752            "the migrated file carries the budget the create arm chooses"
3753        );
3754    }
3755
3756    /// The migration touches only the one migratable case. A corrupt
3757    /// `config.json` still refuses — through haematite's own parse refusal,
3758    /// which is sharper than anything this crate could say — and the bytes are
3759    /// left exactly as found, so the operator diagnoses the file that failed,
3760    /// not one the migration half-rewrote.
3761    #[test]
3762    fn a_corrupt_config_still_refuses_and_is_left_untouched() {
3763        let home = tempfile::tempdir().expect("test can create a temp dir");
3764        let data_dir = home.path().join("durability");
3765        std::fs::create_dir_all(&data_dir).expect("test can create the data dir");
3766        let config_file = data_dir.join("config.json");
3767        let garbage = b"{ not json at all";
3768        std::fs::write(&config_file, garbage).expect("test can write garbage");
3769
3770        let refused = build_durable_store(home.path());
3771        assert!(refused.is_err(), "a corrupt config refuses to open");
3772        assert_eq!(
3773            std::fs::read(&config_file)
3774                .expect("config.json is still readable")
3775                .as_slice(),
3776            garbage,
3777            "the refusing file is left byte-identical for the operator to read"
3778        );
3779    }
3780
3781    /// A config already carrying the budget is not rewritten: reopening leaves
3782    /// the file byte-identical, so the migration cannot churn mtimes, reorder
3783    /// keys, or otherwise touch a store that needs nothing.
3784    #[test]
3785    fn a_config_already_carrying_the_budget_is_left_byte_identical() {
3786        let home = tempfile::tempdir().expect("test can create a temp dir");
3787        let store = build_durable_store(home.path()).expect("persistent store builds");
3788        drop(store);
3789
3790        let config_file = home.path().join("durability").join("config.json");
3791        let before = std::fs::read(&config_file).expect("config.json is readable");
3792        let reopened = build_durable_store(home.path()).expect("a current store reopens");
3793        drop(reopened);
3794        assert_eq!(
3795            std::fs::read(&config_file).expect("config.json is readable after"),
3796            before,
3797            "a config that already carries the budget is left byte-identical"
3798        );
3799    }
3800
3801    /// A durable store created under `KeepAll` is moved onto reclaim by the
3802    /// server's next open: its `config.json` names `reclaim` afterwards, and the
3803    /// copies its old commits left are gone once reclaim has run.
3804    #[test]
3805    fn a_keep_all_durable_store_reclaims_after_the_next_open() {
3806        let dir = tempfile::tempdir().expect("test can create a store dir");
3807        let data_dir = dir.path().join("durability");
3808        let database = haematite::Database::create(
3809            haematite::DatabaseConfig {
3810                data_dir: data_dir.clone(),
3811                shard_count: 1,
3812                distributed: None,
3813                executor_threads: None,
3814                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
3815                retention: haematite::Retention::KeepAll,
3816            },
3817            haematite::RuntimeOptions {
3818                long_wait_interval: std::time::Duration::from_secs(5),
3819                reclaim_start: haematite::ReclaimStart::AtOpen,
3820            },
3821        )
3822        .expect("a keep-all store is created");
3823        for round in 0_u32..200 {
3824            let mut value = round.to_be_bytes().to_vec();
3825            value.resize(1_024, 0x5a);
3826            database.put(b"key".to_vec(), value).expect("put");
3827            database.commit().expect("commit");
3828        }
3829        drop(database);
3830        let grown =
3831            haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir.clone()))
3832                .expect("stats of the grown store")
3833                .totals
3834                .enumerated_bytes;
3835
3836        let reopened = super::open_or_create_database(&data_dir).expect("the server opens it");
3837        // The server holds reclaim at open and releases it once its listeners
3838        // bind (DUR-021 R1); the test releases it the same way.
3839        reopened
3840            .release_reclaim()
3841            .expect("the held reclaim is released");
3842        reopened.quiesce_reclaim().expect("reclaim runs to rest");
3843        drop(reopened);
3844
3845        let config: serde_json::Value = serde_json::from_slice(
3846            &std::fs::read(data_dir.join("config.json")).expect("config is readable"),
3847        )
3848        .expect("config is JSON");
3849        assert_eq!(
3850            config["retention"],
3851            serde_json::to_value(super::DURABLE_RETENTION).expect("the retention serializes")
3852        );
3853        let report = haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir))
3854            .expect("stats of the reclaimed store");
3855        assert!(
3856            report.totals.enumerated_bytes <= report.totals.marked_bytes.saturating_mul(3),
3857            "the reopened store holds about its live set: enumerated {} B, marked {} B",
3858            report.totals.enumerated_bytes,
3859            report.totals.marked_bytes
3860        );
3861        assert!(
3862            report.totals.enumerated_bytes.saturating_mul(10) < grown,
3863            "the old copies are gone: {} B now against {grown} B grown",
3864            report.totals.enumerated_bytes
3865        );
3866    }
3867
3868    /// `config.json` exactly as a store grown by an engine that never reclaimed
3869    /// carries it: `reclaim` with `past_roots` 0 is already written, so the
3870    /// open has no retention to adopt (it does move the format to version 3),
3871    /// and every superseded copy is still on disk.
3872    /// `DATA_DIR` stands for the one field the fixture rewrites.
3873    const STORED_RECLAIM_CONFIG: &str = r#"{
3874  "chunking_policy": {
3875    "chunking": "bytes-v2",
3876    "internal_target_bytes": 49152,
3877    "leaf_target_bytes": 65536
3878  },
3879  "data_dir": "DATA_DIR",
3880  "distributed": null,
3881  "executor_threads": null,
3882  "format_version": 2,
3883  "node_cache_budget": "unlimited",
3884  "retention": {
3885    "reclaim": {
3886      "past_roots": 0
3887    }
3888  },
3889  "shard_count": 8
3890}"#;
3891
3892    /// A store whose `config.json` already names reclaim, but whose history was
3893    /// grown with nothing reclaimed, is opened by the server and gives its old
3894    /// copies back with no write and no quiesce. A store with no reclaim
3895    /// record owes a startup pass, and a shard runs it on its first turn. The
3896    /// seed holds both shapes the pass must free: one row overwritten many
3897    /// times, and rows written and then deleted, as a prune leaves them.
3898    #[test]
3899    fn a_stored_reclaim_store_grown_without_reclaim_gives_its_old_copies_back_on_open() {
3900        let dir = tempfile::tempdir().expect("test can create a store dir");
3901        let data_dir = dir.path().join("durability");
3902        let database = haematite::Database::create(
3903            haematite::DatabaseConfig {
3904                data_dir: data_dir.clone(),
3905                shard_count: 8,
3906                distributed: None,
3907                executor_threads: None,
3908                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
3909                retention: haematite::Retention::KeepAll,
3910            },
3911            haematite::RuntimeOptions {
3912                long_wait_interval: std::time::Duration::from_secs(5),
3913                reclaim_start: haematite::ReclaimStart::AtOpen,
3914            },
3915        )
3916        .expect("a keep-all store is created");
3917        for round in 0_u32..200 {
3918            let mut value = round.to_be_bytes().to_vec();
3919            value.resize(1_024, 0x5a);
3920            database.put(b"key".to_vec(), value).expect("put");
3921            database.commit().expect("commit");
3922        }
3923        for row in 0_u32..200 {
3924            let key = [b"pruned-".as_slice(), &row.to_be_bytes()].concat();
3925            database.put(key, vec![0xa5; 1_024]).expect("put");
3926            database.commit().expect("commit");
3927        }
3928        for row in 0_u32..200 {
3929            let key = [b"pruned-".as_slice(), &row.to_be_bytes()].concat();
3930            database.delete(key).expect("delete");
3931        }
3932        database.commit().expect("commit");
3933        drop(database);
3934        let grown_report =
3935            haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir.clone()))
3936                .expect("stats of the grown store");
3937        let grown = grown_report.totals;
3938        assert!(
3939            grown.unmarked_nodes > 0,
3940            "the seed leaves copies no root reaches"
3941        );
3942        let mut owed: std::collections::BTreeSet<usize> = grown_report
3943            .shards
3944            .iter()
3945            .filter(|shard| shard.nodes.unmarked_nodes > 0)
3946            .map(|shard| shard.shard_id)
3947            .collect();
3948
3949        let config_file = data_dir.join("config.json");
3950        let seeded = STORED_RECLAIM_CONFIG.replace(
3951            "DATA_DIR",
3952            data_dir.to_str().expect("the test dir is UTF-8"),
3953        );
3954        std::fs::write(&config_file, &seeded).expect("config.json is written");
3955
3956        let reopened = super::open_or_create_database(&data_dir).expect("the server opens it");
3957        let watch = reopened
3958            .watch_reclaim_passes(64)
3959            .expect("every shard accepts a pass watch");
3960        // Released as the server releases it once its listeners bind (DUR-021 R1).
3961        reopened
3962            .release_reclaim()
3963            .expect("the held reclaim is released");
3964        let mut unmarked_by_shard = std::collections::BTreeMap::new();
3965        while !owed.is_empty() {
3966            let report = watch
3967                .recv_timeout(std::time::Duration::from_secs(30))
3968                .expect("each pass the open owes ends without a write or a quiesce");
3969            assert_eq!(
3970                report.outcome,
3971                haematite::ReclaimPassOutcome::Passed,
3972                "the pass the open owed on shard {} finished",
3973                report.shard_id
3974            );
3975            if owed.remove(&report.shard_id) {
3976                unmarked_by_shard.insert(report.shard_id, report.unmarked_bytes);
3977            }
3978        }
3979        drop(reopened);
3980
3981        let after = haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir))
3982            .expect("stats of the reclaimed store")
3983            .totals;
3984        eprintln!(
3985            "grown {} B, after {} B, {} unreachable nodes seeded, unmarked bytes by shard {unmarked_by_shard:?}",
3986            grown.enumerated_bytes, after.enumerated_bytes, grown.unmarked_nodes
3987        );
3988        assert!(
3989            after.enumerated_bytes.saturating_mul(10) < grown.enumerated_bytes,
3990            "the old copies are gone: {} B now against {} B grown",
3991            after.enumerated_bytes,
3992            grown.enumerated_bytes
3993        );
3994        let config: serde_json::Value = serde_json::from_slice(
3995            &std::fs::read(&config_file).expect("config.json is readable after"),
3996        )
3997        .expect("config is JSON");
3998        assert_eq!(
3999            config["retention"],
4000            serde_json::to_value(super::DURABLE_RETENTION).expect("the retention serializes"),
4001            "config.json names reclaim after the open"
4002        );
4003    }
4004}