Skip to main content

liminal_server/server/connection/
services.rs

1#[path = "publication_identity.rs"]
2mod publication_identity;
3use liminal::channel::PublicationIdentity;
4use std::collections::HashMap;
5use std::path::{Path, PathBuf};
6use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
7use std::sync::{Arc, Mutex, OnceLock, RwLock};
8use std::time::Instant;
9
10use haematite::{CommitPolicy, Database, DatabaseConfig, EventStore, Retention};
11use liminal::channel::{
12    ChannelConfig, ChannelDelivery, ChannelHandle, ChannelMode, ChannelSupervisor, Schema,
13};
14use liminal::conversation::{ConversationSupervisor, ParticipantBehaviour};
15use liminal::durability::bridge::block_on;
16#[cfg(any(test, feature = "test-support"))]
17use liminal::durability::open_ephemeral;
18use liminal::durability::{
19    CountingStore, DedupCache, DedupDecision, DurableStore, EphemeralRoot, EphemeralSweepCounts,
20    HaematiteStore, ProcessingReceipt, StoreCountsHandle,
21};
22use liminal::protocol::{MessageEnvelope, ProtocolError, SchemaId as ProtocolSchemaId};
23use std::num::NonZeroU64;
24
25use super::channel_registry::{
26    ChannelAccessError, ChannelBuildError, ChannelConfigField, ChannelDescriptor, ChannelOrigin,
27    ChannelRegistration, ChannelRegistryError, ChannelState, ChannelStatus, MAX_CHANNELS_KEY,
28    Registered, STATE_ACTIVE, STATE_QUIESCED, UNRECORDED_QUIESCE_REASON,
29};
30use super::conversation::ConnectionConversation;
31use super::services_cluster::build_channel_cluster;
32use super::services_schema::{ChannelSchema, resolve_channel_schema, resolve_schema_bytes};
33use super::worker_front_door::WorkerFrontDoorServices;
34use crate::ServerError;
35use crate::config::types::{ClusterConfig, ParticipantConfig, ServerConfig, ServiceProfile};
36use crate::config::validation::required_persistence_path;
37use crate::health::reissue::OperatorCredentialReissuer;
38use crate::health::unloadable::UnloadableConversationRecord;
39use crate::server::participant::{
40    InstalledParticipantService, ParticipantWarm, ProductionParticipantHandler,
41};
42use crate::server::self_streams::events::PublishOrigin;
43use crate::server::self_streams::sink::OriginPublication;
44use crate::server::signals::ServerSignals;
45
46pub use super::services_cluster::ChannelCluster;
47
48#[path = "../../agents/host_server.rs"]
49mod agent_host;
50#[path = "../../agents/server.rs"]
51mod agent_services;
52#[path = "../../agents/approval_boot.rs"]
53mod approval_boot;
54#[path = "../../services/site/server.rs"]
55mod site_services;
56pub use site_services::{
57    SiteDraftHolder, evaluate_subject as evaluate_site_subject, is_site_route, plan_subject,
58};
59pub(super) use site_services::{SiteFrame, site_frame_response};
60#[path = "services_lifecycle.rs"]
61mod lifecycle;
62#[path = "services_conversation.rs"]
63mod services_conversation;
64
65/// Registry of custom conversation responders, keyed by conversation subject.
66///
67/// A registered [`ParticipantBehaviour`] becomes the participant for any
68/// conversation opened on its subject; an unregistered subject has no local server.
69type ResponderRegistry = HashMap<String, Arc<dyn ParticipantBehaviour>>;
70
71/// Marker for resources retained by a connection process until unsubscribe.
72pub trait SubscriptionResource: std::fmt::Debug + Send {
73    /// Releases the library subscription resource.
74    ///
75    /// # Errors
76    /// Returns [`ServerError`] when the liminal library reports an unsubscribe failure.
77    fn unsubscribe(self: Box<Self>) -> Result<(), ServerError>;
78
79    /// Attempts to pull the next delivered envelope from the wrapped library
80    /// subscription without blocking.
81    ///
82    /// Returns `None` when the subscriber inbox is empty (or momentarily
83    /// unavailable): the connection process is the delivery pump, so a transient
84    /// empty read is simply "nothing to deliver this slice", never an error.
85    fn try_next(&mut self) -> Option<liminal::envelope::Envelope>;
86
87    /// Non-consuming availability query for the post-arm race barrier.
88    fn has_pending(&self) -> bool;
89
90    /// Whether an overflow has marked this subscription for shedding (§5). The
91    /// delivery pump sheds an overflowed subscription with a typed error frame.
92    /// Defaulted to `false`: a resource with no bounded inbox never overflows.
93    fn is_overflowed(&self) -> bool {
94        false
95    }
96
97    /// The library subscriber's pid, the id a recording publish names it by.
98    /// Defaulted to `None`: a resource with no library subscriber is never a
99    /// recipient.
100    fn subscriber_pid(&self) -> Option<u64> {
101        None
102    }
103
104    /// Reads a delivery and its durable position without acknowledging it.
105    ///
106    /// # Errors
107    /// The underlying subscription cannot be read.
108    fn try_next_positioned(
109        &mut self,
110    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
111        Ok(self.try_next().map(|envelope| (envelope, None)))
112    }
113
114    /// Checkpoints a delivery after its consumer settles it.
115    ///
116    /// # Errors
117    /// The resource has no durable acknowledgement path, or checkpointing failed.
118    fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
119        Err(ServerError::ListenerAccept {
120            message: format!("subscription cannot acknowledge durable position {position}"),
121        })
122    }
123
124    /// Reads the subscription-owned durable checkpoint.
125    ///
126    /// # Errors
127    /// The checkpoint cannot be read.
128    fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
129        Ok(None)
130    }
131}
132
133/// Library subscription resource owned by a single connection process.
134#[derive(Debug)]
135pub struct ConnectionSubscription {
136    id: u64,
137    /// Channel this subscription was opened on, carried for DIAGNOSTICS only —
138    /// the shed warning has to name the channel or an operator reading it learns
139    /// that something was shed without learning what. Empty when unset (the
140    /// scheduler-free unit fixtures, which construct through `new` and never log).
141    channel: String,
142    /// Client-chosen application stream the server delivers this subscription's
143    /// messages on (echoed on `SubscribeAck`, carried on every `Deliver`). Set by
144    /// the connection process from the `Subscribe` frame before the subscription is
145    /// stored; `0` only while momentarily unset during construction.
146    stream_id: u32,
147    selected_schema: ProtocolSchemaId,
148    resource: Box<dyn SubscriptionResource>,
149    issued: super::state::IssuedDeliveries,
150}
151
152impl ConnectionSubscription {
153    /// Creates an owned subscription resource for one connection process.
154    #[must_use]
155    pub fn new(
156        id: u64,
157        selected_schema: ProtocolSchemaId,
158        resource: Box<dyn SubscriptionResource>,
159    ) -> Self {
160        Self {
161            id,
162            channel: String::new(),
163            stream_id: 0,
164            selected_schema,
165            resource,
166            issued: super::state::IssuedDeliveries::new(0),
167        }
168    }
169
170    /// Records the channel this subscription was opened on (diagnostics).
171    ///
172    /// Additive setter rather than a fourth `new` parameter: `new` is public API
173    /// and its existing callers keep compiling byte-for-byte.
174    pub(super) fn set_channel(&mut self, channel: String) {
175        self.channel = channel;
176    }
177
178    /// The channel this subscription was opened on, or `""` when unset.
179    #[must_use]
180    pub(super) fn channel(&self) -> &str {
181        &self.channel
182    }
183
184    /// Returns the protocol subscription id.
185    #[must_use]
186    pub const fn id(&self) -> u64 {
187        self.id
188    }
189
190    /// Records the client-chosen delivery stream id for this subscription.
191    pub(super) const fn set_stream_id(&mut self, stream_id: u32) {
192        self.stream_id = stream_id;
193    }
194
195    /// Returns the client-chosen application stream id deliveries ride on.
196    #[must_use]
197    pub(super) const fn stream_id(&self) -> u32 {
198        self.stream_id
199    }
200
201    pub(super) fn set_pressure_ledger_depth(&mut self, limit: usize) {
202        self.issued = super::state::IssuedDeliveries::new(limit);
203    }
204
205    pub(super) fn issued_delivery(&mut self, sequence: u64) {
206        self.issued.record(sequence);
207    }
208
209    pub(super) fn match_pressure(&mut self, reference: &str, retain: bool) -> bool {
210        self.issued.matches(reference, retain)
211    }
212
213    /// Returns the schema selected for this subscription stream.
214    #[must_use]
215    pub const fn selected_schema(&self) -> ProtocolSchemaId {
216        self.selected_schema
217    }
218
219    /// The library subscriber's pid, when the subscription has one.
220    #[must_use]
221    pub fn subscriber_pid(&self) -> Option<u64> {
222        self.resource.subscriber_pid()
223    }
224
225    /// Attempts to pull the next delivered envelope without blocking.
226    pub(crate) fn try_next(&mut self) -> Option<liminal::envelope::Envelope> {
227        self.resource.try_next()
228    }
229
230    pub(crate) fn has_pending(&self) -> bool {
231        self.resource.has_pending()
232    }
233
234    /// Whether this subscription has been shed by an inbox overflow (§5).
235    pub(crate) fn is_overflowed(&self) -> bool {
236        self.resource.is_overflowed()
237    }
238
239    pub(super) fn unsubscribe(self) -> Result<(), ServerError> {
240        self.resource.unsubscribe()
241    }
242
243    pub(crate) fn try_next_positioned(
244        &mut self,
245    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
246        self.resource.try_next_positioned()
247    }
248
249    pub(crate) fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
250        self.resource.acknowledge_position(position)
251    }
252
253    pub(crate) fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
254        self.resource.consumer_position()
255    }
256}
257
258/// Outcome of a server publish.
259///
260/// Carries the assigned message id plus a genuine delivery ack (`delivered` = the
261/// message was accepted by at least one live subscriber on this publish, after any
262/// dedup-on-delivery suppression).
263#[derive(Clone, Copy, Debug, PartialEq, Eq)]
264pub struct PublishOutcome {
265    /// Transport-scoped message id assigned to the accepted publish.
266    ///
267    /// TRANSPORT-SCOPED, NOT A DURABLE POSITION (G1,
268    /// `docs/design/wire-durability-gaps-20260831.md`): this is a
269    /// process-local counter (`next_message_id`, reset to 1 on every
270    /// construction), so it restarts from 1 with the server and two servers
271    /// mint the same ids. It is monotonic only within one service instance's
272    /// lifetime and identifies the publish on THIS connection's wire, nothing
273    /// more. The durable position of a publish on a durable channel is the
274    /// store-assigned sequence the library now surfaces as
275    /// `ChannelDelivery::durable_position`; putting that on the wire ack is
276    /// the A1 resume lane's frame decision, not a reinterpretation of this
277    /// field.
278    pub message_id: u64,
279    /// Whether the message was genuinely delivered to a subscriber. `false` means
280    /// the publish was accepted but reached no subscriber (empty channel) or was
281    /// a duplicate suppressed by dedup-on-delivery.
282    pub delivered: bool,
283    /// Store-assigned position of the row this outcome answers for, when the
284    /// publish is known to have one. Unlike `message_id` this IS a durable
285    /// position; the connection process puts it on the wire ack only for a
286    /// connection that negotiated `PUBLISH_ACK_POSITION_VERSION`.
287    pub durable_position: Option<u64>,
288    /// Whom the publish reached, the console tap's word for it (SRV-012
289    /// R10): a subscriber, no subscriber, only subscribers that refused it,
290    /// or nobody because dedup suppressed it.
291    pub reach: Reach,
292}
293
294/// Whom a publish reached (SRV-012 R10).
295#[derive(Clone, Copy, Debug, PartialEq, Eq)]
296pub enum Reach {
297    /// At least one live subscriber took it.
298    Delivered,
299    /// The channel had no matching subscriber.
300    NoSubscriber,
301    /// Every matching subscriber refused it.
302    NoneAccepted,
303    /// A dedup-suppressed re-publish, sent to nobody.
304    Suppressed,
305}
306
307impl Reach {
308    /// The reach of a library delivery.
309    #[must_use]
310    pub const fn of(delivery: &ChannelDelivery) -> Self {
311        if delivery.is_delivered() {
312            Self::Delivered
313        } else if delivery.matched_count() == 0 {
314            Self::NoSubscriber
315        } else {
316            Self::NoneAccepted
317        }
318    }
319}
320
321/// Which channel operation is asking the roster for admission.
322///
323/// `Copy`, so the hot path allocates nothing to ask. The contrast is
324/// [`ServerError::UnsupportedOperation`]'s owned `operation: String`: that one is
325/// built on a refusal path and pays for its allocation once per refusal, while
326/// this one is consulted on every publish and every subscribe frame that reaches
327/// the connection process.
328#[derive(Clone, Copy, Debug, PartialEq, Eq)]
329pub enum ChannelOperation {
330    /// A `Publish` frame is asking.
331    Publish,
332    /// A `Subscribe` frame is asking.
333    Subscribe,
334}
335
336/// Operations that adapt wire frames to liminal library calls.
337pub trait ConnectionServices: std::fmt::Debug + Send + Sync {
338    /// Construction built-ins are dispatched before ordinary worker routing.
339    fn construction_request(
340        &self,
341        _: Option<&dyn crate::auth_lys::session::LysSession>,
342        subject: &str,
343        _: &[u8],
344        _: u64,
345    ) -> Option<Result<serde_json::Value, ServerError>> {
346        is_site_route(subject).then(|| {
347            Err(site_services::site_error(
348                crate::services::lifecycle::site::SiteError::Invalid("construction not installed"),
349            ))
350        })
351    }
352    /// Reserves the completion barrier before storing a trial registration.
353    ///
354    /// # Errors
355    /// The trial's identity, contracts or serve decision refused.
356    fn prepare_site_trial(
357        &self,
358        _: u64,
359        _: Option<&dyn crate::auth_lys::session::LysSession>,
360        _: &liminal::protocol::WorkerRegistration,
361        _: Option<&crate::services::lifecycle::ProcessGrant>,
362        _: u64,
363    ) -> Result<Option<crate::services::lifecycle::trial::TrialPermit>, ServerError> {
364        Ok(None)
365    }
366    /// Checks a Building subject before routing and before releasing a reply.
367    ///
368    /// # Errors
369    /// A trial caller or its pass was refused.
370    fn admit_site_call(
371        &self,
372        _: Option<&dyn crate::auth_lys::session::LysSession>,
373        _: &str,
374        _: u64,
375    ) -> Result<bool, ServerError> {
376        Ok(false)
377    }
378    /// Removes trial barriers when the directory removes a connection.
379    ///
380    /// # Errors
381    /// The target store or console projection failed.
382    fn site_registration_closed(&self, _: u64) -> Result<(), ServerError> {
383        Ok(())
384    }
385    /// Publishes with the identity proved at Connect; it adds no authority.
386    ///
387    /// # Errors
388    /// Identity, rights, payload or publication admission refused.
389    fn publish_identified(
390        &self,
391        connection: u64,
392        session: Option<&dyn crate::auth_lys::session::LysSession>,
393        channel: &str,
394        envelope: &MessageEnvelope,
395        key: Option<&str>,
396    ) -> Result<PublishOutcome, ServerError> {
397        crate::auth_membership::check_agent_binding(
398            session,
399            crate::auth_lys::HotAction::Publish,
400            channel,
401        )?;
402        self.publish(connection, channel, envelope, key)
403    }
404    /// Agent station state, when configured.
405    fn agents(&self) -> Option<&Arc<crate::agents::runtime::Agents>> {
406        None
407    }
408
409    /// Subscribes using the identity proved at Connect; it adds no authority.
410    ///
411    /// # Errors
412    /// Identity, rights or subscription admission refused.
413    fn subscribe_identified(
414        &self,
415        connection: u64,
416        session: Option<&dyn crate::auth_lys::session::LysSession>,
417        channel: &str,
418        schemas: &[ProtocolSchemaId],
419        install: Option<liminal::channel::InboxInstall>,
420    ) -> Result<ConnectionSubscription, ServerError> {
421        if connection == 0 {
422            return Err(crate::agents::error::AgentError::Invalid("connection id").into());
423        }
424        crate::auth_membership::check_agent_binding(
425            session,
426            crate::auth_lys::HotAction::Subscribe,
427            channel,
428        )?;
429        self.subscribe(channel, schemas, install)
430    }
431    /// Returns the complete participant service installed on this adapter.
432    ///
433    /// `None` keeps participant capability disabled even when the adapter owns a
434    /// durable store for unrelated channel traffic. The returned token is
435    /// server-sealed and atomically carries declared semantics plus durability,
436    /// making a handler-without-store activation impossible by construction.
437    fn participant_service(&self) -> Option<InstalledParticipantService> {
438        None
439    }
440
441    /// Delegates a publish request to the liminal library.
442    ///
443    /// `connection` is the publishing connection's id (its process pid, as
444    /// `apply_frame` knows it), carried as the publish origin so the console
445    /// tap attributes the message to it (SRV-012). `idempotency_key`, when `Some`, drives dedup-on-delivery: a re-publish with
446    /// the same key is delivered to subscribers at most once. The returned
447    /// [`PublishOutcome`] carries the genuine delivery ack.
448    ///
449    /// # Errors
450    /// Returns [`ServerError`] when the liminal publish operation fails.
451    fn publish(
452        &self,
453        connection: u64,
454        channel: &str,
455        envelope: &MessageEnvelope,
456        idempotency_key: Option<&str>,
457    ) -> Result<PublishOutcome, ServerError>;
458
459    /// Delegates a subscribe request to the liminal library.
460    ///
461    /// `install`, when `Some`, carries the connection's §5 shared inbox byte
462    /// budget, per-inbox fairness cap, and R3 wake notifier. The implementation
463    /// MUST install it on the subscription's inbox BEFORE the registration is
464    /// published to the channel actor (i.e. before any envelope can be
465    /// delivered), so no envelope is ever admitted uncharged, past the depth
466    /// cap, or without a wake. Implementations with no real inbox (test
467    /// stand-ins, capability-scoped profiles that refuse subscribe) may ignore
468    /// it.
469    ///
470    /// # Errors
471    /// Returns [`ServerError`] when the liminal subscribe operation fails.
472    fn subscribe(
473        &self,
474        channel: &str,
475        accepted_schemas: &[ProtocolSchemaId],
476        install: Option<liminal::channel::InboxInstall>,
477    ) -> Result<ConnectionSubscription, ServerError>;
478
479    /// Delegates unsubscribe to the liminal library.
480    ///
481    /// # Errors
482    /// Returns [`ServerError`] when the liminal unsubscribe operation fails.
483    fn unsubscribe(&self, subscription: ConnectionSubscription) -> Result<(), ServerError>;
484
485    /// Delegates conversation open to the liminal library.
486    ///
487    /// # Errors
488    /// Returns [`ServerError`] when the liminal conversation open operation fails.
489    fn open_conversation(
490        &self,
491        conversation_id: u64,
492        subject: &str,
493    ) -> Result<ConnectionConversation, ServerError>;
494
495    /// Delegates a conversation message to the liminal library.
496    ///
497    /// # Errors
498    /// Returns [`ServerError`] when the liminal conversation message operation fails.
499    fn conversation_message(
500        &self,
501        conversation: &ConnectionConversation,
502        envelope: &MessageEnvelope,
503        op_id: Option<u64>,
504    ) -> Result<(), ServerError>;
505
506    /// Delegates conversation close to the liminal library.
507    ///
508    /// # Errors
509    /// Returns [`ServerError`] when the liminal conversation close operation fails.
510    fn close_conversation(&self, conversation: ConnectionConversation) -> Result<(), ServerError>;
511
512    /// Flushes the shared durable store after channel publications have settled.
513    ///
514    /// # Errors
515    /// Returns [`ServerError`] when the shared store flush operation fails.
516    fn flush_durable_state(&self) -> Result<(), ServerError>;
517
518    /// Stops the participant warm and records the clean stop's capacity
519    /// snapshot (DUR-020 R3, R5). Called once, by the shutdown sequence,
520    /// before the final flush; an adapter with no participant does nothing.
521    fn stop_participant_warm(&self) {}
522
523    /// Joins background work that can still write to the store.
524    ///
525    /// # Errors
526    /// Returns a named shutdown failure when an owned worker cannot be joined.
527    fn finish_background_store_work(&self) -> Result<(), ServerError> {
528        Ok(())
529    }
530
531    /// Stops store-owning service workers after the final flush.
532    ///
533    /// # Errors
534    /// Returns a named shutdown failure when a service worker cannot be joined.
535    fn release_store_holders(&self) -> Result<(), ServerError> {
536        Ok(())
537    }
538
539    /// The listener is bound: whatever the start reads that no admission
540    /// needs before it is judged runs from here, behind the listening door.
541    /// An adapter with nothing to read does nothing. Answers the thread it
542    /// started, which the listener leaves to run and a test may join.
543    fn listening(self: Arc<Self>) -> Option<std::thread::JoinHandle<()>> {
544        None
545    }
546
547    /// Whether this adapter backs ordinary channel and conversation operations.
548    ///
549    /// The default is `true` — the full-service adapter serves publish, subscribe,
550    /// and conversation frames, so full mode is byte-for-byte unchanged. The
551    /// capability-scoped worker front door overrides this to `false`, letting
552    /// [`super::apply`] reject the channel/conversation frames it short-circuits on
553    /// empty connection state (`Unsubscribe`, `ConversationMessage`,
554    /// `ConversationClose`) with a typed error frame instead of silently swallowing
555    /// an operation for a resource that could never have been created in this
556    /// profile. Frames that always reach a service method (`Publish`, `Subscribe`,
557    /// `ConversationOpen`) are rejected by the front door's own method bodies and do
558    /// not consult this flag.
559    fn supports_channel_operations(&self) -> bool {
560        true
561    }
562
563    /// Whether `channel` admits `operation` right now.
564    ///
565    /// Consulted by the connection process BEFORE the operation is delegated, so
566    /// a roster refusal is typed at the moment of the decision, by the component
567    /// that made it, from the value it decided on. The alternative — classifying
568    /// an opaque failure afterwards by re-reading the roster in the error arm —
569    /// cannot tell "the roster refused this" from "something else failed while
570    /// the roster happened to change", and would put a confident wrong cause on
571    /// the wire.
572    ///
573    /// It does NOT replace the adapter's own inner check. Admission here is the
574    /// caller's guard; a service method is public and callable without a frame,
575    /// so it keeps its own.
576    ///
577    /// The default ADMITS. An adapter with no roster has nothing to say here and
578    /// its refusals travel as service errors exactly as they do today; only the
579    /// roster-owning adapter overrides this. A default body is also what keeps
580    /// this addition inside "minor": a method added without one breaks every
581    /// downstream implementor of a public trait.
582    ///
583    /// # Errors
584    /// Returns [`ChannelAccessError`] when the roster refuses the operation.
585    fn admit_channel(
586        &self,
587        _operation: ChannelOperation,
588        _channel: &str,
589    ) -> Result<(), ChannelAccessError> {
590        Ok(())
591    }
592}
593
594/// Default adapter from server wire frames to liminal channel/conversation APIs.
595#[derive(Debug)]
596pub struct LiminalConnectionServices {
597    publication_identity: Arc<publication_identity::ServerPublicationIdentity>,
598    agents: Option<Arc<crate::agents::runtime::Agents>>,
599    approval_agents: Option<Arc<crate::agents::waiting::durable::ApprovalAgents>>,
600    agent_delivery: Option<Arc<crate::agents::waiting::AgentDelivery>>,
601    site_lifecycle: OnceLock<Arc<crate::services::lifecycle::Lifecycle>>,
602    site_drafts: OnceLock<Arc<dyn SiteDraftHolder>>,
603    host_config: OnceLock<(crate::config::types::LysConfig, tokio::runtime::Handle)>,
604    draft_hook: OnceLock<std::sync::Weak<dyn crate::auth_lys::drafts::DraftDecisionHook>>,
605
606    /// The channel roster, keyed by channel name.
607    ///
608    /// Behind an [`RwLock`] over `Arc` values so a reader clones one pointer out
609    /// under the guard and works with it after the guard is released: no roster
610    /// read is ever held across a library call, and an entry handed out can
611    /// outlive a concurrent mutation of the map itself.
612    channels: RwLock<HashMap<String, Arc<ConfiguredChannel>>>,
613    export_subscriptions:
614        Mutex<Vec<std::sync::Weak<crate::link_bridge::subscriptions::ExportSubscriptions>>>,
615    /// The operator-declared bound on RUNTIME-registered channels
616    /// (`limits.max_channels`), carried verbatim from config.
617    ///
618    /// `None` is "the operator declared no bound", which refuses every runtime
619    /// registration rather than admitting an unbounded one — the roster's
620    /// aggregate size is otherwise unbounded the moment a registration API
621    /// exists. Boot-configured channels are never counted against it: they are
622    /// the bound the operator already wrote, in the file they wrote it in.
623    max_channels: Option<usize>,
624    cluster: ChannelCluster,
625    durable_store: Arc<dyn DurableStore>,
626    head_checkpoint_stop: Arc<liminal::durability::head_checkpoint_stop::HeadCheckpointStop>,
627    /// Reads the counters of every store these services built: the durable
628    /// store and, when it is a separate store, the ephemeral dedup store.
629    store_counts: StoreCountsHandle,
630    ephemeral_sweep: Option<EphemeralSweepCounts>,
631    signals: ServerSignals,
632    /// Complete participant service, installed only when semantic lifecycle
633    /// handling and its durable aggregate store are both ready.
634    participant_service: Option<InstalledParticipantService>,
635    /// The production handler's refused-load record, captured at construction.
636    ///
637    /// [`InstalledParticipantService`] holds its handler behind
638    /// `dyn ParticipantSemanticHandler`, so the concrete production handler is
639    /// no longer reachable once it is installed; the record is taken here, in
640    /// the one place the concrete handler exists, and carried out to the
641    /// startup path that publishes it onto the health endpoint. `None` is "no
642    /// participant is configured", which the operator surface reports as such
643    /// rather than as an empty refusal set.
644    unloadable_conversations: Option<UnloadableConversationRecord>,
645    /// The production handler as the A7 operator credential-re-issue authority
646    /// (§0.18), captured at construction for exactly the reason above: the
647    /// concrete handler is unreachable once erased behind
648    /// `dyn ParticipantSemanticHandler`, so the operator surface's handle on it
649    /// has to be taken in the one place it still exists. `None` is "no
650    /// participant is configured", which the route reports as such.
651    credential_reissuer: Option<Arc<dyn OperatorCredentialReissuer>>,
652    /// The participant warm's stop handle (DUR-020 R4, R5), captured at
653    /// construction for the same reason as the two above. The shutdown flush
654    /// stops the warm first, so no conversation begins restoring after the
655    /// server has begun to stop. `None` is "no participant is configured".
656    participant_warm: Option<ParticipantWarm>,
657    /// Dedup-on-delivery cache for publishes to **durable** channels. Keyed by
658    /// the per-message idempotency key carried on the publish frame; a duplicate
659    /// key is suppressed before fan-out so a subscriber receives it at most once.
660    ///
661    /// The namespace is stable across restarts by design (A1 §5): on a durable
662    /// channel the MESSAGE survives a crash too, so a surviving receipt is
663    /// truthful.
664    dedup: DedupCache,
665    /// Dedup-on-delivery cache for publishes to **ephemeral** channels
666    /// (A1 §5, graft §0.2 — the panel's sharpest crash hole).
667    ///
668    /// Identical machinery, different namespace: this one carries a per-boot
669    /// incarnation id, so a receipt and the message it vouches for share a
670    /// lifetime. Without it, `ephemeral + keyed publish -> Defer (buffered in
671    /// memory) -> complete_receipt (persisted) -> crash before the consumer
672    /// pops` loses the message while the receipt survives to suppress the
673    /// producer's retry — silent permanent loss of something the producer was
674    /// told was admitted. With it, the post-crash retry re-claims and
675    /// re-delivers, which is exactly ephemeral's at-least-once contract.
676    ///
677    /// This changes no dedup API. It changes only the namespace string the
678    /// server composes.
679    dedup_ephemeral: DedupCache,
680    /// Durable-channel idempotency keys whose publish outcome is not yet
681    /// known, held until their row is looked up by its key. A cache of the
682    /// durable in-flight claims, rebuilt from them when the services start.
683    held_keys: unresolved::HeldKeys,
684    /// The store history whose in-flight claims are still to be resolved:
685    /// some until the resolve has run, behind the listening door or at the
686    /// first keyed durable admission, whichever comes first (`unresolved_start.rs`).
687    claims_start: Mutex<Option<StoreHistory>>,
688    conversation_supervisor: Arc<ConversationSupervisor>,
689    /// Registered custom conversation responders, keyed by conversation subject.
690    ///
691    /// When a conversation is opened (`open_conversation`), the subject is looked
692    /// up here: a registered [`ParticipantBehaviour`] becomes the conversation's
693    /// participant; with no registration the local route refuses by name.
694    /// This is the seam aion #13 plugs a remote worker responder into. Interior
695    /// mutability is required because the services are shared behind `&self`.
696    responders: Mutex<ResponderRegistry>,
697    next_subscription_id: AtomicU64,
698    /// SRV-012: the console's held topology and tap.
699    console: Arc<crate::console::ConsoleHub>,
700}
701
702impl LiminalConnectionServices {
703    /// Builds library-backed services from validated server configuration.
704    ///
705    /// Durable-mode channels are backed by a shared haematite event store so
706    /// their publishes are persisted and survive the graceful-shutdown flush;
707    /// ephemeral channels carry no store.
708    ///
709    /// Full-only: a config selecting the worker-front-door profile is rejected at
710    /// entry (before any store is built), so this constructor can never build full
711    /// services for a profile that forbids them. Profile-aware callers go through
712    /// [`build_connection_services`] instead.
713    ///
714    /// # Errors
715    /// Returns [`ServerError`] when the config selects a non-full profile or a
716    /// configured channel cannot be initialized.
717    pub fn from_config(config: &ServerConfig) -> Result<Self, ServerError> {
718        Self::from_config_with_runtime(config, None)
719    }
720
721    pub(in crate::server) fn from_config_with_runtime(
722        config: &ServerConfig,
723        runtime: Option<tokio::runtime::Handle>,
724    ) -> Result<Self, ServerError> {
725        require_full_profile(config)?;
726        let path = required_persistence_path(config)?;
727        let history = StoreHistory::of(path);
728        let (subsystems, store) = crate::server::boot_steps::timed("store_open", || {
729            let subsystems = crate::server::boot_steps::timed("ephemeral_root_open", || {
730                RootedProductionSubsystems::new(path)
731            })?;
732            let store = subsystems.durable_store(path)?;
733            Ok::<_, ServerError>((subsystems, store))
734        })?;
735        crate::server::boot_steps::timed("channel_construction", || {
736            Self::from_config_with_store_and_runtime(config, store, history, &subsystems, runtime)
737        })
738    }
739
740    /// Builds services over a caller-provided durable store.
741    ///
742    /// Used by tests that need to inspect persisted state through the same store
743    /// handle the durable channels write to.
744    ///
745    /// Full-only: rejects a worker-front-door profile at entry, exactly like
746    /// [`Self::from_config`].
747    ///
748    /// # Errors
749    /// Returns [`ServerError`] when the config selects a non-full profile or a
750    /// configured channel cannot be initialized.
751    pub fn from_config_with_store(
752        config: &ServerConfig,
753        durable_store: Arc<dyn DurableStore>,
754    ) -> Result<Self, ServerError> {
755        require_full_profile(config)?;
756        #[cfg(any(test, feature = "test-support"))]
757        if config.persistence_path.is_none() {
758            return Self::from_config_with_store_via(
759                config,
760                durable_store,
761                StoreHistory::Recorded,
762                &ProductionSubsystems,
763            );
764        }
765        let subsystems = RootedProductionSubsystems::new(required_persistence_path(config)?)?;
766        Self::from_config_with_store_via(config, durable_store, StoreHistory::Recorded, &subsystems)
767    }
768
769    /// [`Self::from_config_with_store`] with the subsystem factory injected.
770    ///
771    /// The channel supervisor and conversation supervisor are constructed ONLY
772    /// through `subsystems` — there is no direct constructor call in this body —
773    /// so a factory that records as a side effect of constructing cannot have its
774    /// recording omitted (§9 D2 seam census, record-by-construction). No profile
775    /// check here: the caller (the public wrapper or the profile dispatch in
776    /// [`build_connection_services`]) has already established the full profile.
777    /// `history` says whether `durable_store` can already hold in-flight
778    /// claims for the services to look up before they judge a publish.
779    fn from_config_with_store_via(
780        config: &ServerConfig,
781        durable_store: Arc<dyn DurableStore>,
782        history: StoreHistory,
783        subsystems: &dyn SubsystemFactory,
784    ) -> Result<Self, ServerError> {
785        Self::from_config_with_store_and_runtime(config, durable_store, history, subsystems, None)
786    }
787
788    fn from_config_with_store_and_runtime(
789        config: &ServerConfig,
790        durable_store: Arc<dyn DurableStore>,
791        history: StoreHistory,
792        subsystems: &dyn SubsystemFactory,
793        runtime: Option<tokio::runtime::Handle>,
794    ) -> Result<Self, ServerError> {
795        let head_checkpoint_stop = Arc::new(
796            liminal::durability::head_checkpoint_stop::HeadCheckpointStop::new(durable_store),
797        );
798        let durable_store: Arc<dyn DurableStore> = head_checkpoint_stop.clone();
799        // Build ONE shared channel supervisor for the whole server. When a
800        // [cluster] section is present it is distribution-enabled, so every
801        // channel actor and subscriber shares the clustered scheduler the cluster
802        // attaches its process-group transport to (SRV-005, Constraint B).
803        let cluster = subsystems.channel_cluster(config.cluster.as_ref())?;
804        let mut channels = HashMap::new();
805        for channel in &config.channels {
806            // Resolve the channel's real JSON Schema (loaded from `schema_ref`
807            // during config validation) or the permissive empty schema when the
808            // channel declared none. The protocol schema id advertised at
809            // subscribe time is derived from the SAME schema bytes so an SDK
810            // deriving ids from schema bytes converges on it.
811            let resolved = resolve_channel_schema(channel);
812            let configured = build_configured_channel(
813                &channel.name,
814                resolved,
815                channel.durable,
816                ChannelOrigin::BootConfigured,
817                &durable_store,
818                cluster.supervisor(),
819            )
820            .map_err(|error| ServerError::ConfigValidation {
821                message: error.boot_message(&channel.name),
822            })?;
823            channels.insert(channel.name.clone(), Arc::new(configured));
824        }
825        let conversation_supervisor = subsystems.conversation_supervisor()?;
826        let dedup = history.delivery_dedup(Arc::clone(&durable_store));
827        // A1 §5: the ephemeral namespace is scoped to THIS incarnation, so an
828        // ephemeral receipt cannot outlive the in-memory message it vouches for.
829        // No earlier process used it, so it starts empty.
830        let ephemeral_store = ephemeral_dedup_store(&durable_store, history, subsystems);
831        let store_counts = combined_store_counts(&durable_store, &ephemeral_store);
832        let dedup_ephemeral = DedupCache::new_empty(ephemeral_store, ephemeral_dedup_namespace());
833        // Production participant activation (LP gap closure, Part B): the
834        // deployment's [participant] section installs the ONE production
835        // semantic handler, sealed together with the same durable store the
836        // conversation logs live in, under the configured wire-frame limit.
837        // No section, no service — the capability bit stays off and the
838        // connection path is byte-identical to the pre-activation build.
839        let signals = ServerSignals::default();
840        let installed_participant = config
841            .participant
842            .as_ref()
843            .map(|participant| install_participant(&durable_store, participant, &signals))
844            .transpose()?;
845        let device = config
846            .cluster
847            .as_ref()
848            .map_or("local", |cluster| cluster.node_name.as_str());
849        let console = Arc::new(console_hub(
850            device,
851            &channels,
852            config.persistence_path.is_some(),
853            config.auth.as_ref().and_then(|auth| auth.lys.as_ref()),
854        )?);
855        signals.install_console(Arc::clone(&console))?;
856        cluster.observer().set_console(Arc::clone(&console))?;
857        let (participant_service, unloadable_conversations, credential_reissuer, participant_warm) =
858            installed_participant.map_or((None, None, None, None), |installed| {
859                (
860                    Some(installed.service),
861                    Some(installed.unloadable_conversations),
862                    Some(installed.credential_reissuer),
863                    Some(installed.warm),
864                )
865            });
866        let (agents, approval_agents, agent_delivery) = Self::configured_agents(
867            config,
868            Arc::clone(&durable_store),
869            cluster.supervisor().clone(),
870            &console,
871            runtime,
872        )?;
873        let services = Self {
874            channels: RwLock::new(channels),
875            export_subscriptions: Mutex::new(Vec::new()),
876            max_channels: config.limits.max_channels,
877            cluster,
878            durable_store,
879            head_checkpoint_stop,
880            store_counts,
881            ephemeral_sweep: subsystems.ephemeral_sweep_counts(),
882            signals,
883            participant_service,
884            unloadable_conversations,
885            credential_reissuer,
886            participant_warm,
887            dedup,
888            dedup_ephemeral,
889            held_keys: unresolved::HeldKeys::default(),
890            // Every durable in-flight claim is looked up by its key before the
891            // services judge a keyed durable publish: behind the listening
892            // door, or by that publish itself when it comes first.
893            claims_start: Mutex::new(Some(history)),
894            conversation_supervisor,
895            responders: Mutex::new(HashMap::new()),
896            publication_identity: publication_identity::ServerPublicationIdentity::new()?,
897            next_subscription_id: AtomicU64::new(1),
898            console,
899            agents,
900            approval_agents,
901            agent_delivery,
902            site_lifecycle: OnceLock::new(),
903            site_drafts: OnceLock::new(),
904            host_config: OnceLock::new(),
905            draft_hook: OnceLock::new(),
906        };
907        services.install_roster_identity()?;
908        Ok(services)
909    }
910
911    /// Builds services with no configured channels.
912    ///
913    /// # Errors
914    /// Returns [`ServerError`] when the conversation supervisor scheduler cannot start.
915    #[cfg(any(test, feature = "test-support"))]
916    pub fn empty() -> Result<Self, ServerError> {
917        let conversation_supervisor = ProductionSubsystems.conversation_supervisor()?;
918        let store =
919            open_ephemeral(DEFAULT_SHARD_COUNT).map_err(|error| ServerError::ConfigValidation {
920                message: format!("failed to open test durable store: {error}"),
921            })?;
922        let head_checkpoint_stop = Arc::new(
923            liminal::durability::head_checkpoint_stop::HeadCheckpointStop::new(Arc::new(
924                CountingStore::new(store),
925            )),
926        );
927        let durable_store: Arc<dyn DurableStore> = head_checkpoint_stop.clone();
928        let store_counts = combined_store_counts(&durable_store, &durable_store);
929        let dedup = StoreHistory::Fresh.delivery_dedup(Arc::clone(&durable_store));
930        // A1 §5: the ephemeral namespace is scoped to THIS incarnation, so an
931        // ephemeral receipt cannot outlive the in-memory message it vouches for.
932        let dedup_ephemeral =
933            DedupCache::new_empty(Arc::clone(&durable_store), ephemeral_dedup_namespace());
934        let signals = ServerSignals::default();
935        let console = Arc::new(console_hub("local", &HashMap::new(), false, None)?);
936        signals.install_console(Arc::clone(&console))?;
937        let cluster = build_channel_cluster(None)?;
938        cluster.observer().set_console(Arc::clone(&console))?;
939        Ok(Self {
940            channels: RwLock::new(HashMap::new()),
941            export_subscriptions: Mutex::new(Vec::new()),
942            // No config, so no declared bound: this builder serves tests and
943            // callers with no channels at all, and a registration against it
944            // refuses `CapNotConfigured` exactly as an undeclared deployment's
945            // would.
946            max_channels: None,
947            cluster,
948            durable_store,
949            head_checkpoint_stop,
950            store_counts,
951            ephemeral_sweep: None,
952            signals,
953            participant_service: None,
954            unloadable_conversations: None,
955            credential_reissuer: None,
956            participant_warm: None,
957            dedup,
958            dedup_ephemeral,
959            held_keys: unresolved::HeldKeys::default(),
960            claims_start: Mutex::new(None),
961            conversation_supervisor,
962            responders: Mutex::new(HashMap::new()),
963            publication_identity: publication_identity::ServerPublicationIdentity::new()?,
964            next_subscription_id: AtomicU64::new(1),
965            console,
966            agents: None,
967            approval_agents: None,
968            agent_delivery: None,
969            site_lifecycle: OnceLock::new(),
970            site_drafts: OnceLock::new(),
971            host_config: OnceLock::new(),
972            draft_hook: OnceLock::new(),
973        })
974    }
975
976    /// The retained counts from the root's one startup sweep.
977    #[must_use]
978    pub const fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
979        self.ephemeral_sweep
980    }
981
982    /// The shared channel supervisor + cluster resolver backing this service.
983    ///
984    /// The server runtime uses this to attach the cluster to the channel
985    /// supervisor's clustered scheduler (SRV-005).
986    #[must_use]
987    pub const fn channel_cluster(&self) -> &ChannelCluster {
988        &self.cluster
989    }
990
991    /// Returns the shared durable store backing this service's durable channels.
992    #[must_use]
993    pub fn durable_store(&self) -> Arc<dyn DurableStore> {
994        Arc::clone(&self.durable_store)
995    }
996
997    /// The console's held topology and tap (SRV-012).
998    #[must_use]
999    pub fn console(&self) -> &crate::console::ConsoleHub {
1000        &self.console
1001    }
1002
1003    /// Applies a topology change; a poisoned model is counted by the hub
1004    /// (`topology_emit_failures`) and logged, never recovered.
1005    fn console_changed(&self, change: crate::console::TopologyChange) {
1006        self.signals.topology(change);
1007    }
1008
1009    /// Conversation subjects with a registered responder, sorted. Read from
1010    /// the in-memory registry; touches no store.
1011    ///
1012    /// # Errors
1013    /// Returns [`ServerError`] when the responder registry lock is poisoned.
1014    pub fn responder_subjects(&self) -> Result<Vec<String>, ServerError> {
1015        let responders = self.lock_responders()?;
1016        let mut subjects: Vec<String> = responders.keys().cloned().collect();
1017        drop(responders);
1018        subjects.sort();
1019        Ok(subjects)
1020    }
1021
1022    /// Restores every participant conversation the warm has not yet reached,
1023    /// on the caller's thread, and answers how many refused (DUR-020 R4).
1024    /// The store-count journeys call it so their windows do not race the
1025    /// warm thread; the server itself never waits for the warm.
1026    ///
1027    /// # Errors
1028    /// Returns [`ServerError`] when the pending-restore lock is poisoned.
1029    pub fn finish_participant_warm(&self) -> Result<usize, ServerError> {
1030        self.participant_warm.as_ref().map_or(Ok(0), |warm| {
1031            warm.finish()
1032                .map_err(|error| ServerError::ParticipantStartupRestore {
1033                    message: error.to_string(),
1034                })
1035        })
1036    }
1037
1038    /// Reads the store work these services have done since they were built,
1039    /// across every store they built.
1040    ///
1041    /// Empty (every snapshot zero) when the durable store was supplied by the
1042    /// caller and does not count.
1043    #[must_use]
1044    pub fn store_counts(&self) -> StoreCountsHandle {
1045        self.store_counts.clone()
1046    }
1047
1048    /// The production participant handler's refused-load record, when a
1049    /// participant is configured.
1050    ///
1051    /// The server's startup path publishes this onto the health endpoint so
1052    /// `GET /unloadable-conversations` answers from the same record the
1053    /// handler writes. `None` means no participant is configured at all, which
1054    /// the surface reports as a distinct state from "nothing was refused".
1055    #[must_use]
1056    pub fn unloadable_conversation_record(&self) -> Option<UnloadableConversationRecord> {
1057        self.unloadable_conversations.clone()
1058    }
1059
1060    /// The A7 operator credential-re-issue authority, when a participant is
1061    /// configured (§0.18).
1062    ///
1063    /// The startup path publishes this onto the health endpoint so
1064    /// `POST /operator/credential-reissue` reaches the SAME serialized
1065    /// participant-state point every wire operation does. `None` means no
1066    /// participant is configured, which the route reports as a distinct state
1067    /// from an unknown identity.
1068    #[must_use]
1069    pub fn credential_reissuer(&self) -> Option<Arc<dyn OperatorCredentialReissuer>> {
1070        self.credential_reissuer.clone()
1071    }
1072
1073    /// Installs a complete participant bundle in full-service supervisor tests.
1074    ///
1075    /// Production full services intentionally stay disabled until a concrete
1076    /// lifecycle handler exists. This consuming test builder exercises the real
1077    /// supervisor activation path without allowing an already-shared adapter to
1078    /// change capability posture.
1079    #[cfg(test)]
1080    #[must_use]
1081    pub(crate) fn with_participant_service(
1082        mut self,
1083        participant_service: InstalledParticipantService,
1084    ) -> Self {
1085        self.participant_service = Some(participant_service);
1086        self
1087    }
1088
1089    /// Returns the conversation supervisor backing supervised conversations.
1090    ///
1091    /// Tests use this to reach the underlying beamr scheduler so they can spawn
1092    /// or terminate participant processes and exercise crash detection.
1093    #[must_use]
1094    pub fn conversation_supervisor(&self) -> Arc<ConversationSupervisor> {
1095        Arc::clone(&self.conversation_supervisor)
1096    }
1097
1098    /// Registers a conversation responder for a routing `subject`.
1099    ///
1100    /// A later conversation on this exact subject runs the registered behaviour
1101    /// as a supervised linked participant. A registration replaces the previous
1102    /// behaviour and returns it to the caller.
1103    ///
1104    /// # Errors
1105    /// Returns [`ServerError`] when the responder registry lock is poisoned.
1106    pub fn register_responder(
1107        &self,
1108        subject: impl Into<String>,
1109        behaviour: Arc<dyn ParticipantBehaviour>,
1110    ) -> Result<Option<Arc<dyn ParticipantBehaviour>>, ServerError> {
1111        let subject = subject.into();
1112        let mut responders = self.lock_responders()?;
1113        let previous = responders.insert(subject.clone(), behaviour);
1114        drop(responders);
1115        self.console_changed(crate::console::TopologyChange::Service {
1116            subject,
1117            present: true,
1118        });
1119        Ok(previous)
1120    }
1121
1122    /// Removes the custom responder registered for `subject`, if any.
1123    ///
1124    /// After removal local calls on the subject are refused with `no_server`.
1125    /// Returns the removed behaviour when one was registered.
1126    ///
1127    /// # Errors
1128    /// Returns [`ServerError`] when the responder registry lock is poisoned.
1129    pub fn unregister_responder(
1130        &self,
1131        subject: &str,
1132    ) -> Result<Option<Arc<dyn ParticipantBehaviour>>, ServerError> {
1133        let mut responders = self.lock_responders()?;
1134        let removed = responders.remove(subject);
1135        drop(responders);
1136        if removed.is_some() {
1137            self.console_changed(crate::console::TopologyChange::Service {
1138                subject: subject.to_owned(),
1139                present: false,
1140            });
1141        }
1142        Ok(removed)
1143    }
1144
1145    /// Locks the responder registry, mapping a poisoned lock to a [`ServerError`]
1146    /// rather than panicking (the workspace denies `unwrap`/`expect`/`panic`).
1147    fn lock_responders(&self) -> Result<std::sync::MutexGuard<'_, ResponderRegistry>, ServerError> {
1148        self.responders
1149            .lock()
1150            .map_err(|_poisoned| ServerError::ListenerAccept {
1151                message: "responder registry lock poisoned".to_owned(),
1152            })
1153    }
1154
1155    /// Takes the channel roster's read guard, mapping a poisoned lock to a
1156    /// [`ServerError`] rather than panicking (the workspace denies
1157    /// `unwrap`/`expect`/`panic`), exactly as [`Self::lock_responders`] does for
1158    /// the responder registry.
1159    ///
1160    /// Every caller clones the `Arc` it needs out of the returned guard and drops
1161    /// the guard before doing anything else: the roster lock is never held across
1162    /// a call into the liminal library.
1163    fn read_channels(
1164        &self,
1165    ) -> Result<std::sync::RwLockReadGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>, ServerError>
1166    {
1167        self.channels
1168            .read()
1169            .map_err(|_poisoned| ServerError::ListenerAccept {
1170                message: "channel roster lock poisoned".to_owned(),
1171            })
1172    }
1173
1174    /// Subscribes to a configured channel and returns the raw library
1175    /// subscription handle so a test can drain the subscriber inbox directly and
1176    /// observe exactly which messages reached a subscriber.
1177    #[cfg(test)]
1178    pub(in crate::server) fn channel_handle_for_test(
1179        &self,
1180        channel: &str,
1181    ) -> Result<ChannelHandle, ServerError> {
1182        self.read_channels()?
1183            .get(channel)
1184            .map(|configured| configured.handle.clone())
1185            .ok_or_else(|| ServerError::ListenerAccept {
1186                message: format!("channel '{channel}' is not configured"),
1187            })
1188    }
1189
1190    #[cfg(test)]
1191    pub(crate) fn subscribe_handle_for_test(
1192        &self,
1193        channel: &str,
1194    ) -> Result<liminal::channel::SubscriptionHandle, ServerError> {
1195        self.subscribe_handle_for_test_with_install(channel, None)
1196    }
1197
1198    /// [`Self::subscribe_handle_for_test`] carrying an explicit install, so a
1199    /// test can put a subscriber under a REAL §5 regime (byte budget, fairness
1200    /// cap) and observe what a publish the bus dropped does to the dedup
1201    /// receipt.
1202    #[cfg(test)]
1203    pub(crate) fn subscribe_handle_for_test_with_install(
1204        &self,
1205        channel: &str,
1206        install: Option<liminal::channel::InboxInstall>,
1207    ) -> Result<liminal::channel::SubscriptionHandle, ServerError> {
1208        let channels = self.read_channels()?;
1209        let configured =
1210            channels
1211                .get(channel)
1212                .map(Arc::clone)
1213                .ok_or_else(|| ServerError::ListenerAccept {
1214                    message: format!("channel '{channel}' is not configured"),
1215                })?;
1216        drop(channels);
1217        install
1218            .map_or_else(
1219                || configured.handle.subscribe(),
1220                |install| configured.handle.subscribe_with_install(install),
1221            )
1222            .map_err(|error| ServerError::ListenerAccept {
1223                message: format!("liminal subscribe failed for channel '{channel}': {error}"),
1224            })
1225    }
1226
1227    /// Claims the delivery right for an idempotency key.
1228    ///
1229    /// Returns `Ok(true)` when this is the first publish for the key (the caller
1230    /// may deliver), and `Ok(false)` when the key was already claimed/completed (a
1231    /// duplicate the caller must suppress). The dedup cache is driven synchronously
1232    /// over the in-memory haematite store via the durable bridge.
1233    fn claim_delivery(cache: &DedupCache, key: &str) -> Result<bool, ServerError> {
1234        let decision = block_on(cache.claim_or_get(key, dedup_timestamp_millis()))
1235            .map_err(|error| ServerError::ListenerAccept {
1236                message: format!("dedup bridge failed for key '{key}': {error}"),
1237            })?
1238            .map_err(|error| ServerError::ListenerAccept {
1239                message: format!("dedup claim failed for key '{key}': {error}"),
1240            })?;
1241        Ok(matches!(decision, DedupDecision::Claimed))
1242    }
1243
1244    /// The dedup namespace a publish to `channel` belongs in (A1 §5).
1245    ///
1246    /// Durability decides, because durability is what determines whether a
1247    /// receipt can outlive the message it vouches for. A durable channel's
1248    /// message survives a crash, so its receipt may; an ephemeral channel's
1249    /// does not, so its receipt must not.
1250    const fn dedup_for(&self, channel: &ConfiguredChannel) -> &DedupCache {
1251        match channel.handle.config().mode {
1252            ChannelMode::Durable => &self.dedup,
1253            ChannelMode::Ephemeral => &self.dedup_ephemeral,
1254        }
1255    }
1256
1257    /// Releases a dangling in-flight dedup claim after a failed delivery.
1258    ///
1259    /// Best-effort: a release failure cannot mask the original publish error, so
1260    /// this returns nothing and logs at `error` level instead of surfacing. It is
1261    /// never silent — the leak (a permanently suppressed key) must be observable.
1262    /// `release_claim` itself never clobbers a stored receipt, so calling it on the
1263    /// failure path is safe even if a concurrent completion raced ahead.
1264    fn release_claim(cache: &DedupCache, key: &str) {
1265        match block_on(cache.release_claim(key)) {
1266            Ok(Ok(())) => {}
1267            Ok(Err(error)) => {
1268                tracing::error!(
1269                    idempotency_key = key,
1270                    %error,
1271                    "failed to release dedup claim after publish failure; key may stay suppressed"
1272                );
1273            }
1274            Err(error) => {
1275                tracing::error!(
1276                    idempotency_key = key,
1277                    %error,
1278                    "dedup release bridge failed after publish failure; key may stay suppressed"
1279                );
1280            }
1281        }
1282    }
1283
1284    /// The next connection-level message id.
1285    fn next_message_id(&self) -> Result<u64, ServerError> {
1286        self.publication_identity
1287            .next()
1288            .map(|message| message.id)
1289            .map_err(|error| ServerError::ListenerAccept {
1290                message: error.to_string(),
1291            })
1292    }
1293
1294    /// The outcome of a publish the channel took, recorded as accepted with
1295    /// its genuine subscriber deliveries. The delivered count (0 for an empty
1296    /// channel) is the same signal the delivery ack is derived from.
1297    fn delivered(delivery: &ChannelDelivery) -> Result<PublishOutcome, ServerError> {
1298        Self::record_delivery_counts(delivery);
1299        let message = delivery
1300            .delivery_id()
1301            .ok_or_else(|| ServerError::ListenerAccept {
1302                message: "publication identity missing after admission".into(),
1303            })?;
1304        Ok(PublishOutcome {
1305            message_id: message.id,
1306            delivered: delivery.is_delivered(),
1307            durable_position: delivery.durable_position(),
1308            reach: Reach::of(delivery),
1309        })
1310    }
1311}
1312
1313#[path = "deferred_store.rs"]
1314mod deferred_store;
1315#[path = "unresolved.rs"]
1316mod unresolved;
1317#[path = "unresolved_start.rs"]
1318mod unresolved_start;
1319#[cfg(test)]
1320#[path = "unresolved_start_tests.rs"]
1321mod unresolved_start_tests;
1322
1323use deferred_store::{DeferredStore, ThrowawayStoreOpener};
1324use unresolved_start::StoreHistory;
1325
1326/// The runtime channel-registration surface.
1327///
1328/// # Reachability: a test seam today, not a production capability
1329///
1330/// Production channels come exclusively from boot configuration; no production
1331/// code path calls `register_channel` (caller census at `c381f01`, G2 in
1332/// `docs/design/wire-durability-gaps-20260831.md` — the only callers are
1333/// `services_registry_tests.rs` and `apply_admission_tests.rs`). The method is
1334/// kept because its semantics — cap gating, typed identity refusals,
1335/// idempotent re-registration — are the contract a future runtime admission
1336/// path must satisfy, and the tests below pin that contract now. Wiring an
1337/// actual runtime admission route (an operator endpoint, a wire frame) is a
1338/// product decision that has NOT been taken; nothing here implies it. Reading
1339/// this method as an existing operator capability is the false-affordance
1340/// mistake this paragraph exists to prevent.
1341///
1342/// Inherent methods, not trait methods, and deliberately so: this is
1343/// authority-moving vocabulary that belongs to the ONE adapter owning a channel
1344/// roster. Putting `register`/`quiesce` on the public [`ConnectionServices`]
1345/// trait would break every external implementor and would hand
1346/// register/quiesce words to a profile that serves no channels at all.
1347///
1348/// The same seam as the existing runtime-mutation API on this type
1349/// (`register_responder`/`unregister_responder`): `&self`, interior mutability,
1350/// typed `Result`.
1351impl LiminalConnectionServices {
1352    /// Registers `spec` on the live roster.
1353    ///
1354    /// Test seam today: see the reachability note on this `impl` block — no
1355    /// production path reaches this method.
1356    ///
1357    /// Idempotent when an entry of that name already has an IDENTICAL
1358    /// configuration — mode, protocol schema id, and schema document, all three
1359    /// — and refuses typed, naming the first differing field, otherwise. An
1360    /// identical registration against a boot-configured entry answers
1361    /// [`Registered::AlreadyIdentical`] and leaves its origin alone: flipping it
1362    /// would make the entry lie about its restart fate and would move it into
1363    /// the counted population without a channel having been created.
1364    ///
1365    /// The cap is consulted first, in two steps with different reach. An absent
1366    /// `limits.max_channels` refuses EVERY call, identical or not, before the
1367    /// roster is read at all: a deployment that has declared no bound has not
1368    /// said what it admits, and unbounded-by-default is not a bound. A REACHED
1369    /// cap refuses only a call that would create an entry — it gates the insert,
1370    /// which is the population it bounds. An identical re-registration inserts
1371    /// nothing and so is answered `AlreadyIdentical` even at a full roster;
1372    /// refusing it would break idempotency at exactly the boundary a projector
1373    /// re-projecting its record crosses.
1374    ///
1375    /// # Errors
1376    /// Returns [`ChannelRegistryError`] when no cap is configured, the cap is
1377    /// reached, the name exists with a different configuration, the schema bytes
1378    /// do not parse or compile, durable initialization over the shared store
1379    /// fails, or the roster lock is poisoned.
1380    pub fn register_channel(
1381        &self,
1382        spec: &ChannelRegistration,
1383    ) -> Result<Registered, ChannelRegistryError> {
1384        // The cap is consulted before the roster is touched: an undeclared bound
1385        // refuses every runtime registration, identical or not, because the
1386        // deployment has not said what it admits.
1387        let Some(limit) = self.max_channels else {
1388            return Err(ChannelRegistryError::CapNotConfigured {
1389                cap: MAX_CHANNELS_KEY,
1390            });
1391        };
1392        // Schema resolution is pure (a JSON parse and a digest), so it runs with
1393        // no lock held and its result serves BOTH the identity comparison and
1394        // the construction below.
1395        let resolved = resolve_schema_bytes(spec.schema_bytes.as_deref()).map_err(|error| {
1396            ChannelRegistryError::SchemaRejected {
1397                name: spec.name.clone(),
1398                message: error.to_string(),
1399            }
1400        })?;
1401
1402        // Fast paths, under a READ lock: an already-identical registration
1403        // builds nothing, and a full roster refuses before paying for a
1404        // construction it would discard. Neither is the authority — the write
1405        // lock below re-decides both, because the roster can move in between.
1406        let (existing, registered_count) = {
1407            let channels = self.read_roster()?;
1408            let existing = channels.get(&spec.name).map(Arc::clone);
1409            let registered_count = runtime_registered_count(&channels);
1410            drop(channels);
1411            (existing, registered_count)
1412        };
1413        if let Some(existing) = existing {
1414            return compare_registration(&existing, spec, &resolved);
1415        }
1416        if registered_count >= limit {
1417            return Err(ChannelRegistryError::CapReached {
1418                cap: MAX_CHANNELS_KEY,
1419                limit,
1420            });
1421        }
1422
1423        // Construction runs with NO lock held. A durable channel recovers its
1424        // per-partition sequence counters from the store here, which is O(stream
1425        // length) in store reads; holding the roster lock across it would put a
1426        // slow store walk on every connection's publish and subscribe path.
1427        let configured = build_configured_channel(
1428            &spec.name,
1429            resolved,
1430            spec.durable,
1431            ChannelOrigin::RuntimeRegistered,
1432            &self.durable_store,
1433            self.cluster.supervisor(),
1434        )
1435        .map_err(|error| error.into_registry_error(&spec.name))?;
1436
1437        // The authoritative decision: identity, cap, and insert under ONE write
1438        // lock, so the count that admitted the entry and the insert that added
1439        // it cannot be separated by a concurrent registration.
1440        let mut exporters = self.export_subscriptions.lock().map_err(|_| {
1441            ChannelRegistryError::RosterUnavailable {
1442                message: "export subscription registry lock poisoned".to_owned(),
1443            }
1444        })?;
1445        exporters.retain(|exporter| exporter.strong_count() > 0);
1446        let mut channels = self.write_roster()?;
1447        let raced = channels.get(&spec.name).map(Arc::clone);
1448        let registered_count = runtime_registered_count(&channels);
1449        if raced.is_none() && registered_count < limit {
1450            let mut prepared = Vec::new();
1451            for exporter in exporters.iter().filter_map(std::sync::Weak::upgrade) {
1452                if let Some(binding) = exporter.binding(&spec.name) {
1453                    let refused = |error: ServerError| ChannelRegistryError::RosterUnavailable {
1454                        message: format!(
1455                            "export '{}' cannot subscribe to '{}': {error}",
1456                            exporter.link(),
1457                            spec.name
1458                        ),
1459                    };
1460                    let subscription = self
1461                        .build_subscription(
1462                            &spec.name,
1463                            &configured.handle,
1464                            configured.protocol_schema,
1465                            Some(binding.install),
1466                            Some(&binding.consumer),
1467                        )
1468                        .map_err(refused)?;
1469                    let subscription =
1470                        crate::link_bridge::subscriptions::PreparedSubscription::new(
1471                            binding.prefix,
1472                            spec.name.clone(),
1473                            subscription,
1474                        )
1475                        .map_err(refused)?;
1476                    prepared.push((exporter, subscription));
1477                }
1478            }
1479            // Bind every exporter before a publish can find the new roster entry.
1480            for (exporter, subscription) in prepared {
1481                exporter.attach(subscription);
1482            }
1483            self.identify_registered(&configured)?;
1484            channels.insert(spec.name.clone(), Arc::new(configured));
1485            drop(channels);
1486            drop(exporters);
1487            self.console_changed(crate::console::TopologyChange::Channel {
1488                name: spec.name.clone(),
1489                life: line_life(spec.durable),
1490                state: crate::console::records::LineState::Live,
1491            });
1492            return Ok(Registered::Created);
1493        }
1494        drop(channels);
1495        drop(exporters);
1496        if let Some(raced) = raced {
1497            // A racer registered this name while the channel above was being
1498            // built. The built entry is dropped unused: it owns no actor (the
1499            // actor is spawned lazily on first use) and its durable
1500            // construction only READ the store, so discarding it changes
1501            // nothing an observer could see.
1502            return compare_registration(&raced, spec, &schema_of(&configured));
1503        }
1504        Err(ChannelRegistryError::CapReached {
1505            cap: MAX_CHANNELS_KEY,
1506            limit,
1507        })
1508    }
1509
1510    /// Moves `name` from active to quiesced with a named `reason`. ONE-WAY.
1511    ///
1512    /// New publishes and new subscribes are refused afterwards, carrying the
1513    /// reason. Existing subscriptions are UNTOUCHED: nothing revokes a
1514    /// subscription handle, the actor's subscriber list is not walked, no EXIT
1515    /// is sent, and the channel actor keeps running. Quiesce is a roster-level
1516    /// admission decision, not an actor command.
1517    ///
1518    /// Re-quiescing under the IDENTICAL reason is `Ok(())`; a DIFFERENT reason
1519    /// refuses, carrying the reason already on record.
1520    ///
1521    /// The return does NOT mean "no new subscriber can appear". A subscribe that
1522    /// has already passed admission completes and gets its stream — the
1523    /// linearisation point is the admission read, not the subscribe's
1524    /// completion. A consumer that needs "nobody is attached" must observe
1525    /// attachment directly.
1526    ///
1527    /// # Errors
1528    /// Returns [`ChannelRegistryError`] when the name is not registered, is
1529    /// already quiesced under a different reason, or the roster lock is
1530    /// poisoned.
1531    pub fn quiesce_channel(
1532        &self,
1533        name: &str,
1534        reason: impl Into<String>,
1535    ) -> Result<(), ChannelRegistryError> {
1536        let reason = reason.into();
1537        // A READ lock: the state machine lives on the ENTRY, not in the map, so
1538        // the operation the design most wants to be safe never blocks a reader.
1539        let configured = {
1540            let channels = self.read_roster()?;
1541            channels.get(name).map(Arc::clone).ok_or_else(|| {
1542                ChannelRegistryError::NotRegistered {
1543                    name: name.to_owned(),
1544                }
1545            })?
1546        };
1547        configured.quiesce(name, &reason)?;
1548        self.console_changed(crate::console::TopologyChange::Channel {
1549            name: name.to_owned(),
1550            life: line_life(configured.durable),
1551            state: crate::console::records::LineState::Quiesced,
1552        });
1553        Ok(())
1554    }
1555
1556    /// Cheap typed probe: one roster read plus one atomic load.
1557    ///
1558    /// Touches no actor and therefore CANNOT spawn one. Every handle accessor
1559    /// that could answer a question about a channel's activity routes through
1560    /// the lazy-spawn path, so a probe built on one would materialise the actor
1561    /// of the idle channel it was asked about — turning a read into a side
1562    /// effect. This reads the roster entry's own recorded fields and nothing
1563    /// else.
1564    ///
1565    /// # Errors
1566    /// Returns [`ChannelRegistryError::RosterUnavailable`] only.
1567    pub fn channel_status(&self, name: &str) -> Result<ChannelStatus, ChannelRegistryError> {
1568        let configured = {
1569            let channels = self.read_roster()?;
1570            channels.get(name).map(Arc::clone)
1571        };
1572        Ok(
1573            configured.map_or(ChannelStatus::NotRegistered, |configured| {
1574                status_of(&configured)
1575            }),
1576        )
1577    }
1578
1579    /// Every roster entry with its status, sorted by name, from ONE roster
1580    /// read (SRV-012 R7: the console's topology snapshot).
1581    ///
1582    /// # Errors
1583    /// Returns [`ChannelRegistryError::RosterUnavailable`] when the roster
1584    /// lock is poisoned.
1585    pub fn channel_census(&self) -> Result<Vec<(String, ChannelStatus)>, ChannelRegistryError> {
1586        let mut census: Vec<(String, ChannelStatus)> = self
1587            .read_roster()?
1588            .iter()
1589            .map(|(name, configured)| (name.clone(), status_of(configured)))
1590            .collect();
1591        census.sort_by(|left, right| left.0.cmp(&right.0));
1592        Ok(census)
1593    }
1594
1595    /// The whole roster: one minimal descriptor per entry, sorted by name.
1596    ///
1597    /// The census companion to [`Self::channel_status`]. A by-name probe answers
1598    /// about a name the caller already suspects; only an enumeration can reveal
1599    /// a name the caller does not know to ask about, and a verification sweep
1600    /// with no population denominator is an instrument shape this estate
1601    /// forbids. Touches no actor, under the same constraint as the probe.
1602    ///
1603    /// # Errors
1604    /// Returns [`ChannelRegistryError::RosterUnavailable`] only.
1605    pub fn registered_channels(&self) -> Result<Vec<ChannelDescriptor>, ChannelRegistryError> {
1606        let entries: Vec<(String, Arc<ConfiguredChannel>)> = {
1607            let channels = self.read_roster()?;
1608            channels
1609                .iter()
1610                .map(|(name, configured)| (name.clone(), Arc::clone(configured)))
1611                .collect()
1612        };
1613        let mut descriptors: Vec<ChannelDescriptor> = entries
1614            .into_iter()
1615            .map(|(name, configured)| ChannelDescriptor {
1616                name,
1617                origin: configured.origin,
1618                state: configured.state(),
1619            })
1620            .collect();
1621        descriptors.sort_by(|left, right| left.name.cmp(&right.name));
1622        Ok(descriptors)
1623    }
1624
1625    /// The roster admission funnel: the ONE place a channel operation's
1626    /// permission is decided.
1627    ///
1628    /// Returns the admitted entry, so the caller works from the value the
1629    /// decision was made on rather than reading the roster a second time and
1630    /// risking a different answer. This read is the LINEARISATION POINT for the
1631    /// quiesce race: everything the caller does with the returned entry happens
1632    /// outside the lock and is not re-checked, which is exactly why a quiesce
1633    /// that commits after this returns does not stop the operation it admitted.
1634    ///
1635    /// # Errors
1636    /// Returns [`ChannelAccessError`] when the channel is absent, quiesced, or
1637    /// the roster lock is poisoned.
1638    fn admit_channel(&self, channel: &str) -> Result<Arc<ConfiguredChannel>, ChannelAccessError> {
1639        let configured = {
1640            let channels = self.channels.read().map_err(|_poisoned| {
1641                ChannelAccessError::RosterUnavailable {
1642                    message: ROSTER_POISONED.to_owned(),
1643                }
1644            })?;
1645            channels.get(channel).map(Arc::clone)
1646        };
1647        let configured = configured.ok_or_else(|| ChannelAccessError::NotRegistered {
1648            name: channel.to_owned(),
1649        })?;
1650        if configured.state.load(Ordering::Acquire) == STATE_QUIESCED {
1651            return Err(ChannelAccessError::Quiesced {
1652                name: channel.to_owned(),
1653                reason: configured.recorded_quiesce_reason(),
1654            });
1655        }
1656        Ok(configured)
1657    }
1658
1659    /// Takes the roster's READ guard for the registration surface, mapping a
1660    /// poisoned lock to a typed [`ChannelRegistryError`].
1661    ///
1662    /// Separate from [`Self::read_channels`] because the two surfaces answer
1663    /// different callers with different error types; recovering one from the
1664    /// other would mean inspecting a message.
1665    fn read_roster(
1666        &self,
1667    ) -> Result<
1668        std::sync::RwLockReadGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>,
1669        ChannelRegistryError,
1670    > {
1671        #[cfg(test)]
1672        ROSTER_READS.with(|reads| reads.set(reads.get() + 1));
1673        self.channels
1674            .read()
1675            .map_err(|_poisoned| ChannelRegistryError::RosterUnavailable {
1676                message: ROSTER_POISONED.to_owned(),
1677            })
1678    }
1679
1680    /// Takes the roster's WRITE guard — held only for the check-and-insert that
1681    /// must be atomic, never across a call into the liminal library.
1682    fn write_roster(
1683        &self,
1684    ) -> Result<
1685        std::sync::RwLockWriteGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>,
1686        ChannelRegistryError,
1687    > {
1688        self.channels
1689            .write()
1690            .map_err(|_poisoned| ChannelRegistryError::RosterUnavailable {
1691                message: ROSTER_POISONED.to_owned(),
1692            })
1693    }
1694
1695    /// Gives back the tap slot a subscribe to `channel` reserved at
1696    /// admission, taking its tap channel off the roster when no tap stands
1697    /// behind it (SRV-012 R6).
1698    fn give_back_tap(&self, channel: &str) {
1699        self.console.release_tap(channel, |subject| {
1700            crate::console::ServerPublisher::retire_console_channel(self, subject)
1701        });
1702    }
1703
1704    /// The trait's subscribe, from the console guard to the subscription;
1705    /// [`ConnectionServices::subscribe`] gives back the tap reservation when
1706    /// this fails.
1707    fn open_subscription(
1708        &self,
1709        channel: &str,
1710        accepted_schemas: &[ProtocolSchemaId],
1711        install: Option<liminal::channel::InboxInstall>,
1712        consumer: Option<&str>,
1713    ) -> Result<ConnectionSubscription, ServerError> {
1714        // The frame path refused this already at admission; this guard is for
1715        // a caller holding the trait directly.
1716        refuse_console_tap_target(channel).map_err(|error| access_to_server_error(&error))?;
1717        // As in `publish`: the admission funnel clones the entry's `Arc` out
1718        // under the read guard and the guard is released before schema
1719        // negotiation or the actor round-trip below. That release is what makes
1720        // this the linearisation point — a quiesce landing after it does not
1721        // stop the subscription this call is already building.
1722        let configured = self
1723            .admit_channel(channel)
1724            .map_err(|error| access_to_server_error(&error))?;
1725        let selected_schema = if accepted_schemas.is_empty() {
1726            configured.protocol_schema
1727        } else {
1728            liminal::protocol::negotiate_schema(configured.protocol_schema, accepted_schemas)
1729                .map_err(|error| server_error_from_protocol(&error))?
1730        };
1731        self.build_subscription(
1732            channel,
1733            &configured.handle,
1734            selected_schema,
1735            install,
1736            consumer,
1737        )
1738    }
1739
1740    fn build_subscription(
1741        &self,
1742        channel: &str,
1743        handle: &ChannelHandle,
1744        selected_schema: ProtocolSchemaId,
1745        install: Option<liminal::channel::InboxInstall>,
1746        consumer: Option<&str>,
1747    ) -> Result<ConnectionSubscription, ServerError> {
1748        // `subscribe_with_install` installs the §5 budget/fairness cap and the R3
1749        // wake notifier on the inbox at construction — strictly before the
1750        // registration is published to the channel actor — so there is no window
1751        // in which a publish can land uncharged or without a wake.
1752        let subscription = match (consumer, install) {
1753            (Some(consumer), Some(install)) => handle.subscribe_consumer(consumer, install),
1754            (Some(_), None) => {
1755                return Err(ServerError::ListenerAccept {
1756                    message: "a durable consumer requires a bounded inbox install".to_owned(),
1757                });
1758            }
1759            (None, Some(install)) => handle.subscribe_with_install(install),
1760            (None, None) => handle.subscribe(),
1761        }
1762        .map_err(|error| ServerError::ListenerAccept {
1763            message: format!("liminal subscribe failed for channel '{channel}': {error}"),
1764        })?;
1765        let id = self.next_subscription_id.fetch_add(1, Ordering::Relaxed);
1766        let mut connection_subscription = ConnectionSubscription::new(
1767            id,
1768            selected_schema,
1769            Box::new(LiminalSubscriptionResource { subscription }),
1770        );
1771        connection_subscription.set_channel(channel.to_owned());
1772        // SRV-012 R6: the console tap attaches and detaches through the
1773        // supervisor's observer, which hears every subscribe and every
1774        // departure once (ServerObserver).
1775        Ok(connection_subscription)
1776    }
1777
1778    pub(crate) fn subscribe_consumer(
1779        &self,
1780        channel: &str,
1781        consumer: &str,
1782        install: liminal::channel::InboxInstall,
1783    ) -> Result<ConnectionSubscription, ServerError> {
1784        self.open_subscription(channel, &[], Some(install), Some(consumer))
1785    }
1786
1787    pub(crate) fn install_exporter(
1788        &self,
1789        exporter: &Arc<crate::link_bridge::subscriptions::ExportSubscriptions>,
1790    ) -> Result<(), ServerError> {
1791        let mut exporters =
1792            self.export_subscriptions
1793                .lock()
1794                .map_err(|_| ServerError::ListenerAccept {
1795                    message: "export subscription registry lock poisoned".to_owned(),
1796                })?;
1797        exporters.retain(|exporter| exporter.strong_count() > 0);
1798        if exporters
1799            .iter()
1800            .filter_map(std::sync::Weak::upgrade)
1801            .any(|existing| existing.link() == exporter.link())
1802        {
1803            return Err(ServerError::ListenerAccept {
1804                message: format!(
1805                    "link '{}' already owns an export subscription",
1806                    exporter.link()
1807                ),
1808            });
1809        }
1810        let mut prepared = Vec::new();
1811        for channel in self
1812            .registered_channels()
1813            .map_err(|error| ServerError::ListenerAccept {
1814                message: error.to_string(),
1815            })?
1816        {
1817            if matches!(channel.state, ChannelState::Active) {
1818                if let Some(binding) = exporter.binding(&channel.name) {
1819                    let subscription =
1820                        self.subscribe_consumer(&channel.name, &binding.consumer, binding.install)?;
1821                    prepared.push(
1822                        crate::link_bridge::subscriptions::PreparedSubscription::new(
1823                            binding.prefix,
1824                            channel.name,
1825                            subscription,
1826                        )?,
1827                    );
1828                }
1829            }
1830        }
1831        for subscription in prepared {
1832            exporter.attach(subscription);
1833        }
1834        exporters.push(Arc::downgrade(exporter));
1835        drop(exporters);
1836        Ok(())
1837    }
1838
1839    /// Refuses a subscribe that would start one more live tap than
1840    /// [`crate::console::MAX_LIVE_TAPS`] (SRV-012 R6); a subscriber to a tap
1841    /// already live is not a new tap.
1842    ///
1843    /// A poisoned tap table does not refuse the subscribe (R8: a console
1844    /// bookkeeping failure never fails the client operation it watches); the
1845    /// attach that follows meets the poison and counts it.
1846    ///
1847    /// # Errors
1848    /// Returns [`ChannelAccessError::ConsoleTapLimit`] naming the channel.
1849    fn refuse_console_tap_limit(&self, channel: &str) -> Result<(), ChannelAccessError> {
1850        match self.console.reserve_tap(channel) {
1851            Ok(false) => Err(ChannelAccessError::ConsoleTapLimit {
1852                name: channel.to_owned(),
1853                limit: crate::console::MAX_LIVE_TAPS,
1854            }),
1855            Ok(true) | Err(_) => Ok(()),
1856        }
1857    }
1858}
1859
1860impl crate::console::ServerPublisher for LiminalConnectionServices {
1861    /// The server's own record on the one publish path, with a server origin:
1862    /// attributed to `liminal.<device>`, and never counted as a connection's
1863    /// work by the cost tick.
1864    fn publish_console(
1865        &self,
1866        subject: &str,
1867        payload: &[u8],
1868    ) -> Result<(), crate::console::ConsoleError> {
1869        let payload = self.agent_topology_payload(subject, payload)?;
1870        let payload = if subject == crate::console::TOPOLOGY_SUBJECT {
1871            if let Some(lifecycle) = self.site_lifecycle.get() {
1872                std::borrow::Cow::Owned(
1873                    crate::console::topology::with_sites(
1874                        &payload,
1875                        lifecycle,
1876                        self.console.device(),
1877                    )
1878                    .map_err(|error| crate::console::ConsoleError::Server {
1879                        subject: subject.into(),
1880                        source: site_services::site_error(error),
1881                    })?,
1882                )
1883            } else {
1884                payload
1885            }
1886        } else {
1887            payload
1888        };
1889        let envelope = MessageEnvelope::new(
1890            ProtocolSchemaId::new([0_u8; ProtocolSchemaId::WIRE_LEN]),
1891            liminal::protocol::CausalContext::independent(),
1892            payload.to_vec(),
1893        );
1894        self.publish_with_origin(
1895            subject,
1896            &envelope,
1897            None,
1898            PublishOrigin::Server {
1899                device: self.console.device(),
1900            },
1901        )
1902        .map(|_published| ())
1903        .map_err(|source| crate::console::ConsoleError::Server {
1904            subject: subject.to_owned(),
1905            source,
1906        })
1907    }
1908
1909    /// Takes a `console.tap.<target>` channel off the roster (SRV-012 R6).
1910    /// Only a console subject is ever removed here.
1911    fn retire_console_channel(&self, subject: &str) -> Result<(), crate::console::ConsoleError> {
1912        if !super::channel_registry::is_console_subject(subject) {
1913            return Ok(());
1914        }
1915        let mut channels =
1916            self.write_roster()
1917                .map_err(|error| crate::console::ConsoleError::Server {
1918                    subject: subject.to_owned(),
1919                    source: ServerError::ListenerAccept {
1920                        message: error.to_string(),
1921                    },
1922                })?;
1923        channels.remove(subject);
1924        drop(channels);
1925        Ok(())
1926    }
1927}
1928
1929/// The production participant service and the operator surfaces taken from
1930/// its one live handler before it is erased behind
1931/// `dyn ParticipantSemanticHandler`.
1932struct InstalledParticipant {
1933    service: InstalledParticipantService,
1934    unloadable_conversations: UnloadableConversationRecord,
1935    credential_reissuer: Arc<dyn OperatorCredentialReissuer>,
1936    warm: ParticipantWarm,
1937}
1938
1939/// Production participant activation (LP gap closure, Part B): the
1940/// deployment's [participant] section installs the ONE production semantic
1941/// handler, sealed together with the same durable store the conversation logs
1942/// live in, under the configured wire-frame limit.
1943///
1944/// # Errors
1945/// Returns [`ServerError::ParticipantStartupRestore`] when the handler cannot
1946/// be built or wired, and [`ServerError::ConfigValidation`] when the
1947/// wire-frame limit is below the protocol's minimum participant frame.
1948fn install_participant(
1949    durable_store: &Arc<dyn DurableStore>,
1950    participant: &ParticipantConfig,
1951    signals: &ServerSignals,
1952) -> Result<InstalledParticipant, ServerError> {
1953    let handler = ProductionParticipantHandler::new(Arc::clone(durable_store), *participant)
1954        .map_err(|error| ServerError::ParticipantStartupRestore {
1955            message: error.to_string(),
1956        })?;
1957    // Taken BEFORE the handler is erased behind
1958    // `dyn ParticipantSemanticHandler`. Construction restores no
1959    // conversation (DUR-020), so the record fills as conversations restore,
1960    // on first touch or in the warm, and is complete once the warm has
1961    // finished.
1962    handler.install_signals(signals.clone()).map_err(|error| {
1963        ServerError::ParticipantStartupRestore {
1964            message: error.to_string(),
1965        }
1966    })?;
1967    let unloadable_conversations = handler.unloadable_record();
1968    let handler = Arc::new(handler);
1969    let warm = handler
1970        .install_warm()
1971        .map_err(|error| ServerError::ParticipantStartupRestore {
1972            message: error.to_string(),
1973        })?;
1974    // A7 (§0.18): taken here for the same reason, and from the same one live
1975    // handler, so the operator surface and the participant wire cannot end up
1976    // talking to two different owners.
1977    let credential_reissuer: Arc<dyn OperatorCredentialReissuer> =
1978        Arc::<ProductionParticipantHandler>::clone(&handler);
1979    let service = InstalledParticipantService::new(
1980        handler,
1981        Arc::clone(durable_store),
1982        participant.wire_frame_limit,
1983    )
1984    .map_err(|error| ServerError::ConfigValidation {
1985        message: format!(
1986            "participant.wire_frame_limit: {} is below the protocol's minimum \
1987             complete participant frame ({error:?})",
1988            participant.wire_frame_limit
1989        ),
1990    })?;
1991    Ok(InstalledParticipant {
1992        service,
1993        unloadable_conversations,
1994        credential_reissuer,
1995        warm,
1996    })
1997}
1998
1999/// The console hub for `device`, its model built from the roster as it
2000/// stands (responders, connections and peers arrive as changes).
2001///
2002/// # Errors
2003/// Returns [`ServerError::ListenerAccept`] when the device is blank or the
2004/// wall clock cannot be read.
2005fn console_hub(
2006    device: &str,
2007    channels: &HashMap<String, Arc<ConfiguredChannel>>,
2008    depot: bool,
2009    lys: Option<&crate::config::types::LysConfig>,
2010) -> Result<crate::console::ConsoleHub, ServerError> {
2011    let this = crate::console::ThisDevice {
2012        id: device.to_owned(),
2013        liminal: env!("CARGO_PKG_VERSION").to_owned(),
2014        started: Instant::now(),
2015        depot,
2016    };
2017    let lines = channels.iter().map(|(name, configured)| {
2018        let state = match configured.state() {
2019            ChannelState::Active => crate::console::records::LineState::Live,
2020            ChannelState::Quiesced { .. } => crate::console::records::LineState::Quiesced,
2021        };
2022        (name.clone(), line_life(configured.durable), state)
2023    });
2024    let mut model = crate::console::TopologyModel::new(this, [], lines, [], []);
2025    if let Some(machine) = lys.and_then(|lys| lys.machine.as_ref()) {
2026        model = model.with_machine(machine);
2027    }
2028    crate::console::wall::WallBase::now()
2029        .and_then(|wall| crate::console::ConsoleHub::new(device, model, wall))
2030        .map_err(|error| ServerError::ListenerAccept {
2031            message: format!("console hub: {error}"),
2032        })
2033}
2034
2035/// A roster entry's status as [`LiminalConnectionServices::channel_status`]
2036/// answers it.
2037fn status_of(configured: &ConfiguredChannel) -> ChannelStatus {
2038    let mode = configured.handle.config().mode;
2039    match configured.state() {
2040        ChannelState::Active => ChannelStatus::Active {
2041            origin: configured.origin,
2042            mode,
2043            schema: configured.protocol_schema,
2044        },
2045        ChannelState::Quiesced { reason } => ChannelStatus::Quiesced {
2046            reason,
2047            origin: configured.origin,
2048            mode,
2049        },
2050    }
2051}
2052
2053#[cfg(test)]
2054thread_local! {
2055    /// Roster read locks taken on this thread. Test-only.
2056    static ROSTER_READS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
2057}
2058
2059/// Roster read locks taken on the calling thread so far. Test-only.
2060#[cfg(test)]
2061pub(crate) fn roster_reads() -> u64 {
2062    ROSTER_READS.with(std::cell::Cell::get)
2063}
2064
2065/// A channel's line life: durable channels keep, ephemeral ones pass.
2066const fn line_life(durable: bool) -> crate::console::records::LineLife {
2067    if durable {
2068        crate::console::records::LineLife::Kept
2069    } else {
2070        crate::console::records::LineLife::Passing
2071    }
2072}
2073
2074/// The diagnostic carried when the roster lock is poisoned. One string, shared
2075/// by every surface that reports it, so the wording cannot drift between them.
2076const ROSTER_POISONED: &str = "channel roster lock poisoned";
2077
2078/// How many roster entries the registration cap counts.
2079///
2080/// Runtime-registered entries ONLY. Boot-configured channels are the operator's
2081/// own authored bound — they are in the file the operator wrote — and counting
2082/// them would make one number mean two different things depending on how the
2083/// deployment was configured. The origin never flips, so this population is
2084/// well defined over time.
2085fn runtime_registered_count(channels: &HashMap<String, Arc<ConfiguredChannel>>) -> usize {
2086    channels
2087        .values()
2088        .filter(|configured| configured.origin == ChannelOrigin::RuntimeRegistered)
2089        .count()
2090}
2091
2092/// The three-field identity comparison behind idempotent-if-identical
2093/// registration.
2094///
2095/// All three must match; the FIRST mismatch is reported by name. The name itself
2096/// is not compared — it is the roster key, a precondition of the comparison
2097/// rather than a member of it — and neither are the supervisor or the durable
2098/// store, which are one server-wide instance each and cannot differ between two
2099/// registrations in one process.
2100///
2101/// The schema is compared as BOTH its protocol id and its parsed document, on
2102/// purpose. The id is a 64-bit non-cryptographic digest, so the document guards
2103/// against a collision accepting a different schema as identical; and the two
2104/// fail in opposite directions, because two byte sequences that parse to the
2105/// same document but differ in whitespace produce different ids — which must
2106/// refuse, since the id is what every future subscriber negotiates.
2107///
2108/// The channel's own `Schema` is not compared: it is not `PartialEq`, and its
2109/// identifier is a fresh value per construction, so comparing it would refuse
2110/// every idempotent re-registration.
2111fn compare_registration(
2112    existing: &ConfiguredChannel,
2113    spec: &ChannelRegistration,
2114    resolved: &ChannelSchema,
2115) -> Result<Registered, ChannelRegistryError> {
2116    let requested_mode = if spec.durable {
2117        ChannelMode::Durable
2118    } else {
2119        ChannelMode::Ephemeral
2120    };
2121    let config = existing.handle.config();
2122    let mismatch = if config.mode == requested_mode {
2123        if existing.protocol_schema == resolved.protocol_id {
2124            if *config.schema.definition() == resolved.document {
2125                None
2126            } else {
2127                Some(ChannelConfigField::SchemaDocument)
2128            }
2129        } else {
2130            Some(ChannelConfigField::SchemaId)
2131        }
2132    } else {
2133        Some(ChannelConfigField::Mode)
2134    };
2135    mismatch.map_or(Ok(Registered::AlreadyIdentical), |field| {
2136        Err(ChannelRegistryError::AlreadyRegistered {
2137            name: spec.name.clone(),
2138            field,
2139        })
2140    })
2141}
2142
2143/// Recovers the resolved schema of an already-built entry, so the write-lock
2144/// re-check compares the SAME three fields against the same values the fast path
2145/// would have.
2146fn schema_of(configured: &ConfiguredChannel) -> ChannelSchema {
2147    ChannelSchema {
2148        document: configured.handle.config().schema.definition().clone(),
2149        protocol_id: configured.protocol_schema,
2150    }
2151}
2152
2153/// Builds one roster entry: the channel's validation engine, its library handle
2154/// (durable or ephemeral), and the protocol schema id advertised at subscribe
2155/// time.
2156///
2157/// This is the SOLE place a [`ConfiguredChannel`] is constructed. Keeping
2158/// construction in one function is what makes "every channel on this server is
2159/// the same kind of object" structural rather than a promise: there is no second
2160/// body a channel could be built by, so no channel can drift into a different
2161/// shape. The boot loop in [`LiminalConnectionServices::from_config_with_store_via`]
2162/// is its caller.
2163///
2164/// `resolved` is consumed because its JSON Schema document is moved into the
2165/// channel's [`Schema`]; the protocol id is carried onto the entry unchanged.
2166/// `origin` is stamped here and never written again.
2167///
2168/// The failure is typed ([`ChannelBuildError`]) rather than a [`ServerError`] so
2169/// BOTH callers can render it without inspecting a message: the boot loop maps
2170/// it back to the exact `ConfigValidation` strings it has always produced, and
2171/// `register_channel` maps it to the registry's own typed variants.
2172///
2173/// # Errors
2174/// Returns [`ChannelBuildError`] when the JSON Schema document does not compile
2175/// or durable initialization over the shared store fails.
2176fn build_configured_channel(
2177    name: &str,
2178    resolved: ChannelSchema,
2179    durable: bool,
2180    origin: ChannelOrigin,
2181    durable_store: &Arc<dyn DurableStore>,
2182    supervisor: &ChannelSupervisor,
2183) -> Result<ConfiguredChannel, ChannelBuildError> {
2184    let schema =
2185        Schema::new(resolved.document).map_err(|error| ChannelBuildError::SchemaRejected {
2186            message: error.to_string(),
2187        })?;
2188    let channel_config = if durable {
2189        ChannelConfig::new(name.to_owned(), schema, ChannelMode::Durable)
2190    } else {
2191        ChannelConfig::new(name.to_owned(), schema, ChannelMode::Ephemeral)
2192    };
2193    let handle = if durable {
2194        ChannelHandle::new_durable_with_supervisor(
2195            channel_config,
2196            Arc::clone(durable_store),
2197            supervisor.clone(),
2198        )
2199        .map_err(|error| ChannelBuildError::DurableInitFailed {
2200            message: error.to_string(),
2201        })?
2202    } else {
2203        ChannelHandle::with_supervisor(channel_config, supervisor.clone())
2204    };
2205    Ok(ConfiguredChannel {
2206        handle,
2207        protocol_schema: resolved.protocol_id,
2208        durable,
2209        origin,
2210        state: AtomicU8::new(STATE_ACTIVE),
2211        quiesce_reason: OnceLock::new(),
2212    })
2213}
2214
2215/// Returns the current epoch-millis timestamp used as the dedup entry anchor.
2216///
2217/// A clock error before the Unix epoch yields `0`: the timestamp is only a TTL
2218/// anchor for the in-memory cache and a zero anchor never breaks the at-most-once
2219/// claim semantics, so this avoids surfacing a clock fault on the publish path.
2220fn dedup_timestamp_millis() -> u64 {
2221    use std::time::{SystemTime, UNIX_EPOCH};
2222    SystemTime::now()
2223        .duration_since(UNIX_EPOCH)
2224        .ok()
2225        .and_then(|duration| u64::try_from(duration.as_millis()).ok())
2226        .unwrap_or(0)
2227}
2228
2229/// Default shard count for an on-disk durable store.
2230///
2231/// Haematite routes keys across this many single-threaded shard actors; a small
2232/// power of two gives parallelism across cursors/streams without spawning an
2233/// actor per core. The value is fixed (haematite has no silent default) and not
2234/// yet surfaced in server config.
2235const DEFAULT_SHARD_COUNT: usize = 8;
2236
2237/// Shard count of the throwaway store an incarnation's ephemeral dedup
2238/// namespace lives in: it holds one namespace, and dies with the process.
2239const THROWAWAY_SHARD_COUNT: usize = 1;
2240
2241/// Namespace prefix for the dedup-on-delivery cache streams. Keeps delivery dedup
2242/// keys from colliding with any other haematite streams in the shared store.
2243///
2244/// This is the **durable** namespace (A1 §5): stable across restarts, because a
2245/// durable channel's message outlives a restart too.
2246const DELIVERY_DEDUP_NAMESPACE: &str = "liminal:delivery-dedup";
2247
2248/// The **ephemeral** dedup namespace for this server incarnation (A1 §5,
2249/// graft §0.2).
2250///
2251/// Scoped to the process incarnation so an ephemeral channel's receipt shares a
2252/// lifetime with the in-memory message it vouches for. Composed from the
2253/// process id and the boot instant, which is enough to make two incarnations
2254/// disagree; nothing reads the components back, so no meaning is attached to
2255/// them beyond distinctness.
2256///
2257/// The namespace lives in [`ephemeral_dedup_store`], never in a store that
2258/// outlives the process, so no incarnation's records are left behind for a
2259/// later one to read past.
2260fn ephemeral_dedup_namespace() -> String {
2261    use std::time::{SystemTime, UNIX_EPOCH};
2262    let boot_nanos = SystemTime::now()
2263        .duration_since(UNIX_EPOCH)
2264        .map_or(0, |since| since.as_nanos());
2265    format!(
2266        "{DELIVERY_DEDUP_NAMESPACE}:incarnation:{}-{boot_nanos}",
2267        std::process::id()
2268    )
2269}
2270
2271/// One reader over the counters of `durable_store` and, when it is a separate
2272/// store, `ephemeral_store`; a store that is the same `Arc` is read once.
2273fn combined_store_counts(
2274    durable_store: &Arc<dyn DurableStore>,
2275    ephemeral_store: &Arc<dyn DurableStore>,
2276) -> StoreCountsHandle {
2277    let durable = durable_store.store_counts().unwrap_or_default();
2278    if Arc::ptr_eq(durable_store, ephemeral_store) {
2279        return durable;
2280    }
2281    durable.with(&ephemeral_store.store_counts().unwrap_or_default())
2282}
2283
2284/// The store an incarnation's ephemeral dedup namespace is written to: the
2285/// services' own store when it was created empty for this process and dies
2286/// with it, otherwise a store of its own that does, opened by the namespace's
2287/// first write rather than at start.
2288///
2289/// Written into a persistent store, every incarnation's namespace would stay
2290/// there for good, under the durable namespace's prefix, with nothing that
2291/// ever reads it again to prune it.
2292fn ephemeral_dedup_store(
2293    durable_store: &Arc<dyn DurableStore>,
2294    history: StoreHistory,
2295    subsystems: &dyn SubsystemFactory,
2296) -> Arc<dyn DurableStore> {
2297    match (history, durable_store) {
2298        #[cfg(any(test, feature = "test-support"))]
2299        (StoreHistory::Fresh, store) => Arc::clone(store),
2300        (StoreHistory::Created | StoreHistory::Recorded, _) => Arc::new(CountingStore::new(
2301            DeferredStore::new(subsystems.throwaway_store()),
2302        )),
2303    }
2304}
2305
2306/// Beamr-scheduler-owning subsystems the full-service construction path builds
2307/// beyond the connection supervisor's own scheduler. The §9 D2 seam census counts
2308/// these: the worker-front-door profile must construct NONE of them. Test-gated:
2309/// this is the recording vocabulary of the gate's instrument, not production state.
2310#[cfg(test)]
2311#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
2312pub(super) enum SchedulerSubsystem {
2313    /// The shared channel supervisor (its own beamr scheduler).
2314    ChannelSupervisor,
2315    /// The conversation supervisor (its own beamr scheduler).
2316    ConversationSupervisor,
2317    /// The haematite store's database (its shard-actor scheduler).
2318    HaematiteStore,
2319}
2320
2321/// Constructor seam for every scheduler-owning subsystem (`SchedulerSubsystem`)
2322/// the profile-aware construction path can create.
2323///
2324/// These methods are the ONLY route through which [`build_connection_services`]
2325/// and the [`LiminalConnectionServices`] config constructors reach
2326/// `build_channel_cluster`, `ConversationSupervisor::new`, and the durable-store
2327/// constructors — no direct constructor call exists in those bodies. The §9 D2
2328/// gate therefore injects a factory that records as a side effect of
2329/// constructing (the D3 store-seam ownership move applied to schedulers): a
2330/// recording cannot be omitted without also failing to construct the subsystem,
2331/// which closes the "hand-placed census call beside the constructor" gap where a
2332/// future subsystem could be built without its courtesy call.
2333pub(super) trait SubsystemFactory {
2334    /// Constructs the shared channel supervisor + cluster resolver (a beamr
2335    /// scheduler).
2336    ///
2337    /// # Errors
2338    /// Returns [`ServerError`] when the channel supervisor scheduler cannot start.
2339    fn channel_cluster(
2340        &self,
2341        cluster_config: Option<&ClusterConfig>,
2342    ) -> Result<ChannelCluster, ServerError>;
2343
2344    /// Constructs the conversation supervisor (a beamr scheduler).
2345    ///
2346    /// # Errors
2347    /// Returns [`ServerError`] when the conversation supervisor scheduler cannot
2348    /// start.
2349    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError>;
2350
2351    /// Constructs the persistent durable store under the explicit directory.
2352    ///
2353    /// # Errors
2354    /// Returns [`ServerError`] when the store cannot be opened.
2355    fn durable_store(&self, persistence_path: &Path) -> Result<Arc<dyn DurableStore>, ServerError>;
2356
2357    /// The opener of the throwaway store an incarnation's ephemeral dedup
2358    /// namespace lives in when the durable store outlives the process. The
2359    /// store is opened by the namespace's first write, never by this call.
2360    fn throwaway_store(&self) -> ThrowawayStoreOpener;
2361
2362    fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
2363        None
2364    }
2365}
2366
2367/// Test construction factory for callers that own their fixture directories.
2368#[cfg(any(test, feature = "test-support"))]
2369pub(super) struct ProductionSubsystems;
2370
2371#[cfg(any(test, feature = "test-support"))]
2372impl SubsystemFactory for ProductionSubsystems {
2373    fn channel_cluster(
2374        &self,
2375        cluster_config: Option<&ClusterConfig>,
2376    ) -> Result<ChannelCluster, ServerError> {
2377        build_channel_cluster(cluster_config)
2378    }
2379
2380    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2381        Ok(Arc::new(ConversationSupervisor::new().map_err(
2382            |error| ServerError::ConfigValidation {
2383                message: format!("failed to start conversation supervisor: {error}"),
2384            },
2385        )?))
2386    }
2387
2388    fn durable_store(&self, persistence_path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2389        build_durable_store(persistence_path)
2390    }
2391
2392    fn throwaway_store(&self) -> ThrowawayStoreOpener {
2393        Box::new(|| open_ephemeral(THROWAWAY_SHARD_COUNT))
2394    }
2395}
2396
2397struct RootedProductionSubsystems {
2398    root: Arc<EphemeralRoot>,
2399}
2400
2401impl RootedProductionSubsystems {
2402    fn new(path: &Path) -> Result<Self, ServerError> {
2403        let root = EphemeralRoot::open(path).map_err(|error| ServerError::ConfigValidation {
2404            message: error.to_string(),
2405        })?;
2406        Ok(Self { root })
2407    }
2408}
2409
2410impl SubsystemFactory for RootedProductionSubsystems {
2411    fn channel_cluster(
2412        &self,
2413        config: Option<&ClusterConfig>,
2414    ) -> Result<ChannelCluster, ServerError> {
2415        build_channel_cluster(config)
2416    }
2417
2418    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2419        Ok(Arc::new(ConversationSupervisor::new().map_err(
2420            |error| ServerError::ConfigValidation {
2421                message: format!("failed to start conversation supervisor: {error}"),
2422            },
2423        )?))
2424    }
2425
2426    fn durable_store(&self, path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2427        build_durable_store(path)
2428    }
2429
2430    fn throwaway_store(&self) -> ThrowawayStoreOpener {
2431        let root = Arc::clone(&self.root);
2432        Box::new(move || root.open_store(THROWAWAY_SHARD_COUNT))
2433    }
2434
2435    fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
2436        Some(self.root.sweep_counts())
2437    }
2438}
2439
2440/// Test-only recording implementation of [`SubsystemFactory`] for the §9 D2
2441/// construction gate. Lives here (not in a test module's private scope) so the
2442/// supervisor-level gate test reuses the same instrument.
2443#[cfg(test)]
2444pub(super) mod subsystem_census {
2445    use std::path::{Path, PathBuf};
2446    use std::sync::Arc;
2447    use std::sync::atomic::{AtomicUsize, Ordering};
2448
2449    use liminal::conversation::ConversationSupervisor;
2450    use liminal::durability::{DurableStore, open_ephemeral_rooted};
2451
2452    use super::{
2453        ChannelCluster, ProductionSubsystems, SchedulerSubsystem, SubsystemFactory,
2454        THROWAWAY_SHARD_COUNT, ThrowawayStoreOpener, build_durable_store,
2455    };
2456    use crate::ServerError;
2457    use crate::config::types::ClusterConfig;
2458
2459    /// Records each [`SchedulerSubsystem`] AS A SIDE EFFECT of constructing it,
2460    /// then hands back the production-constructed subsystem (with ephemeral
2461    /// stores rooted in an isolated directory, so the fs half of the gate is a
2462    /// real negative assertion). Because [`SubsystemFactory`] is the only route
2463    /// the profile-aware construction path has to these constructors, a
2464    /// recording cannot be omitted without also failing to construct — the
2465    /// record-by-construction guarantee.
2466    pub struct RecordingSubsystems {
2467        /// How many of each subsystem were constructed, in the enum's order.
2468        census: [AtomicUsize; 3],
2469        ephemeral_root: PathBuf,
2470    }
2471
2472    impl RecordingSubsystems {
2473        /// A recording factory whose ephemeral stores live under `ephemeral_root`.
2474        pub fn rooted(ephemeral_root: &Path) -> Self {
2475            Self {
2476                census: [
2477                    AtomicUsize::new(0),
2478                    AtomicUsize::new(0),
2479                    AtomicUsize::new(0),
2480                ],
2481                ephemeral_root: ephemeral_root.to_path_buf(),
2482            }
2483        }
2484
2485        /// The recorded construction census, sorted for order-independent
2486        /// comparison.
2487        pub fn recorded(&self) -> Vec<SchedulerSubsystem> {
2488            KINDS
2489                .iter()
2490                .zip(&self.census)
2491                .flat_map(|(kind, count)| std::iter::repeat_n(*kind, count.load(Ordering::SeqCst)))
2492                .collect()
2493        }
2494
2495        fn record(&self, subsystem: SchedulerSubsystem) {
2496            if let Some(count) = KINDS
2497                .iter()
2498                .position(|kind| *kind == subsystem)
2499                .and_then(|index| self.census.get(index))
2500            {
2501                count.fetch_add(1, Ordering::SeqCst);
2502            }
2503        }
2504    }
2505
2506    /// Every [`SchedulerSubsystem`], in its sort order, indexing `census`.
2507    const KINDS: [SchedulerSubsystem; 3] = [
2508        SchedulerSubsystem::ChannelSupervisor,
2509        SchedulerSubsystem::ConversationSupervisor,
2510        SchedulerSubsystem::HaematiteStore,
2511    ];
2512
2513    impl SubsystemFactory for RecordingSubsystems {
2514        fn channel_cluster(
2515            &self,
2516            cluster_config: Option<&ClusterConfig>,
2517        ) -> Result<ChannelCluster, ServerError> {
2518            let cluster = ProductionSubsystems.channel_cluster(cluster_config)?;
2519            self.record(SchedulerSubsystem::ChannelSupervisor);
2520            Ok(cluster)
2521        }
2522
2523        fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2524            let supervisor = ProductionSubsystems.conversation_supervisor()?;
2525            self.record(SchedulerSubsystem::ConversationSupervisor);
2526            Ok(supervisor)
2527        }
2528
2529        fn durable_store(
2530            &self,
2531            persistence_path: &Path,
2532        ) -> Result<Arc<dyn DurableStore>, ServerError> {
2533            let store = build_durable_store(persistence_path)?;
2534            self.record(SchedulerSubsystem::HaematiteStore);
2535            Ok(store)
2536        }
2537
2538        fn throwaway_store(&self) -> ThrowawayStoreOpener {
2539            let root = self.ephemeral_root.clone();
2540            Box::new(move || open_ephemeral_rooted(&root, THROWAWAY_SHARD_COUNT))
2541        }
2542    }
2543}
2544
2545/// Full-only constructor guard: rejects a config whose profile is not `Full`.
2546///
2547/// [`LiminalConnectionServices`]' config-based constructors call this at entry so
2548/// the full service stack can never be built for a worker-front-door config —
2549/// profile enforcement holds on every public construction path, not only the
2550/// file-loading pipeline.
2551fn require_full_profile(config: &ServerConfig) -> Result<(), ServerError> {
2552    match config.services.profile()? {
2553        ServiceProfile::Full => Ok(()),
2554        ServiceProfile::WorkerFrontDoor => Err(ServerError::ConfigValidation {
2555            message: format!(
2556                "services.profile: \"{}\" cannot construct the full LiminalConnectionServices; \
2557                 build profile-selected services via build_connection_services",
2558                ServiceProfile::WORKER_FRONT_DOOR
2559            ),
2560        }),
2561    }
2562}
2563
2564/// Builds the connection-services adapter selected by `config`'s service profile.
2565///
2566/// `Full` builds [`LiminalConnectionServices`] (channels, conversations, durable
2567/// store, dedup cache) exactly as today. `WorkerFrontDoor` builds
2568/// [`WorkerFrontDoorServices`], which constructs none of that machinery. This is the
2569/// single profile-dispatch authority: [`super::supervisor::ConnectionSupervisor`]'s
2570/// config constructor and the standalone runtime's worker arm both route through it
2571/// (the runtime's full arm stays on the explicit
2572/// [`LiminalConnectionServices::from_config`] path because it also needs the shared
2573/// channel cluster, which the trait object does not expose — that path is guarded
2574/// full-only at entry).
2575///
2576/// # Errors
2577/// Returns [`ServerError`] when the selected adapter cannot be constructed, the
2578/// configured profile value is not recognised, or the worker-front-door profile is
2579/// combined with full-only config fields.
2580pub fn build_connection_services(
2581    config: &ServerConfig,
2582) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2583    match config.services.profile()? {
2584        ServiceProfile::Full => {
2585            let subsystems = RootedProductionSubsystems::new(required_persistence_path(config)?)?;
2586            build_connection_services_via(config, &subsystems)
2587        }
2588        ServiceProfile::WorkerFrontDoor => build_worker_front_door(config),
2589    }
2590}
2591
2592/// [`build_connection_services`] with the subsystem factory injected — the §9 D2
2593/// gate seam, both halves at once.
2594///
2595/// Every scheduler-owning subsystem the `Full` branch creates is constructed
2596/// through `subsystems` and nowhere else, so a recording factory observes exactly
2597/// what was built (thread half, record-by-construction); the gate's factory also
2598/// roots its ephemeral stores in an isolated directory, so "the root stays empty
2599/// on the worker branch" is a real negative assertion (fs half, the D3 pattern).
2600/// The `WorkerFrontDoor` branch never touches the factory — a regression that gave
2601/// the front door any subsystem would both record in the census and land a store
2602/// directory in the injected root.
2603///
2604/// The worker branch re-runs the cross-field checks here, not only in file-loading
2605/// validation, so a directly-constructed config cannot smuggle full-only machinery
2606/// past the profile.
2607pub(super) fn build_connection_services_via(
2608    config: &ServerConfig,
2609    subsystems: &dyn SubsystemFactory,
2610) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2611    match config.services.profile()? {
2612        ServiceProfile::Full => {
2613            let path = required_persistence_path(config)?;
2614            let history = StoreHistory::of(path);
2615            let store = subsystems.durable_store(path)?;
2616            Ok(Arc::new(
2617                LiminalConnectionServices::from_config_with_store_via(
2618                    config, store, history, subsystems,
2619                )?,
2620            ))
2621        }
2622        ServiceProfile::WorkerFrontDoor => build_worker_front_door(config),
2623    }
2624}
2625
2626fn build_worker_front_door(
2627    config: &ServerConfig,
2628) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2629    let errors = crate::config::validation::worker_front_door_field_errors(config);
2630    if !errors.is_empty() {
2631        return Err(ServerError::ConfigValidation {
2632            message: errors.join("; "),
2633        });
2634    }
2635    Ok(Arc::new(WorkerFrontDoorServices::new()))
2636}
2637
2638/// Opens the persistent durable store under the configured directory.
2639pub(crate) fn open_persistent_store(
2640    persistence_path: &Path,
2641) -> Result<Arc<dyn DurableStore>, ServerError> {
2642    build_durable_store(persistence_path)
2643}
2644
2645/// Every append is permanent through its WAL frames and one device flush
2646/// before it returns under either commit policy; what the policy decides is
2647/// when the shard folds its staged appends into a new tree version, the fold
2648/// that writes the leaf and every internal node above it as fresh files. Per
2649/// append, one keyed publish left 8 new node files (82,137 bytes) on the J3
2650/// store (NODE-FILES-PER-POST, check 15 on Dean at 68a08b35, 10 October
2651/// 2026), and the live spine reached 2,856,796 files that way. Buffered at
2652/// 1 MiB, Waffles' number (10 October 2026 11:19): the threshold haematite's
2653/// own `LeafLog` runs on, and the most an unclean restart replays from staged
2654/// frames per shard. The store's own reads merge the buffer, so a publish
2655/// reads back at once; a clean stop commits what is staged; no policy carries
2656/// a timer.
2657const SPINE_COMMIT_THRESHOLD_BYTES: NonZeroU64 = match NonZeroU64::new(1 << 20) {
2658    Some(threshold) => threshold,
2659    None => unreachable!(),
2660};
2661
2662/// The commit policy the spine's event store opens under.
2663const SPINE_COMMIT_POLICY: CommitPolicy = CommitPolicy::buffered(SPINE_COMMIT_THRESHOLD_BYTES);
2664
2665fn build_durable_store(path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2666    let data_dir = durable_data_dir(path);
2667    let database = open_or_create_database(&data_dir)?;
2668    Ok(crate::server::boot_steps::timed("event_store_wrap", || {
2669        let event_store = EventStore::with_commit_policy(database, SPINE_COMMIT_POLICY);
2670        let store: Arc<dyn DurableStore> = Arc::new(CountingStore::new(HaematiteStore::new(
2671            Arc::new(event_store),
2672        )));
2673        store
2674    }))
2675}
2676
2677/// The directory the persistent database under `persistence_path` lives in.
2678pub(super) fn durable_data_dir(persistence_path: &Path) -> PathBuf {
2679    persistence_path.join("durability")
2680}
2681
2682/// Whether a haematite database already exists at `data_dir`: its
2683/// `config.json` is written when it is created.
2684pub(super) fn database_exists(data_dir: &Path) -> bool {
2685    database_config_file(data_dir).exists()
2686}
2687
2688/// What creating a haematite database leaves in its directory. A shard's
2689/// directory appears beside them the first time anything opens the shard, so
2690/// a database holding nothing else has never been read or written.
2691const UNTOUCHED_DATABASE_ENTRIES: [&str; 2] = ["config.json", "writer.lock"];
2692
2693/// Whether the database at `data_dir` holds only what its creation wrote.
2694/// Anything this cannot read, or does not recognise, answers `false`: the
2695/// database is then taken to hold what an earlier server wrote.
2696pub(super) fn database_untouched(data_dir: &Path) -> bool {
2697    std::fs::read_dir(data_dir).is_ok_and(|entries| {
2698        entries.into_iter().all(|entry| {
2699            entry.is_ok_and(|entry| {
2700                entry
2701                    .file_name()
2702                    .to_str()
2703                    .is_some_and(|name| UNTOUCHED_DATABASE_ENTRIES.contains(&name))
2704            })
2705        })
2706    })
2707}
2708
2709fn database_config_file(data_dir: &Path) -> PathBuf {
2710    data_dir.join("config.json")
2711}
2712
2713/// Every commit writes new copies of the nodes it changes; the durable store
2714/// deletes the copies its committed root no longer reaches while it runs, so it
2715/// holds its live records and not every version ever written. A store created
2716/// under `KeepAll` is moved onto this retention by its next open.
2717pub(super) const DURABLE_RETENTION: Retention = Retention::reclaim(0);
2718
2719/// Opens an existing haematite database at `data_dir`, or creates one, timed
2720/// as the boot step named by the arm that runs (SRV-045 R1).
2721fn open_or_create_database(data_dir: &Path) -> Result<Database, ServerError> {
2722    let config_file = database_config_file(data_dir);
2723    let exists = config_file.exists();
2724    let step = if exists {
2725        "database_open"
2726    } else {
2727        "database_create"
2728    };
2729    crate::server::boot_steps::timed(step, || open_or_create_arm(data_dir, &config_file, exists))
2730}
2731
2732/// [`open_or_create_database`]'s work: the open arm when `exists`, else the
2733/// create arm, then the store's open is counted and its reclaim watch armed.
2734fn open_or_create_arm(
2735    data_dir: &Path,
2736    config_file: &Path,
2737    exists: bool,
2738) -> Result<Database, ServerError> {
2739    let result = if exists {
2740        adopt_missing_config_keys(data_dir, config_file)?;
2741        Database::open_with_retention(
2742            data_dir,
2743            haematite::RuntimeOptions {
2744                long_wait_interval: std::time::Duration::from_secs(5),
2745                // DUR-021 R1: no pass starts and no step runs until the
2746                // listeners are bound and runtime.rs releases it.
2747                reclaim_start: haematite::ReclaimStart::Held,
2748            },
2749            DURABLE_RETENTION,
2750        )
2751    } else {
2752        Database::create(
2753            DatabaseConfig {
2754                data_dir: data_dir.to_path_buf(),
2755                shard_count: DEFAULT_SHARD_COUNT,
2756                distributed: None,
2757                executor_threads: None,
2758                // haematite 0.8.3 requires this field and refuses `None` at
2759                // validation; `Unlimited` is the crate's explicit spelling of the
2760                // pre-budget (0.8.1) behaviour — the behaviour-preserving choice
2761                // for a dependency hop. A real byte ceiling is a deployment
2762                // decision for its own measured lane. A store whose `config.json`
2763                // predates the field is migrated by the open arm above
2764                // (`adopt_missing_config_keys`) — same value, written once, on
2765                // upgrade.
2766                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
2767                retention: DURABLE_RETENTION,
2768            },
2769            haematite::RuntimeOptions {
2770                long_wait_interval: std::time::Duration::from_secs(5),
2771                // DUR-021 R1: no pass starts and no step runs until the
2772                // listeners are bound and runtime.rs releases it.
2773                reclaim_start: haematite::ReclaimStart::Held,
2774            },
2775        )
2776    };
2777    let database = result.map_err(|error| ServerError::ConfigValidation {
2778        message: format!(
2779            "failed to open durable store at {}: {error}",
2780            data_dir.display()
2781        ),
2782    })?;
2783    liminal::durability::record_store_opened();
2784    super::reclaim_placement::arm_watch(&database);
2785    Ok(database)
2786}
2787
2788/// The stored keys this crate's create arm always writes, with the value it
2789/// writes, for a `config.json` written before haematite had the key.
2790///
2791/// - `node_cache_budget`: haematite 0.8.3 refuses a store without it
2792///   (`MissingNodeCacheBudget`); `"unlimited"` is the pre-budget behaviour.
2793/// - `retention`: `"keep_all"` is the 0.14.1 behaviour, written in so the
2794///   store's stored retention is explicit before the open. The open itself is
2795///   [`Database::open_with_retention`] under [`DURABLE_RETENTION`], which then
2796///   records reclaim in place of it, so an adopted store reclaims from that
2797///   same open onward (spine reclaim, Hermes's ruling of 6 October 2026:
2798///   `past_roots` 0).
2799const ADOPTED_CONFIG_KEYS: [(&str, &str); 2] = [
2800    ("node_cache_budget", "unlimited"),
2801    ("retention", "keep_all"),
2802];
2803
2804/// Writes the values of [`ADOPTED_CONFIG_KEYS`] into a `config.json` that
2805/// lacks any of them, so a store created before haematite had the key opens on
2806/// upgrade with no operator hand. The open that follows then moves its
2807/// retention onto [`DURABLE_RETENTION`].
2808///
2809/// This crate IS the caller that owns these decisions, and it already makes
2810/// them on every create, so answering them here, once, on upgrade, changes
2811/// nothing about how the store runs and takes the file off the operator's
2812/// desk (the store migrates itself or it doesn't ship).
2813///
2814/// Touches the file in exactly one case: a readable JSON object missing one of
2815/// those keys. Anything else — unreadable bytes, a non-object, an explicit
2816/// value however wrong — is left byte-identical so [`Database::open`] surfaces
2817/// haematite's own sharpest refusal instead of this function guessing over
2818/// it. The write is atomic and durable (tmp file beside the target, fsync,
2819/// rename, directory fsync): a crash mid-migration leaves the original bytes,
2820/// and the next boot migrates again.
2821fn adopt_missing_config_keys(data_dir: &Path, config_file: &Path) -> Result<(), ServerError> {
2822    let Ok(bytes) = std::fs::read(config_file) else {
2823        return Ok(());
2824    };
2825    let Ok(mut config) = serde_json::from_slice::<serde_json::Value>(&bytes) else {
2826        return Ok(());
2827    };
2828    let Some(object) = config.as_object_mut() else {
2829        return Ok(());
2830    };
2831    let mut adopted = Vec::new();
2832    for (key, value) in ADOPTED_CONFIG_KEYS {
2833        if !object.contains_key(key) {
2834            object.insert(key.to_owned(), serde_json::Value::String(value.to_owned()));
2835            adopted.push(key);
2836        }
2837    }
2838    if adopted.is_empty() {
2839        return Ok(());
2840    }
2841    let keys = adopted.join(", ");
2842    let migrated =
2843        serde_json::to_vec_pretty(&config).map_err(|error| ServerError::ConfigValidation {
2844            message: format!(
2845                "durable store config at {} predates `{keys}` and its migration could not be \
2846                 serialized: {error}",
2847                config_file.display()
2848            ),
2849        })?;
2850    let write_error = |step: &str, error: std::io::Error| ServerError::ConfigValidation {
2851        message: format!(
2852            "durable store config at {} predates `{keys}` and migrating it failed at {step}: \
2853             {error}",
2854            config_file.display()
2855        ),
2856    };
2857    let tmp_file = data_dir.join("config.json.migrate-tmp");
2858    let mut tmp = std::fs::File::create(&tmp_file)
2859        .map_err(|error| write_error("creating the temporary file", error))?;
2860    std::io::Write::write_all(&mut tmp, &migrated)
2861        .map_err(|error| write_error("writing the temporary file", error))?;
2862    tmp.sync_all()
2863        .map_err(|error| write_error("syncing the temporary file", error))?;
2864    drop(tmp);
2865    std::fs::rename(&tmp_file, config_file)
2866        .map_err(|error| write_error("renaming over config.json", error))?;
2867    std::fs::File::open(data_dir)
2868        .and_then(|dir| dir.sync_all())
2869        .map_err(|error| write_error("syncing the data directory", error))?;
2870    tracing::warn!(
2871        config = %config_file.display(),
2872        keys = %keys,
2873        "durable store config predates these keys -- wrote the create-arm values (the \
2874         behaviour-preserving ones) so the store opens with no operator action"
2875    );
2876    Ok(())
2877}
2878
2879pub(super) fn reserved_namespace_refusal(channel: &str) -> Option<ServerError> {
2880    if channel.starts_with("liminal.") {
2881        crate::metrics::self_stream_reserved_publication();
2882        Some(ServerError::ReservedNamespace {
2883            channel: channel.to_owned(),
2884        })
2885    } else {
2886        None
2887    }
2888}
2889
2890impl LiminalConnectionServices {
2891    pub(in crate::server) const fn signals(&self) -> &ServerSignals {
2892        &self.signals
2893    }
2894
2895    pub(in crate::server) fn self_stream_head(&self, name: &str) -> Result<u64, ServerError> {
2896        let channels = self.read_channels()?;
2897        let channel = channels
2898            .get(name)
2899            .ok_or_else(|| ServerError::ListenerAccept {
2900                message: format!("self stream '{name}' is absent after append"),
2901            })?;
2902        let heads =
2903            channel
2904                .handle
2905                .durable_positions()
2906                .map_err(|error| ServerError::ListenerAccept {
2907                    message: format!("self stream '{name}' head is unavailable: {error}"),
2908                })?;
2909        drop(channels);
2910        match heads.as_slice() {
2911            [head] => Ok(*head),
2912            _ => Err(ServerError::ListenerAccept {
2913                message: format!("self stream '{name}' must have exactly one durable partition"),
2914            }),
2915        }
2916    }
2917
2918    pub(in crate::server) fn publish_self_stream_batch(
2919        &self,
2920        payloads: &[Vec<u8>],
2921        publisher: &str,
2922    ) -> Result<u64, ServerError> {
2923        if publisher.trim().is_empty() || payloads.is_empty() {
2924            return Err(ServerError::ConfigValidation {
2925                message: "self-stream batch requires a publisher and rows".to_owned(),
2926            });
2927        }
2928        let configured = self
2929            .admit_channel("liminal.events")
2930            .map_err(|error| access_to_server_error(&error))?;
2931        let deliveries = configured
2932            .handle
2933            .publish_batch_with_delivery(
2934                payloads,
2935                &liminal::envelope::PublisherId::new(publisher),
2936                None,
2937            )
2938            .map_err(|error| match error {
2939                liminal::error::LiminalError::PublishUnresolved { append } => {
2940                    ServerError::PublishUnresolved {
2941                        channel: "liminal.events".to_owned(),
2942                        idempotency_key: None,
2943                        sequence: Some(append.sequence()),
2944                        message: append.to_string(),
2945                    }
2946                }
2947                error => ServerError::ListenerAccept {
2948                    message: format!("self-stream receipt batch failed: {error}"),
2949                },
2950            })?;
2951        if deliveries.len() != payloads.len()
2952            || deliveries
2953                .iter()
2954                .any(|delivery| delivery.durable_position().is_none())
2955        {
2956            return Err(ServerError::ListenerAccept {
2957                message: "self-stream receipt batch was refused before append".to_owned(),
2958            });
2959        }
2960        for pair in deliveries.windows(2) {
2961            if pair[0]
2962                .durable_position()
2963                .and_then(|position| position.checked_add(1))
2964                != pair[1].durable_position()
2965            {
2966                return Err(ServerError::ListenerAccept {
2967                    message: "self-stream receipt batch positions are not contiguous".to_owned(),
2968                });
2969            }
2970        }
2971        for delivery in &deliveries {
2972            Self::record_delivery_counts(delivery);
2973        }
2974        deliveries
2975            .last()
2976            .and_then(ChannelDelivery::durable_position)
2977            .and_then(|position| position.checked_add(1))
2978            .ok_or_else(|| ServerError::ListenerAccept {
2979                message: "self-stream receipt batch head is unavailable".to_owned(),
2980            })
2981    }
2982
2983    pub(in crate::server) fn checkpoint_self_stream_heads(
2984        &self,
2985        heads: [u64; 2],
2986    ) -> Result<(), ServerError> {
2987        for (name, head) in ["liminal.events:0", "liminal.cost:0"]
2988            .into_iter()
2989            .zip(heads)
2990        {
2991            block_on(liminal::durability::head_checkpoint::advance_head(
2992                self.durable_store.as_ref(),
2993                name,
2994                head,
2995            ))
2996            .map_err(|error| ServerError::ShutdownFlush {
2997                message: format!("self-stream head checkpoint bridge for '{name}' failed: {error}"),
2998            })?
2999            .map_err(|error| ServerError::ShutdownFlush {
3000                message: format!("self-stream head checkpoint for '{name}' failed: {error}"),
3001            })?;
3002        }
3003        Ok(())
3004    }
3005
3006    pub(in crate::server) fn install_self_stream_channels(&self) -> Result<(), ServerError> {
3007        let names = ["liminal.cost", "liminal.events"];
3008        {
3009            let channels = self.read_channels()?;
3010            for name in names {
3011                if channels.contains_key(name) {
3012                    return Err(ServerError::ConfigValidation {
3013                        message: format!("self-stream channel '{name}' is already configured"),
3014                    });
3015                }
3016            }
3017        }
3018        let mut built = Vec::with_capacity(names.len());
3019        for name in names {
3020            let schema =
3021                resolve_schema_bytes(None).map_err(|error| ServerError::ConfigValidation {
3022                    message: format!("self-stream channel '{name}' schema failed: {error}"),
3023                })?;
3024            let channel = build_configured_channel(
3025                name,
3026                schema,
3027                true,
3028                ChannelOrigin::BootConfigured,
3029                &self.durable_store,
3030                self.cluster.supervisor(),
3031            )
3032            .map_err(|error| ServerError::ConfigValidation {
3033                message: error.boot_message(name),
3034            })?;
3035            channel
3036                .handle
3037                .install_publication_identity(self.publication_identity.clone())
3038                .map_err(|error| ServerError::ConfigValidation {
3039                    message: error.to_string(),
3040                })?;
3041            built.push((name.to_owned(), Arc::new(channel)));
3042        }
3043        let mut channels =
3044            self.channels
3045                .write()
3046                .map_err(|error| ServerError::ConfigValidation {
3047                    message: format!("self-stream channel roster lock poisoned: {error}"),
3048                })?;
3049        for (name, _) in &built {
3050            if channels.contains_key(name) {
3051                return Err(ServerError::ConfigValidation {
3052                    message: format!("self-stream channel '{name}' was concurrently configured"),
3053                });
3054            }
3055        }
3056        channels.extend(built);
3057        drop(channels);
3058        // SRV-012 R7: every roster channel but `console.*` is a line, and
3059        // these join after the hub's model was built from the roster.
3060        for name in names {
3061            self.console_changed(crate::console::TopologyChange::Channel {
3062                name: name.to_owned(),
3063                life: line_life(true),
3064                state: crate::console::records::LineState::Live,
3065            });
3066        }
3067        Ok(())
3068    }
3069
3070    /// SRV-012 R4: registers `console.topology` as an ephemeral,
3071    /// server-owned channel (never a store write). A channel of that name
3072    /// already configured (ephemeral, since config refuses a durable one) is
3073    /// kept as it is.
3074    pub(in crate::server) fn install_console_channels(&self) -> Result<(), ServerError> {
3075        self.register_console_channel(crate::console::TOPOLOGY_SUBJECT)
3076            .map_err(|error| ServerError::ConfigValidation {
3077                message: format!("console channel registration failed: {error}"),
3078            })
3079    }
3080
3081    /// Puts the ephemeral console channel `name` on the roster unless it is
3082    /// there.
3083    fn register_console_channel(&self, name: &str) -> Result<(), ChannelRegistryError> {
3084        if self.read_roster()?.contains_key(name) {
3085            return Ok(());
3086        }
3087        let schema = resolve_schema_bytes(None).map_err(|error| {
3088            ChannelRegistryError::RosterUnavailable {
3089                message: format!("console channel '{name}' schema failed: {error}"),
3090            }
3091        })?;
3092        let channel = build_configured_channel(
3093            name,
3094            schema,
3095            false,
3096            ChannelOrigin::BootConfigured,
3097            &self.durable_store,
3098            self.cluster.supervisor(),
3099        )
3100        .map_err(|error| error.into_registry_error(name))?;
3101        channel
3102            .handle
3103            .install_publication_identity(self.publication_identity.clone())
3104            .map_err(|error| ChannelRegistryError::RosterUnavailable {
3105                message: error.to_string(),
3106            })?;
3107        self.write_roster()?
3108            .entry(name.to_owned())
3109            .or_insert_with(|| Arc::new(channel));
3110        Ok(())
3111    }
3112
3113    /// Publishes on an ordinary channel for an application embedding the server.
3114    ///
3115    /// The live and durable publisher identity is liminal.<host>. Reserved self
3116    /// streams and console channels retain the same refusal as connection publishes.
3117    /// An idempotency key has the same delivery semantics as a connection publish.
3118    ///
3119    /// # Errors
3120    ///
3121    /// Returns a configuration error for an empty or whitespace-only host name,
3122    /// a namespace refusal for a reserved channel, or the existing admission,
3123    /// durability, and publication error.
3124    pub fn publish_embedded(
3125        &self,
3126        host: &str,
3127        channel: &str,
3128        envelope: &MessageEnvelope,
3129        idempotency_key: Option<&str>,
3130    ) -> Result<PublishOutcome, ServerError> {
3131        self.publish_with_origin(
3132            channel,
3133            envelope,
3134            idempotency_key,
3135            PublishOrigin::Host { name: host },
3136        )
3137        .map(|published| published.outcome)
3138    }
3139
3140    pub(in crate::server) fn publish_with_origin(
3141        &self,
3142        channel: &str,
3143        envelope: &MessageEnvelope,
3144        idempotency_key: Option<&str>,
3145        origin: PublishOrigin<'_>,
3146    ) -> Result<OriginPublication, ServerError> {
3147        if !origin.is_server() {
3148            if plan_subject(channel).is_some() {
3149                return Err(site_services::site_error(
3150                    crate::services::lifecycle::site::SiteError::Invalid(
3151                        "plan subject is server-owned",
3152                    ),
3153                ));
3154            }
3155            crate::agents::binding::refuse_server_publish(channel)?;
3156        }
3157        validate_publication_origin(channel, idempotency_key, origin)?;
3158        // SRV-012 to_stations: only while a console tap is live does the
3159        // fan-out record whom the message reached, in a buffer the tap lends
3160        // and takes back; otherwise the publish is the plain one.
3161        let mut recipients = self.signals.lend_recipients();
3162        // Lane brief ruling 6: a tapped message is first seen as the server
3163        // receives it, before admission; the clock is read only while a tap
3164        // lent a record.
3165        let received_at = recipients.is_some().then(Instant::now);
3166        let result = (|| {
3167            // The roster read holds its guard only long enough to clone the entry's
3168            // `Arc` out; everything below — the dedup bridge and the publish into the
3169            // channel actor — runs with no roster lock held.
3170            //
3171            // This inner admission STAYS even once the connection process consults
3172            // the roster ahead of delegating: this method is public and callable
3173            // without going through a frame at all, and a guard that only exists in
3174            // the caller is not a guard.
3175            let configured = self
3176                .admit_channel(channel)
3177                .map_err(|error| access_to_server_error(&error))?;
3178
3179            // Dedup-on-delivery: a publish carrying an idempotency key is delivered to
3180            // subscribers AT MOST ONCE across re-publishes of the same key. Only a
3181            // fresh `Claimed` decision proceeds to fan-out; a `Completed`/`InFlight`
3182            // decision is a duplicate and is suppressed (no second delivery), which is
3183            // the at-most-once guarantee the aion outbox relies on.
3184            // A1 §5: the namespace is chosen by the channel's DURABILITY, so an
3185            // ephemeral receipt cannot outlive the message it vouches for.
3186            // A keyed publish to a durable channel claims its key WITH the channel
3187            // as evidence and holds the claim across an unresolved append
3188            // (`unresolved.rs`); the ephemeral namespace keeps the claim below.
3189            if let (Some(key), ChannelMode::Durable) =
3190                (idempotency_key, configured.handle.config().mode)
3191            {
3192                return self
3193                    .publish_durable_keyed(
3194                        &configured,
3195                        channel,
3196                        (envelope, origin),
3197                        key,
3198                        recipients.as_mut(),
3199                    )
3200                    .map(OriginPublication::connection);
3201            }
3202            let dedup = self.dedup_for(&configured);
3203            if let Some(key) = idempotency_key {
3204                if !Self::claim_delivery(dedup, key)? {
3205                    // A dedup-suppressed re-publish is still an accepted publish (it is
3206                    // assigned a message id), but it reaches no subscriber, so it counts
3207                    // toward publishes and not deliveries.
3208                    crate::metrics::publish_accepted();
3209                    return Ok(OriginPublication::connection(PublishOutcome {
3210                        message_id: self.next_message_id()?,
3211                        delivered: false,
3212                        durable_position: None,
3213                        reach: Reach::Suppressed,
3214                    }));
3215                }
3216            }
3217
3218            let delivery = fan_out(&configured, envelope, origin, recipients.as_mut());
3219            let delivery = match delivery {
3220                Ok(delivery) => delivery,
3221                // An unkeyed durable publish whose append outcome is not known is
3222                // answered as unresolved, never as a definite failure a producer
3223                // would retry into a second row.
3224                Err(liminal::error::LiminalError::PublishUnresolved { append }) => {
3225                    return Err(ServerError::PublishUnresolved {
3226                        channel: channel.to_owned(),
3227                        idempotency_key: None,
3228                        sequence: Some(append.sequence()),
3229                        message: append.to_string(),
3230                    });
3231                }
3232                Err(error) => {
3233                    // The claim above appended an `InFlight` entry but the delivery
3234                    // failed before `complete_receipt` could run. Release the claim so
3235                    // the key is re-claimable; otherwise every re-publish would see
3236                    // `InFlight` and be suppressed forever. Best-effort: surface the
3237                    // ORIGINAL publish error regardless, but never swallow a release
3238                    // failure silently (it leaves the leak intact).
3239                    if let Some(key) = idempotency_key {
3240                        Self::release_claim(dedup, key);
3241                    }
3242                    return Err(ServerError::ListenerAccept {
3243                        message: format!("liminal publish failed for channel '{channel}': {error}"),
3244                    });
3245                }
3246            };
3247
3248            // A1 §5, the two-way dedup mapping. Every publish outcome maps to
3249            // EXACTLY ONE of complete/release — the same shape as the C1 contract:
3250            //
3251            // * **Defer (or Accept) => `complete_receipt`.** The message is
3252            //   admitted and bus-held, so the key is done: a producer retrying it
3253            //   hits `Completed` and there is no duplicate delivery and no InFlight
3254            //   leak. This is the path that makes "re-publishing with the same key
3255            //   is safe and is suppressed while the bus owns the message" true.
3256            // * **Reject => `release_claim`.** Aggregation makes a producer-visible
3257            //   Reject mean delivered-to-zero, so the correct dedup state is "as if
3258            //   never published": tombstone it so a deliberate retry can re-claim.
3259            //   `release_claim` never clobbers a receipt, so the at-most-once guard
3260            //   survives a concurrent completion racing ahead.
3261            //
3262            // Mixed accept/reject reports Defer and so completes — consistent with
3263            // "at least one delivery happened".
3264            if let Some(key) = idempotency_key {
3265                if delivery.is_admitted() {
3266                    block_on(dedup.complete_receipt(key, ProcessingReceipt::new(Vec::new())))
3267                        .map_err(|error| ServerError::ListenerAccept {
3268                            message: format!(
3269                                "dedup receipt bridge failed for key '{key}': {error}"
3270                            ),
3271                        })?
3272                        .map_err(|error| ServerError::ListenerAccept {
3273                            message: format!("dedup receipt write failed for key '{key}': {error}"),
3274                        })?;
3275                } else {
3276                    Self::release_claim(dedup, key);
3277                }
3278            }
3279
3280            let self_row_at = delivery.durable_position().filter(|_| origin.is_server());
3281            Ok(OriginPublication {
3282                outcome: Self::delivered(&delivery)?,
3283                self_row_appended: self_row_at.is_some(),
3284                self_row_at,
3285            })
3286        })();
3287        self.signals.published(
3288            channel,
3289            origin,
3290            envelope,
3291            &result,
3292            recipients.as_ref().zip(received_at),
3293        );
3294        if let Some(buffer) = recipients {
3295            self.signals.give_back(buffer);
3296        }
3297        result
3298    }
3299}
3300
3301/// The library publish of an unkeyed message, recording what it saw in
3302/// `recipients` when the tap lent a record.
3303fn fan_out(
3304    configured: &ConfiguredChannel,
3305    envelope: &MessageEnvelope,
3306    origin: PublishOrigin<'_>,
3307    recipients: Option<&mut liminal::channel::PublishRecord>,
3308) -> Result<liminal::channel::ChannelDelivery, liminal::error::LiminalError> {
3309    let publisher = || publisher_for_origin(origin);
3310    recipients.map_or_else(
3311        || {
3312            configured
3313                .handle
3314                .publish_with_delivery(&envelope.payload, publisher(), None)
3315        },
3316        |recipients| {
3317            configured.handle.publish_recording(
3318                &envelope.payload,
3319                publisher(),
3320                None,
3321                None,
3322                recipients,
3323            )
3324        },
3325    )
3326}
3327
3328fn validate_publication_origin(
3329    channel: &str,
3330    idempotency_key: Option<&str>,
3331    origin: PublishOrigin<'_>,
3332) -> Result<(), ServerError> {
3333    if !origin.is_server() {
3334        refuse_console_namespace(channel).map_err(|error| access_to_server_error(&error))?;
3335        if let Some(error) = reserved_namespace_refusal(channel) {
3336            return Err(error);
3337        }
3338    }
3339    if let PublishOrigin::Host { name } = origin
3340        && name.trim().is_empty()
3341    {
3342        return Err(ServerError::ConfigValidation {
3343            message: "embedded publication requires a nonempty host name".to_owned(),
3344        });
3345    }
3346    if let PublishOrigin::Server { device } = origin {
3347        if device.trim().is_empty() || idempotency_key.is_some() {
3348            return Err(ServerError::ConfigValidation {
3349                message: "server-origin publication requires a device and no idempotency key"
3350                    .to_owned(),
3351            });
3352        }
3353    }
3354    Ok(())
3355}
3356
3357fn publisher_for_origin(origin: PublishOrigin<'_>) -> liminal::envelope::PublisherId {
3358    match origin {
3359        PublishOrigin::Connection { id } => liminal::envelope::PublisherId::new(id.to_string()),
3360        PublishOrigin::Host { name } => {
3361            liminal::envelope::PublisherId::new(format!("liminal.{name}"))
3362        }
3363        PublishOrigin::Server { device } => {
3364            liminal::envelope::PublisherId::new(format!("liminal.{device}"))
3365        }
3366    }
3367}
3368
3369impl ConnectionServices for LiminalConnectionServices {
3370    fn construction_request(
3371        &self,
3372        session: Option<&dyn crate::auth_lys::session::LysSession>,
3373        subject: &str,
3374        bytes: &[u8],
3375        at_ms: u64,
3376    ) -> Option<Result<serde_json::Value, ServerError>> {
3377        if let Some(answer) = self.agent_host_request(session, subject, bytes, at_ms) {
3378            return Some(answer);
3379        }
3380        is_site_route(subject).then(|| {
3381            self.serve_site_tool(session, subject, bytes, at_ms)
3382                .map_err(site_services::site_error)
3383        })
3384    }
3385    fn prepare_site_trial(
3386        &self,
3387        connection: u64,
3388        session: Option<&dyn crate::auth_lys::session::LysSession>,
3389        registration: &liminal::protocol::WorkerRegistration,
3390        process: Option<&crate::services::lifecycle::ProcessGrant>,
3391        at_ms: u64,
3392    ) -> Result<Option<crate::services::lifecycle::trial::TrialPermit>, ServerError> {
3393        self.site_lifecycle.get().map_or(Ok(None), |lifecycle| {
3394            lifecycle
3395                .prepare_trial(connection, session, registration, process, at_ms)
3396                .map_err(site_services::site_error)
3397        })
3398    }
3399    fn admit_site_call(
3400        &self,
3401        session: Option<&dyn crate::auth_lys::session::LysSession>,
3402        subject: &str,
3403        at_ms: u64,
3404    ) -> Result<bool, ServerError> {
3405        self.site_lifecycle.get().map_or(Ok(false), |lifecycle| {
3406            lifecycle
3407                .admit_trial_call(session, subject, at_ms)
3408                .map_err(site_services::site_error)
3409        })
3410    }
3411    fn site_registration_closed(&self, connection: u64) -> Result<(), ServerError> {
3412        if let Some(lifecycle) = self.site_lifecycle.get() {
3413            lifecycle
3414                .trial_closed(connection)
3415                .map_err(site_services::site_error)?;
3416            lifecycle
3417                .refresh_sites()
3418                .map_err(site_services::site_error)?;
3419        }
3420        Ok(())
3421    }
3422    fn publish_identified(
3423        &self,
3424        connection: u64,
3425        session: Option<&dyn crate::auth_lys::session::LysSession>,
3426        channel: &str,
3427        envelope: &MessageEnvelope,
3428        key: Option<&str>,
3429    ) -> Result<PublishOutcome, ServerError> {
3430        if plan_subject(channel).is_some() {
3431            return Err(site_services::site_error(
3432                crate::services::lifecycle::site::SiteError::Invalid(
3433                    "plan subject is server-owned",
3434                ),
3435            ));
3436        }
3437        self.publish_agent(connection, session, channel, envelope, key)
3438    }
3439    fn agents(&self) -> Option<&Arc<crate::agents::runtime::Agents>> {
3440        self.agents.as_ref()
3441    }
3442
3443    fn subscribe_identified(
3444        &self,
3445        connection: u64,
3446        session: Option<&dyn crate::auth_lys::session::LysSession>,
3447        channel: &str,
3448        schemas: &[ProtocolSchemaId],
3449        install: Option<liminal::channel::InboxInstall>,
3450    ) -> Result<ConnectionSubscription, ServerError> {
3451        if plan_subject(channel).is_some() {
3452            return self.subscribe_site_plan(session, channel, schemas, install);
3453        }
3454        self.subscribe_agent(connection, session, channel, schemas, install)
3455    }
3456    fn participant_service(&self) -> Option<InstalledParticipantService> {
3457        self.participant_service.clone()
3458    }
3459
3460    /// The roster's admission decision, exposed across the trait boundary as the
3461    /// permission alone.
3462    ///
3463    /// Two methods share the name `admit_channel` on this type: this trait
3464    /// method, and the private inherent funnel it delegates to. They are the
3465    /// same decision at two different boundaries — the funnel returns the ENTRY
3466    /// the decision was made on, because its in-crate callers go on to publish
3467    /// or subscribe through it, and this one returns `()`, because a caller
3468    /// outside the crate is asking whether it MAY, not for the thing itself.
3469    /// Handing the entry out here would make [`ConfiguredChannel`] public
3470    /// surface and hand a frame-level caller a channel handle it has no business
3471    /// holding. Rust resolves an inherent method ahead of a trait method of the
3472    /// same name, so the funnel's existing callers — and the `Self::` call below
3473    /// — reach the funnel, not this method; the trait form is reached only
3474    /// through a `dyn ConnectionServices`, which is exactly the caller it exists
3475    /// for. `an_admitted_operation_the_service_then_refuses_stays_undifferentiated`
3476    /// is the instrument that would catch this delegation turning into a
3477    /// recursion.
3478    ///
3479    /// `operation` is where the answers part: absence and quiesce refuse
3480    /// publishes and subscribes alike, but the console namespace refuses only
3481    /// a connection's publish (a console reads `console.*` as an ordinary
3482    /// subscriber, and only the server may write there), and a tap target
3483    /// refuses only a subscribe. Both are checked before the roster, so a
3484    /// client is told the same thing whether or not the subject is registered
3485    /// yet.
3486    fn admit_channel(
3487        &self,
3488        operation: ChannelOperation,
3489        channel: &str,
3490    ) -> Result<(), ChannelAccessError> {
3491        if let Some(name) = plan_subject(channel) {
3492            let refused = |message| ChannelAccessError::RosterUnavailable { message };
3493            if operation == ChannelOperation::Publish {
3494                return Err(refused("plan subject is server-owned".into()));
3495            }
3496            let lifecycle = self
3497                .site_lifecycle
3498                .get()
3499                .ok_or_else(|| refused("construction not installed".into()))?;
3500            lifecycle
3501                .record(name)
3502                .map_err(|error| refused(error.to_string()))?;
3503            return Ok(());
3504        }
3505        if let Some(admitted) = self.agent_channel_admission(channel) {
3506            return admitted;
3507        }
3508        match operation {
3509            ChannelOperation::Publish => refuse_console_namespace(channel)?,
3510            ChannelOperation::Subscribe => {
3511                refuse_console_tap_target(channel)?;
3512                self.refuse_console_tap_limit(channel)?;
3513                // SRV-012 R4/R6: a tap channel exists while it has a
3514                // subscriber, and only while the console streams run.
3515                let admitted = if self.signals.console_enabled()
3516                    && super::channel_registry::console_tap_target(channel).is_some()
3517                {
3518                    self.register_console_channel(channel).map_err(|error| {
3519                        ChannelAccessError::RosterUnavailable {
3520                            message: error.to_string(),
3521                        }
3522                    })
3523                } else {
3524                    Ok(())
3525                }
3526                .and_then(|()| Self::admit_channel(self, channel).map(|_| ()));
3527                if admitted.is_err() {
3528                    self.give_back_tap(channel);
3529                }
3530                return admitted;
3531            }
3532        }
3533        Self::admit_channel(self, channel)?;
3534        Ok(())
3535    }
3536
3537    fn publish(
3538        &self,
3539        connection: u64,
3540        channel: &str,
3541        envelope: &MessageEnvelope,
3542        idempotency_key: Option<&str>,
3543    ) -> Result<PublishOutcome, ServerError> {
3544        // The frame path refused this already at admission. This guard is for
3545        // a caller holding the trait directly: a guard that only exists in the
3546        // caller is not a guard.
3547        refuse_console_namespace(channel).map_err(|error| access_to_server_error(&error))?;
3548        if plan_subject(channel).is_some() {
3549            return Err(site_services::site_error(
3550                crate::services::lifecycle::site::SiteError::Invalid(
3551                    "plan subject is server-owned",
3552                ),
3553            ));
3554        }
3555        crate::agents::binding::refuse_server_publish(channel)?;
3556        if crate::agents::binding::subject(channel).is_some() {
3557            return Err(crate::agents::error::AgentError::Invalid(
3558                "agent publish requires the verified identity route",
3559            )
3560            .into());
3561        }
3562        self.publish_with_origin(
3563            channel,
3564            envelope,
3565            idempotency_key,
3566            PublishOrigin::Connection { id: connection },
3567        )
3568        .map(|published| published.outcome)
3569    }
3570
3571    fn subscribe(
3572        &self,
3573        channel: &str,
3574        accepted_schemas: &[ProtocolSchemaId],
3575        install: Option<liminal::channel::InboxInstall>,
3576    ) -> Result<ConnectionSubscription, ServerError> {
3577        if plan_subject(channel).is_some() {
3578            return Err(site_services::site_error(
3579                crate::services::lifecycle::site::SiteError::Invalid(
3580                    "plan subscription needs an admitted pass",
3581                ),
3582            ));
3583        }
3584        if crate::agents::binding::subject(channel).is_some() {
3585            return Err(crate::agents::error::AgentError::Invalid(
3586                "agent subscribe requires the verified identity route",
3587            )
3588            .into());
3589        }
3590        let subscribed = self.open_subscription(channel, accepted_schemas, install, None);
3591        if subscribed.is_err() {
3592            // SRV-012 R6: a tap slot reserved at admission for a subscribe
3593            // that did not happen is given back, and a tap channel it
3594            // registered with no tap behind it leaves the roster.
3595            self.give_back_tap(channel);
3596        }
3597        subscribed
3598    }
3599
3600    fn unsubscribe(&self, subscription: ConnectionSubscription) -> Result<(), ServerError> {
3601        subscription.unsubscribe()
3602    }
3603
3604    fn open_conversation(
3605        &self,
3606        conversation_id: u64,
3607        subject: &str,
3608    ) -> Result<ConnectionConversation, ServerError> {
3609        self.open_routed_conversation(conversation_id, subject)
3610    }
3611
3612    fn conversation_message(
3613        &self,
3614        conversation: &ConnectionConversation,
3615        envelope: &MessageEnvelope,
3616        op_id: Option<u64>,
3617    ) -> Result<(), ServerError> {
3618        self.routed_conversation_message(conversation, envelope, op_id)
3619    }
3620
3621    fn close_conversation(&self, conversation: ConnectionConversation) -> Result<(), ServerError> {
3622        conversation.close()
3623    }
3624
3625    fn listening(self: Arc<Self>) -> Option<std::thread::JoinHandle<()>> {
3626        self.resolve_claims_behind_listener()
3627    }
3628
3629    fn stop_participant_warm(&self) {
3630        // DUR-020 R5: the warm stops at its next conversation boundary and
3631        // the stop does not wait for it; whatever it had not restored
3632        // restores on the next start. Then the clean stop records the
3633        // capacity snapshot (R3), and the final flush makes it durable.
3634        if let Some(warm) = &self.participant_warm {
3635            warm.stop();
3636            warm.record_clean_stop();
3637        }
3638    }
3639
3640    fn finish_background_store_work(&self) -> Result<(), ServerError> {
3641        self.finish_background_work()
3642    }
3643
3644    fn release_store_holders(&self) -> Result<(), ServerError> {
3645        self.release_holders()
3646    }
3647
3648    fn flush_durable_state(&self) -> Result<(), ServerError> {
3649        // A flush observer publishes to a durable channel, so no channel append lock can be held.
3650        // Separate ephemeral dedup state dies with this incarnation; shared dedup state is covered here.
3651        self.signals.with_flush_barrier(|| {
3652            block_on(self.head_checkpoint_stop.checkpoint_heads())
3653                .map_err(|error| ServerError::ShutdownFlush {
3654                    message: format!("clean-stop head checkpoint bridge failed: {error}"),
3655                })?
3656                .map_err(|error| ServerError::ShutdownFlush {
3657                    message: format!("clean-stop head checkpoint failed: {error}"),
3658                })?;
3659            block_on(self.durable_store.flush())
3660                .map_err(|error| ServerError::ShutdownFlush {
3661                    message: format!("shared durable store flush bridge failed: {error}"),
3662                })?
3663                .map_err(|error| ServerError::ShutdownFlush {
3664                    message: format!("shared durable store flush failed: {error}"),
3665                })?;
3666            Ok(())
3667        })?;
3668        Ok(())
3669    }
3670}
3671
3672/// One roster entry: a channel's library handle, the protocol schema id
3673/// advertised for it, where it came from, and its admission state.
3674#[derive(Debug)]
3675pub(super) struct ConfiguredChannel {
3676    handle: ChannelHandle,
3677    protocol_schema: ProtocolSchemaId,
3678    /// Durable (stored) or ephemeral, as built; the console's line life.
3679    durable: bool,
3680    /// Boot-configured or runtime-registered. Written once at construction and
3681    /// NEVER flipped: it is the only field that predicts what a restart does to
3682    /// this entry, and it defines the population the registration cap counts.
3683    origin: ChannelOrigin,
3684    /// [`STATE_ACTIVE`] or [`STATE_QUIESCED`]. Written at most once, by one
3685    /// `compare_exchange`, so the transition is one-way and exactly one caller
3686    /// can perform it.
3687    state: AtomicU8,
3688    /// The quiesce cause, set STRICTLY BEFORE `state` flips so any reader that
3689    /// observes [`STATE_QUIESCED`] can read it. Written once (the `OnceLock`
3690    /// enforces that structurally), which is why a re-quiesce under a different
3691    /// reason must refuse rather than silently keep or replace one of them.
3692    quiesce_reason: OnceLock<String>,
3693}
3694
3695impl ConfiguredChannel {
3696    /// The entry's admission state as a value.
3697    ///
3698    /// One `Acquire` load and, when quiesced, one read of the already-written
3699    /// reason. Touches no actor: the whole point of recording state on the entry
3700    /// is that observing it cannot spawn the thing being observed.
3701    fn state(&self) -> ChannelState {
3702        if self.state.load(Ordering::Acquire) == STATE_QUIESCED {
3703            ChannelState::Quiesced {
3704                reason: self.recorded_quiesce_reason(),
3705            }
3706        } else {
3707            ChannelState::Active
3708        }
3709    }
3710
3711    /// The recorded quiesce reason, for a caller that has already observed
3712    /// [`STATE_QUIESCED`] with an `Acquire` load.
3713    fn recorded_quiesce_reason(&self) -> String {
3714        self.quiesce_reason
3715            .get()
3716            .map_or_else(|| UNRECORDED_QUIESCE_REASON.to_owned(), Clone::clone)
3717    }
3718
3719    /// Moves this entry from active to quiesced, recording `reason` first.
3720    ///
3721    /// The reason is written to the `OnceLock` BEFORE the `Release`
3722    /// `compare_exchange` that flips the state, so no reader can observe
3723    /// `QUIESCED` without being able to read why. Re-quiescing under the
3724    /// IDENTICAL reason is `Ok(())` — the caller's intent already holds — and a
3725    /// DIFFERENT reason refuses, because the recorded one cannot be replaced
3726    /// without losing a cause and cannot be kept without lying to the caller.
3727    fn quiesce(&self, name: &str, reason: &str) -> Result<(), ChannelRegistryError> {
3728        // Whoever wins the `OnceLock` writes the cause on record; every other
3729        // caller — a re-quiesce or a concurrent racer — is judged against THAT
3730        // reason, never against whether it happened to perform the flip itself.
3731        // Reporting success for having won the CAS would tell a racer whose
3732        // reason lost that its reason took effect.
3733        let recorded = match self.quiesce_reason.set(reason.to_owned()) {
3734            Ok(()) => reason.to_owned(),
3735            Err(_rejected) => self.recorded_quiesce_reason(),
3736        };
3737        // The reason is now readable, so the flip may become visible. The CAS
3738        // result is deliberately unused: the state is `QUIESCED` afterwards
3739        // whether this call or a racer performed the transition, and the answer
3740        // to the caller is governed by the recorded reason above.
3741        let _flipped_here = self.state.compare_exchange(
3742            STATE_ACTIVE,
3743            STATE_QUIESCED,
3744            Ordering::Release,
3745            Ordering::Acquire,
3746        );
3747        if recorded == reason {
3748            return Ok(());
3749        }
3750        Err(ChannelRegistryError::AlreadyQuiesced {
3751            name: name.to_owned(),
3752            reason: recorded,
3753        })
3754    }
3755}
3756
3757#[derive(Debug)]
3758struct LiminalSubscriptionResource {
3759    subscription: liminal::channel::SubscriptionHandle,
3760}
3761
3762impl SubscriptionResource for LiminalSubscriptionResource {
3763    fn try_next_positioned(
3764        &mut self,
3765    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
3766        self.subscription
3767            .try_next_positioned()
3768            .map_err(|error| subscription_failure(&error))
3769    }
3770
3771    fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
3772        self.subscription
3773            .acknowledge_position(position)
3774            .map_err(|error| subscription_failure(&error))
3775    }
3776
3777    fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
3778        self.subscription
3779            .consumer_position()
3780            .map_err(|error| subscription_failure(&error))
3781    }
3782    fn unsubscribe(self: Box<Self>) -> Result<(), ServerError> {
3783        drop(self.subscription);
3784        Ok(())
3785    }
3786
3787    fn is_overflowed(&self) -> bool {
3788        self.subscription.is_overflowed()
3789    }
3790
3791    fn subscriber_pid(&self) -> Option<u64> {
3792        Some(self.subscription.subscriber_pid())
3793    }
3794
3795    fn has_pending(&self) -> bool {
3796        self.subscription.has_pending()
3797    }
3798
3799    fn try_next(&mut self) -> Option<liminal::envelope::Envelope> {
3800        match self.subscription.try_next() {
3801            Ok(envelope) => envelope,
3802            Err(error) => {
3803                tracing::error!(
3804                    %error,
3805                    "subscription could not receive its next envelope"
3806                );
3807                None
3808            }
3809        }
3810    }
3811}
3812
3813/// Preserves the library's subscription failure at the connection boundary.
3814fn subscription_failure(error: &liminal::error::LiminalError) -> ServerError {
3815    ServerError::ListenerAccept {
3816        message: format!("liminal subscription: {error}"),
3817    }
3818}
3819
3820/// Renders an admission refusal as the service error the trait's callers expect.
3821///
3822/// The two refusals that already existed keep their EXACT bytes: an absent
3823/// channel is still `channel '<name>' is not configured` and a poisoned roster
3824/// is still `channel roster lock poisoned`, so nothing that reads these messages
3825/// today sees a change. Discrimination lives in the reason code
3826/// ([`ChannelAccessError::reason_code`]), which the connection process carries to
3827/// the wire — this rendering is the degraded path for a caller that reaches the
3828/// service directly and has no code to carry.
3829///
3830/// `Quiesced` has no predecessor string to preserve; it carries the refusal's own
3831/// message, reason included.
3832///
3833/// The connection process renders its admission refusals through this SAME
3834/// function rather than through [`ChannelAccessError`]'s own `Display`. Two
3835/// consequences, both wanted. The bytes on the wire for an absent channel stay
3836/// exactly what they have always been, which is the whole of the semver promise
3837/// this lane makes. And an operation refused AT admission and one refused after
3838/// admission by the service produce the identical message, differing only in the
3839/// reason code — so the code really is the only discriminator, instead of the
3840/// message quietly becoming a second one.
3841pub(super) fn access_to_server_error(error: &ChannelAccessError) -> ServerError {
3842    let message = match error {
3843        ChannelAccessError::NotRegistered { name } => {
3844            format!("channel '{name}' is not configured")
3845        }
3846        ChannelAccessError::RosterUnavailable { message } => message.clone(),
3847        ChannelAccessError::Quiesced { .. }
3848        | ChannelAccessError::ConsoleNamespace { .. }
3849        | ChannelAccessError::ConsoleTapTarget { .. }
3850        | ChannelAccessError::ConsoleTapLimit { .. } => error.to_string(),
3851    };
3852    ServerError::ListenerAccept { message }
3853}
3854
3855/// Refuses a subscribe to `console.tap.<target>` whose target is empty or
3856/// under `console.`: the tap never taps the console's own records.
3857///
3858/// # Errors
3859/// Returns [`ChannelAccessError::ConsoleTapTarget`] naming the channel.
3860fn refuse_console_tap_target(channel: &str) -> Result<(), ChannelAccessError> {
3861    use super::channel_registry::{console_tap_target, is_console_subject};
3862    match console_tap_target(channel) {
3863        Some(target) if target.is_empty() || is_console_subject(target) => {
3864            Err(ChannelAccessError::ConsoleTapTarget {
3865                name: channel.to_owned(),
3866            })
3867        }
3868        _ => Ok(()),
3869    }
3870}
3871
3872/// Refuses a connection's publish under `console.`, the server's own namespace.
3873///
3874/// # Errors
3875/// Returns [`ChannelAccessError::ConsoleNamespace`] naming the channel.
3876fn refuse_console_namespace(channel: &str) -> Result<(), ChannelAccessError> {
3877    if super::channel_registry::is_console_subject(channel) {
3878        return Err(ChannelAccessError::ConsoleNamespace {
3879            name: channel.to_owned(),
3880        });
3881    }
3882    Ok(())
3883}
3884
3885pub(super) fn server_error_from_protocol(error: &ProtocolError) -> ServerError {
3886    ServerError::ListenerAccept {
3887        message: format!("protocol operation failed: {error}"),
3888    }
3889}
3890
3891/// The three pinned registration tests. A CHILD module of `services` because
3892/// test 2 must hold the exact entry the admission funnel handed out.
3893#[cfg(test)]
3894#[path = "services_registry_tests.rs"]
3895mod services_registry_tests;
3896
3897#[cfg(test)]
3898mod durable_store_tests {
3899    /// A test's answer: a failure is an error naming what failed, never a panic.
3900    type TestResult = Result<(), Box<dyn std::error::Error>>;
3901
3902    /// Turns a missing value or an error into a named failure for `?`.
3903    trait Must<T> {
3904        fn must(self, what: &str) -> Result<T, String>;
3905    }
3906
3907    impl<T, E: std::fmt::Debug> Must<T> for Result<T, E> {
3908        fn must(self, what: &str) -> Result<T, String> {
3909            self.map_err(|error| format!("{what}: {error:?}"))
3910        }
3911    }
3912
3913    impl<T> Must<T> for Option<T> {
3914        fn must(self, what: &str) -> Result<T, String> {
3915            self.ok_or_else(|| what.to_owned())
3916        }
3917    }
3918
3919    use liminal::durability::open_ephemeral_rooted;
3920
3921    use super::subsystem_census::RecordingSubsystems;
3922    use super::{
3923        ConnectionServices, DEFAULT_SHARD_COUNT, LiminalConnectionServices, SchedulerSubsystem,
3924        build_connection_services, build_connection_services_via, build_durable_store,
3925    };
3926    use crate::ServerError;
3927    use crate::config::types::{LimitsConfig, ServerConfig, ServicesConfig};
3928
3929    /// Counts directory entries under `root`, for the empty/one-dir assertions
3930    /// on an injected ephemeral root.
3931    fn entry_count(root: &std::path::Path) -> Result<usize, String> {
3932        Ok(std::fs::read_dir(root)
3933            .must("ephemeral root is readable")?
3934            .count())
3935    }
3936
3937    /// A minimal channel-free config with the given service `profile`. No channels,
3938    /// routing, persistence, or cluster — the shape both profiles accept (the full
3939    /// profile simply builds an empty channel set; the worker-front-door profile
3940    /// requires exactly this shape).
3941    fn config_with_profile(profile: &str) -> Result<ServerConfig, String> {
3942        Ok(ServerConfig {
3943            listen_address: "127.0.0.1:0".parse().must("valid socket addr")?,
3944            extra_listen_addresses: Vec::new(),
3945            health_listen_address: "127.0.0.1:1".parse().must("valid socket addr")?,
3946            drain_timeout_ms: 30_000,
3947            channels: Vec::new(),
3948            routing_rules: Vec::new(),
3949            persistence_path: None,
3950            cluster: None,
3951            auth: None,
3952            services: ServicesConfig {
3953                profile: profile.to_owned(),
3954                ..ServicesConfig::default()
3955            },
3956            limits: LimitsConfig::default(),
3957            participant: None,
3958            websocket: None,
3959        })
3960    }
3961
3962    /// §9 D2 front-door construction gate (fs half): building the worker-front-door
3963    /// services creates NO haematite store and NO temp dir, while the full profile
3964    /// over an equally-rooted factory DOES create exactly one store directory.
3965    ///
3966    /// The injected root is the only place an ephemeral store directory can appear
3967    /// (the recording factory roots its stores there), so "the root stays empty on
3968    /// the front-door branch" is a real negative assertion: a regression that gave
3969    /// the front door a store would land its directory here and fail this test. The
3970    /// full-profile arm is the positive control proving the seam genuinely
3971    /// constructs a store when the profile asks for one.
3972    #[test]
3973    fn worker_front_door_builds_no_store_and_no_temp_dir() -> TestResult {
3974        let front_door_root = tempfile::tempdir().must("test can create an ephemeral root")?;
3975        let full_root = tempfile::tempdir().must("test can create an ephemeral root")?;
3976
3977        let front_door_subsystems = RecordingSubsystems::rooted(front_door_root.path());
3978        let front_door: std::sync::Arc<dyn ConnectionServices> = build_connection_services_via(
3979            &config_with_profile("worker-front-door")?,
3980            &front_door_subsystems,
3981        )
3982        .must("worker-front-door services build")?;
3983        assert!(
3984            !front_door.supports_channel_operations(),
3985            "the worker front door serves no channel operations"
3986        );
3987        assert_eq!(
3988            entry_count(front_door_root.path())?,
3989            0,
3990            "the worker front door creates no ephemeral store directory (no haematite, no temp dir)"
3991        );
3992
3993        let full_subsystems = RecordingSubsystems::rooted(full_root.path());
3994        let mut config = config_with_profile("full")?;
3995        config.persistence_path = Some(full_root.path().to_path_buf());
3996        let full =
3997            build_connection_services_via(&config, &full_subsystems).must("full services build")?;
3998        assert!(
3999            full.supports_channel_operations(),
4000            "full mode serves channel operations"
4001        );
4002        assert_eq!(
4003            entry_count(full_root.path())?,
4004            1,
4005            "full mode builds its persistent database in the explicit directory"
4006        );
4007
4008        drop(front_door);
4009        drop(full);
4010        Ok(())
4011    }
4012
4013    /// §9 D2 front-door construction gate (thread half — record-by-construction
4014    /// census): the worker profile constructs NO channel-supervisor,
4015    /// conversation-supervisor, or haematite scheduler, while the SAME instrument
4016    /// over the full profile records all three — the positive control proving the
4017    /// census detects the extra schedulers, so an empty census on the worker branch
4018    /// is a real observation, not a decoration.
4019    ///
4020    /// The instrument's boundary: recording happens INSIDE the [`SubsystemFactory`]
4021    /// methods that are the profile-aware path's only route to these constructors,
4022    /// so a recording cannot be silently omitted — a future subsystem added to this
4023    /// path either goes through the factory (and is recorded by construction) or
4024    /// bypasses it, which is a code-review-visible structural violation of the
4025    /// factory seam, not a silently-missing side call. The connection supervisor's
4026    /// own scheduler is the shared baseline of both profiles and is asserted at the
4027    /// supervisor level (`supervisor::tests`); an OS-level thread census upgrades
4028    /// this when the beamr composition lane's scheduler-inventory API (currently on
4029    /// their branch, not yet consumable from liminal) lands.
4030    #[test]
4031    fn worker_profile_census_is_empty_and_full_profile_records_all_schedulers() -> TestResult {
4032        let worker_root = tempfile::tempdir().must("test can create an ephemeral root")?;
4033        let full_root = tempfile::tempdir().must("test can create an ephemeral root")?;
4034
4035        let worker_subsystems = RecordingSubsystems::rooted(worker_root.path());
4036        let front_door = build_connection_services_via(
4037            &config_with_profile("worker-front-door")?,
4038            &worker_subsystems,
4039        )
4040        .must("worker-front-door services build")?;
4041        assert_eq!(
4042            worker_subsystems.recorded(),
4043            Vec::<SchedulerSubsystem>::new(),
4044            "the worker front door constructs no scheduler-owning subsystem"
4045        );
4046
4047        let full_subsystems = RecordingSubsystems::rooted(full_root.path());
4048        let mut config = config_with_profile("full")?;
4049        config.persistence_path = Some(full_root.path().to_path_buf());
4050        let full =
4051            build_connection_services_via(&config, &full_subsystems).must("full services build")?;
4052        assert_eq!(
4053            full_subsystems.recorded(),
4054            vec![
4055                SchedulerSubsystem::ChannelSupervisor,
4056                SchedulerSubsystem::ConversationSupervisor,
4057                SchedulerSubsystem::HaematiteStore,
4058            ],
4059            "the full profile constructs every scheduler-owning subsystem, once each — \
4060             the positive control proving the census instrument detects them"
4061        );
4062
4063        drop(front_door);
4064        drop(full);
4065        Ok(())
4066    }
4067
4068    /// MAJOR-1 regression: the full-only constructors reject a worker-front-door
4069    /// config with a typed `ConfigValidation` error AT ENTRY — no full service can
4070    /// be created through any public config-based constructor under that profile.
4071    #[test]
4072    fn full_only_constructors_reject_worker_profile() -> TestResult {
4073        let config = config_with_profile("worker-front-door")?;
4074
4075        let from_config = LiminalConnectionServices::from_config(&config);
4076        assert!(
4077            matches!(from_config, Err(ServerError::ConfigValidation { .. })),
4078            "from_config must reject a worker-front-door profile with ConfigValidation, got {from_config:?}"
4079        );
4080
4081        let root = tempfile::tempdir().must("test can create an ephemeral root")?;
4082        let store = open_ephemeral_rooted(root.path(), DEFAULT_SHARD_COUNT)
4083            .must("test store for the rejection check builds")?;
4084        let from_config_with_store =
4085            LiminalConnectionServices::from_config_with_store(&config, std::sync::Arc::new(store));
4086        assert!(
4087            matches!(
4088                from_config_with_store,
4089                Err(ServerError::ConfigValidation { .. })
4090            ),
4091            "from_config_with_store must reject a worker-front-door profile with ConfigValidation"
4092        );
4093        Ok(())
4094    }
4095
4096    /// MAJOR-1 regression: the profile-aware factory itself re-runs the
4097    /// worker-front-door cross-field checks, so a directly-constructed config (one
4098    /// that never passed file-loading validation) combining the worker profile with
4099    /// full-only machinery is refused with the same typed `ConfigValidation` errors.
4100    #[test]
4101    fn build_connection_services_rejects_worker_profile_with_full_only_fields() -> TestResult {
4102        let mut config = config_with_profile("worker-front-door")?;
4103        config.channels = vec![crate::config::types::ChannelDef {
4104            name: "orders".to_owned(),
4105            schema_ref: None,
4106            durable: false,
4107            loaded_schema: None,
4108        }];
4109        config.persistence_path = Some(std::path::PathBuf::from("/tmp"));
4110
4111        let result = build_connection_services(&config);
4112        let Err(ServerError::ConfigValidation { message }) = result else {
4113            return Err(
4114                "expected ConfigValidation for worker profile with full-only fields".into(),
4115            );
4116        };
4117        assert!(message.contains("builds no channels"), "got: {message}");
4118        assert!(
4119            message.contains("builds no durable store"),
4120            "got: {message}"
4121        );
4122        Ok(())
4123    }
4124
4125    /// The required-path constructor creates the persistent database in its
4126    /// configured directory without populating the separate throwaway root.
4127    #[test]
4128    fn persistent_store_uses_configured_path_and_creates_no_temp_dir() -> TestResult {
4129        let home = tempfile::tempdir().must("test can create a temp dir")?;
4130        let ephemeral_root = tempfile::tempdir().must("test can create an ephemeral root")?;
4131
4132        let store = build_durable_store(home.path()).must("persistent store builds")?;
4133
4134        assert!(
4135            home.path().join("durability").join("config.json").exists(),
4136            "the persistent database is created under the configured path"
4137        );
4138        assert_eq!(
4139            entry_count(ephemeral_root.path())?,
4140            0,
4141            "the persistent branch creates no ephemeral guard directory"
4142        );
4143
4144        drop(store);
4145        Ok(())
4146    }
4147
4148    #[test]
4149    fn missing_path_refuses_before_any_store_is_opened() -> Result<(), Box<dyn std::error::Error>> {
4150        let root = tempfile::tempdir()?;
4151        let factory = RecordingSubsystems::rooted(root.path());
4152        let result = build_connection_services_via(&config_with_profile("full")?, &factory);
4153        assert!(
4154            matches!(result, Err(ServerError::ConfigValidation { message }) if message.contains("no store is opened"))
4155        );
4156        assert_eq!(entry_count(root.path())?, 0);
4157        assert!(factory.recorded().is_empty());
4158        Ok(())
4159    }
4160
4161    /// The store migrates itself: a `config.json` written before haematite
4162    /// required `node_cache_budget` opens on the new server with no operator
4163    /// hand, and the file afterwards carries the key — spelled exactly as the
4164    /// create arm spells it. The pre-field store is fabricated by creating a
4165    /// current one and deleting the key, which is byte-faithful to what a
4166    /// 0.8.1-era store looks like to the parser: the key absent, everything
4167    /// else in place.
4168    #[test]
4169    fn a_store_whose_config_predates_the_budget_field_opens_with_no_operator_hand() -> TestResult {
4170        let home = tempfile::tempdir().must("test can create a temp dir")?;
4171        let store = build_durable_store(home.path()).must("persistent store builds")?;
4172        drop(store);
4173
4174        let config_file = home.path().join("durability").join("config.json");
4175        let bytes = std::fs::read(&config_file).must("config.json is readable")?;
4176        let mut config: serde_json::Value =
4177            serde_json::from_slice(&bytes).must("config.json parses")?;
4178        config
4179            .as_object_mut()
4180            .must("config.json is an object")?
4181            .remove("node_cache_budget")
4182            .must("the created config carries the budget to remove")?;
4183        std::fs::write(
4184            &config_file,
4185            serde_json::to_vec_pretty(&config).must("serializes")?,
4186        )
4187        .must("test can age the config")?;
4188
4189        let reopened = build_durable_store(home.path())
4190            .must("a pre-budget store opens with no operator action")?;
4191        drop(reopened);
4192
4193        let after = std::fs::read(&config_file).must("config.json is readable after")?;
4194        let migrated: serde_json::Value =
4195            serde_json::from_slice(&after).must("migrated config parses")?;
4196        assert_eq!(
4197            migrated.get("node_cache_budget"),
4198            Some(&serde_json::Value::String("unlimited".to_owned())),
4199            "the migrated file carries the budget the create arm chooses"
4200        );
4201        Ok(())
4202    }
4203
4204    /// The migration touches only the one migratable case. A corrupt
4205    /// `config.json` still refuses — through haematite's own parse refusal,
4206    /// which is sharper than anything this crate could say — and the bytes are
4207    /// left exactly as found, so the operator diagnoses the file that failed,
4208    /// not one the migration half-rewrote.
4209    #[test]
4210    fn a_corrupt_config_still_refuses_and_is_left_untouched() -> TestResult {
4211        let home = tempfile::tempdir().must("test can create a temp dir")?;
4212        let data_dir = home.path().join("durability");
4213        std::fs::create_dir_all(&data_dir).must("test can create the data dir")?;
4214        let config_file = data_dir.join("config.json");
4215        let garbage = b"{ not json at all";
4216        std::fs::write(&config_file, garbage).must("test can write garbage")?;
4217
4218        let refused = build_durable_store(home.path());
4219        assert!(refused.is_err(), "a corrupt config refuses to open");
4220        assert_eq!(
4221            std::fs::read(&config_file)
4222                .must("config.json is still readable")?
4223                .as_slice(),
4224            garbage,
4225            "the refusing file is left byte-identical for the operator to read"
4226        );
4227        Ok(())
4228    }
4229
4230    /// A config already carrying the budget is not rewritten: reopening leaves
4231    /// the file byte-identical, so the migration cannot churn mtimes, reorder
4232    /// keys, or otherwise touch a store that needs nothing.
4233    #[test]
4234    fn a_config_already_carrying_the_budget_is_left_byte_identical() -> TestResult {
4235        let home = tempfile::tempdir().must("test can create a temp dir")?;
4236        let store = build_durable_store(home.path()).must("persistent store builds")?;
4237        drop(store);
4238
4239        let config_file = home.path().join("durability").join("config.json");
4240        let before = std::fs::read(&config_file).must("config.json is readable")?;
4241        let reopened = build_durable_store(home.path()).must("a current store reopens")?;
4242        drop(reopened);
4243        assert_eq!(
4244            std::fs::read(&config_file).must("config.json is readable after")?,
4245            before,
4246            "a config that already carries the budget is left byte-identical"
4247        );
4248        Ok(())
4249    }
4250
4251    /// A durable store created under `KeepAll` is moved onto reclaim by the
4252    /// server's next open: its `config.json` names `reclaim` afterwards, and the
4253    /// copies its old commits left are gone once reclaim has run.
4254    #[test]
4255    fn a_keep_all_durable_store_reclaims_after_the_next_open() -> TestResult {
4256        let dir = tempfile::tempdir().must("test can create a store dir")?;
4257        let data_dir = dir.path().join("durability");
4258        let database = haematite::Database::create(
4259            haematite::DatabaseConfig {
4260                data_dir: data_dir.clone(),
4261                shard_count: 1,
4262                distributed: None,
4263                executor_threads: None,
4264                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
4265                retention: haematite::Retention::KeepAll,
4266            },
4267            haematite::RuntimeOptions {
4268                long_wait_interval: std::time::Duration::from_secs(5),
4269                reclaim_start: haematite::ReclaimStart::AtOpen,
4270            },
4271        )
4272        .must("a keep-all store is created")?;
4273        for round in 0_u32..200 {
4274            let mut value = round.to_be_bytes().to_vec();
4275            value.resize(1_024, 0x5a);
4276            database.put(b"key".to_vec(), value).must("put")?;
4277            database.commit().must("commit")?;
4278        }
4279        drop(database);
4280        let grown =
4281            haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir.clone()))
4282                .must("stats of the grown store")?
4283                .totals
4284                .enumerated_bytes;
4285
4286        let reopened = super::open_or_create_database(&data_dir).must("the server opens it")?;
4287        // The server holds reclaim at open and releases it once its listeners
4288        // bind (DUR-021 R1); the test releases it the same way.
4289        reopened
4290            .release_reclaim()
4291            .must("the held reclaim is released")?;
4292        reopened.quiesce_reclaim().must("reclaim runs to rest")?;
4293        drop(reopened);
4294
4295        let config: serde_json::Value = serde_json::from_slice(
4296            &std::fs::read(data_dir.join("config.json")).must("config is readable")?,
4297        )
4298        .must("config is JSON")?;
4299        assert_eq!(
4300            config["retention"],
4301            serde_json::to_value(super::DURABLE_RETENTION).must("the retention serializes")?
4302        );
4303        let report = haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir))
4304            .must("stats of the reclaimed store")?;
4305        assert!(
4306            report.totals.enumerated_bytes <= report.totals.marked_bytes.saturating_mul(3),
4307            "the reopened store holds about its live set: enumerated {} B, marked {} B",
4308            report.totals.enumerated_bytes,
4309            report.totals.marked_bytes
4310        );
4311        assert!(
4312            report.totals.enumerated_bytes.saturating_mul(10) < grown,
4313            "the old copies are gone: {} B now against {grown} B grown",
4314            report.totals.enumerated_bytes
4315        );
4316        Ok(())
4317    }
4318
4319    /// `config.json` exactly as a store grown by an engine that never reclaimed
4320    /// carries it: `reclaim` with `past_roots` 0 is already written, so the
4321    /// open has no retention to adopt (it does move the format to version 3),
4322    /// and every superseded copy is still on disk.
4323    /// `DATA_DIR` stands for the one field the fixture rewrites.
4324    const STORED_RECLAIM_CONFIG: &str = r#"{
4325  "chunking_policy": {
4326    "chunking": "bytes-v2",
4327    "internal_target_bytes": 49152,
4328    "leaf_target_bytes": 65536
4329  },
4330  "data_dir": "DATA_DIR",
4331  "distributed": null,
4332  "executor_threads": null,
4333  "format_version": 2,
4334  "node_cache_budget": "unlimited",
4335  "retention": {
4336    "reclaim": {
4337      "past_roots": 0
4338    }
4339  },
4340  "shard_count": 8
4341}"#;
4342
4343    /// A store whose `config.json` already names reclaim, but whose history was
4344    /// grown with nothing reclaimed, is opened by the server and gives its old
4345    /// copies back with no write and no quiesce. A store with no reclaim
4346    /// record owes a startup pass, and a shard runs it on its first turn. The
4347    /// seed holds both shapes the pass must free: one row overwritten many
4348    /// times, and rows written and then deleted, as a prune leaves them.
4349    #[test]
4350    fn a_stored_reclaim_store_grown_without_reclaim_gives_its_old_copies_back_on_open() -> TestResult
4351    {
4352        let dir = tempfile::tempdir().must("test can create a store dir")?;
4353        let data_dir = dir.path().join("durability");
4354        let database = haematite::Database::create(
4355            haematite::DatabaseConfig {
4356                data_dir: data_dir.clone(),
4357                shard_count: 8,
4358                distributed: None,
4359                executor_threads: None,
4360                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
4361                retention: haematite::Retention::KeepAll,
4362            },
4363            haematite::RuntimeOptions {
4364                long_wait_interval: std::time::Duration::from_secs(5),
4365                reclaim_start: haematite::ReclaimStart::AtOpen,
4366            },
4367        )
4368        .must("a keep-all store is created")?;
4369        for round in 0_u32..200 {
4370            let mut value = round.to_be_bytes().to_vec();
4371            value.resize(1_024, 0x5a);
4372            database.put(b"key".to_vec(), value).must("put")?;
4373            database.commit().must("commit")?;
4374        }
4375        for row in 0_u32..200 {
4376            let key = [b"pruned-".as_slice(), &row.to_be_bytes()].concat();
4377            database.put(key, vec![0xa5; 1_024]).must("put")?;
4378            database.commit().must("commit")?;
4379        }
4380        for row in 0_u32..200 {
4381            let key = [b"pruned-".as_slice(), &row.to_be_bytes()].concat();
4382            database.delete(key).must("delete")?;
4383        }
4384        database.commit().must("commit")?;
4385        drop(database);
4386        let grown_report =
4387            haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir.clone()))
4388                .must("stats of the grown store")?;
4389        let grown = grown_report.totals;
4390        assert!(
4391            grown.unmarked_nodes > 0,
4392            "the seed leaves copies no root reaches"
4393        );
4394        let mut owed: std::collections::BTreeSet<usize> = grown_report
4395            .shards
4396            .iter()
4397            .filter(|shard| shard.nodes.unmarked_nodes > 0)
4398            .map(|shard| shard.shard_id)
4399            .collect();
4400
4401        let config_file = data_dir.join("config.json");
4402        let seeded = STORED_RECLAIM_CONFIG
4403            .replace("DATA_DIR", data_dir.to_str().must("the test dir is UTF-8")?);
4404        std::fs::write(&config_file, &seeded).must("config.json is written")?;
4405
4406        let reopened = super::open_or_create_database(&data_dir).must("the server opens it")?;
4407        let watch = reopened
4408            .watch_reclaim_passes(64)
4409            .must("every shard accepts a pass watch")?;
4410        // Released as the server releases it once its listeners bind (DUR-021 R1).
4411        reopened
4412            .release_reclaim()
4413            .must("the held reclaim is released")?;
4414        let mut unmarked_by_shard = std::collections::BTreeMap::new();
4415        while !owed.is_empty() {
4416            let report = watch
4417                .recv_timeout(std::time::Duration::from_secs(30))
4418                .must("each pass the open owes ends without a write or a quiesce")?;
4419            assert_eq!(
4420                report.outcome,
4421                haematite::ReclaimPassOutcome::Passed,
4422                "the pass the open owed on shard {} finished",
4423                report.shard_id
4424            );
4425            if owed.remove(&report.shard_id) {
4426                unmarked_by_shard.insert(report.shard_id, report.unmarked_bytes);
4427            }
4428        }
4429        drop(reopened);
4430
4431        let after = haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir))
4432            .must("stats of the reclaimed store")?
4433            .totals;
4434        eprintln!(
4435            "grown {} B, after {} B, {} unreachable nodes seeded, unmarked bytes by shard {unmarked_by_shard:?}",
4436            grown.enumerated_bytes, after.enumerated_bytes, grown.unmarked_nodes
4437        );
4438        assert!(
4439            after.enumerated_bytes.saturating_mul(10) < grown.enumerated_bytes,
4440            "the old copies are gone: {} B now against {} B grown",
4441            after.enumerated_bytes,
4442            grown.enumerated_bytes
4443        );
4444        let config: serde_json::Value = serde_json::from_slice(
4445            &std::fs::read(&config_file).must("config.json is readable after")?,
4446        )
4447        .must("config is JSON")?;
4448        assert_eq!(
4449            config["retention"],
4450            serde_json::to_value(super::DURABLE_RETENTION).must("the retention serializes")?,
4451            "config.json names reclaim after the open"
4452        );
4453        Ok(())
4454    }
4455}