Skip to main content

liminal_server/server/connection/
services.rs

1#[path = "publication_identity.rs"]
2mod publication_identity;
3use liminal::channel::PublicationIdentity;
4use std::collections::HashMap;
5use std::path::{Path, PathBuf};
6use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
7use std::sync::{Arc, Mutex, OnceLock, RwLock};
8use std::time::Instant;
9
10use haematite::{CommitPolicy, Database, DatabaseConfig, EventStore, Retention};
11use liminal::channel::{
12    ChannelConfig, ChannelDelivery, ChannelHandle, ChannelMode, ChannelSupervisor, Schema,
13};
14use liminal::conversation::{ConversationSupervisor, ParticipantBehaviour};
15use liminal::durability::bridge::block_on;
16#[cfg(any(test, feature = "test-support"))]
17use liminal::durability::open_ephemeral;
18use liminal::durability::{
19    CountingStore, DedupCache, DedupDecision, DurableStore, EphemeralRoot, EphemeralSweepCounts,
20    HaematiteStore, ProcessingReceipt, StoreCountsHandle,
21};
22use liminal::protocol::{MessageEnvelope, ProtocolError, SchemaId as ProtocolSchemaId};
23use std::num::NonZeroU64;
24
25use super::channel_registry::{
26    ChannelAccessError, ChannelBuildError, ChannelConfigField, ChannelDescriptor, ChannelOrigin,
27    ChannelRegistration, ChannelRegistryError, ChannelState, ChannelStatus, MAX_CHANNELS_KEY,
28    Registered, STATE_ACTIVE, STATE_QUIESCED, UNRECORDED_QUIESCE_REASON,
29};
30use super::conversation::ConnectionConversation;
31use super::services_cluster::build_channel_cluster;
32use super::services_schema::{ChannelSchema, resolve_channel_schema, resolve_schema_bytes};
33use super::worker_front_door::WorkerFrontDoorServices;
34use crate::ServerError;
35use crate::config::types::{ClusterConfig, ParticipantConfig, ServerConfig, ServiceProfile};
36use crate::config::validation::required_persistence_path;
37use crate::health::reissue::OperatorCredentialReissuer;
38use crate::health::unloadable::UnloadableConversationRecord;
39use crate::server::participant::{
40    InstalledParticipantService, ParticipantWarm, ProductionParticipantHandler,
41};
42use crate::server::self_streams::events::PublishOrigin;
43use crate::server::self_streams::sink::OriginPublication;
44use crate::server::signals::ServerSignals;
45
46pub use super::services_cluster::ChannelCluster;
47
48#[path = "../../agents/host_server.rs"]
49mod agent_host;
50#[path = "../../agents/server.rs"]
51mod agent_services;
52#[path = "../../agents/approval_boot.rs"]
53mod approval_boot;
54#[path = "../../services/site/server.rs"]
55mod site_services;
56pub use site_services::{
57    SiteDraftHolder, evaluate_subject as evaluate_site_subject, is_site_route, plan_subject,
58};
59pub(super) use site_services::{SiteFrame, site_frame_response};
60#[path = "services_lifecycle.rs"]
61mod lifecycle;
62#[path = "services_conversation.rs"]
63mod services_conversation;
64
65/// Registry of custom conversation responders, keyed by conversation subject.
66///
67/// A registered [`ParticipantBehaviour`] becomes the participant for any
68/// conversation opened on its subject; an unregistered subject has no local server.
69type ResponderRegistry = HashMap<String, Arc<dyn ParticipantBehaviour>>;
70
71/// Marker for resources retained by a connection process until unsubscribe.
72pub trait SubscriptionResource: std::fmt::Debug + Send {
73    /// Releases the library subscription resource.
74    ///
75    /// # Errors
76    /// Returns [`ServerError`] when the liminal library reports an unsubscribe failure.
77    fn unsubscribe(self: Box<Self>) -> Result<(), ServerError>;
78
79    /// Attempts to pull the next delivered envelope from the wrapped library
80    /// subscription without blocking.
81    ///
82    /// Returns `None` when the subscriber inbox is empty (or momentarily
83    /// unavailable): the connection process is the delivery pump, so a transient
84    /// empty read is simply "nothing to deliver this slice", never an error.
85    fn try_next(&mut self) -> Option<liminal::envelope::Envelope>;
86
87    /// Non-consuming availability query for the post-arm race barrier.
88    fn has_pending(&self) -> bool;
89
90    /// Whether an overflow has marked this subscription for shedding (§5). The
91    /// delivery pump sheds an overflowed subscription with a typed error frame.
92    /// Defaulted to `false`: a resource with no bounded inbox never overflows.
93    fn is_overflowed(&self) -> bool {
94        false
95    }
96
97    /// The library subscriber's pid, the id a recording publish names it by.
98    /// Defaulted to `None`: a resource with no library subscriber is never a
99    /// recipient.
100    fn subscriber_pid(&self) -> Option<u64> {
101        None
102    }
103
104    /// Reads a delivery and its durable position without acknowledging it.
105    ///
106    /// # Errors
107    /// The underlying subscription cannot be read.
108    fn try_next_positioned(
109        &mut self,
110    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
111        Ok(self.try_next().map(|envelope| (envelope, None)))
112    }
113
114    /// Checkpoints a delivery after its consumer settles it.
115    ///
116    /// # Errors
117    /// The resource has no durable acknowledgement path, or checkpointing failed.
118    fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
119        Err(ServerError::ListenerAccept {
120            message: format!("subscription cannot acknowledge durable position {position}"),
121        })
122    }
123
124    /// Reads the subscription-owned durable checkpoint.
125    ///
126    /// # Errors
127    /// The checkpoint cannot be read.
128    fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
129        Ok(None)
130    }
131}
132
133/// Library subscription resource owned by a single connection process.
134#[derive(Debug)]
135pub struct ConnectionSubscription {
136    id: u64,
137    /// Channel this subscription was opened on, carried for DIAGNOSTICS only —
138    /// the shed warning has to name the channel or an operator reading it learns
139    /// that something was shed without learning what. Empty when unset (the
140    /// scheduler-free unit fixtures, which construct through `new` and never log).
141    channel: String,
142    /// Client-chosen application stream the server delivers this subscription's
143    /// messages on (echoed on `SubscribeAck`, carried on every `Deliver`). Set by
144    /// the connection process from the `Subscribe` frame before the subscription is
145    /// stored; `0` only while momentarily unset during construction.
146    stream_id: u32,
147    selected_schema: ProtocolSchemaId,
148    resource: Box<dyn SubscriptionResource>,
149    issued: super::state::IssuedDeliveries,
150}
151
152impl ConnectionSubscription {
153    /// Creates an owned subscription resource for one connection process.
154    #[must_use]
155    pub fn new(
156        id: u64,
157        selected_schema: ProtocolSchemaId,
158        resource: Box<dyn SubscriptionResource>,
159    ) -> Self {
160        Self {
161            id,
162            channel: String::new(),
163            stream_id: 0,
164            selected_schema,
165            resource,
166            issued: super::state::IssuedDeliveries::new(0),
167        }
168    }
169
170    /// Records the channel this subscription was opened on (diagnostics).
171    ///
172    /// Additive setter rather than a fourth `new` parameter: `new` is public API
173    /// and its existing callers keep compiling byte-for-byte.
174    pub(super) fn set_channel(&mut self, channel: String) {
175        self.channel = channel;
176    }
177
178    /// The channel this subscription was opened on, or `""` when unset.
179    #[must_use]
180    pub(super) fn channel(&self) -> &str {
181        &self.channel
182    }
183
184    /// Returns the protocol subscription id.
185    #[must_use]
186    pub const fn id(&self) -> u64 {
187        self.id
188    }
189
190    /// Records the client-chosen delivery stream id for this subscription.
191    pub(super) const fn set_stream_id(&mut self, stream_id: u32) {
192        self.stream_id = stream_id;
193    }
194
195    /// Returns the client-chosen application stream id deliveries ride on.
196    #[must_use]
197    pub(super) const fn stream_id(&self) -> u32 {
198        self.stream_id
199    }
200
201    pub(super) fn set_pressure_ledger_depth(&mut self, limit: usize) {
202        self.issued = super::state::IssuedDeliveries::new(limit);
203    }
204
205    pub(super) fn issued_delivery(&mut self, sequence: u64) {
206        self.issued.record(sequence);
207    }
208
209    pub(super) fn match_pressure(&mut self, reference: &str, retain: bool) -> bool {
210        self.issued.matches(reference, retain)
211    }
212
213    /// Returns the schema selected for this subscription stream.
214    #[must_use]
215    pub const fn selected_schema(&self) -> ProtocolSchemaId {
216        self.selected_schema
217    }
218
219    /// The library subscriber's pid, when the subscription has one.
220    #[must_use]
221    pub fn subscriber_pid(&self) -> Option<u64> {
222        self.resource.subscriber_pid()
223    }
224
225    /// Attempts to pull the next delivered envelope without blocking.
226    pub(crate) fn try_next(&mut self) -> Option<liminal::envelope::Envelope> {
227        self.resource.try_next()
228    }
229
230    pub(crate) fn has_pending(&self) -> bool {
231        self.resource.has_pending()
232    }
233
234    /// Whether this subscription has been shed by an inbox overflow (§5).
235    pub(crate) fn is_overflowed(&self) -> bool {
236        self.resource.is_overflowed()
237    }
238
239    pub(super) fn unsubscribe(self) -> Result<(), ServerError> {
240        self.resource.unsubscribe()
241    }
242
243    pub(crate) fn try_next_positioned(
244        &mut self,
245    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
246        self.resource.try_next_positioned()
247    }
248
249    pub(crate) fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
250        self.resource.acknowledge_position(position)
251    }
252
253    pub(crate) fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
254        self.resource.consumer_position()
255    }
256}
257
258/// Outcome of a server publish.
259///
260/// Carries the assigned message id plus a genuine delivery ack (`delivered` = the
261/// message was accepted by at least one live subscriber on this publish, after any
262/// dedup-on-delivery suppression).
263#[derive(Clone, Copy, Debug, PartialEq, Eq)]
264pub struct PublishOutcome {
265    /// Transport-scoped message id assigned to the accepted publish.
266    ///
267    /// TRANSPORT-SCOPED, NOT A DURABLE POSITION (G1,
268    /// `docs/design/wire-durability-gaps-20260831.md`): this is a
269    /// process-local counter (`next_message_id`, reset to 1 on every
270    /// construction), so it restarts from 1 with the server and two servers
271    /// mint the same ids. It is monotonic only within one service instance's
272    /// lifetime and identifies the publish on THIS connection's wire, nothing
273    /// more. The durable position of a publish on a durable channel is the
274    /// store-assigned sequence the library now surfaces as
275    /// `ChannelDelivery::durable_position`; putting that on the wire ack is
276    /// the A1 resume lane's frame decision, not a reinterpretation of this
277    /// field.
278    pub message_id: u64,
279    /// Whether the message was genuinely delivered to a subscriber. `false` means
280    /// the publish was accepted but reached no subscriber (empty channel) or was
281    /// a duplicate suppressed by dedup-on-delivery.
282    pub delivered: bool,
283    /// Store-assigned position of the row this outcome answers for, when the
284    /// publish is known to have one. Unlike `message_id` this IS a durable
285    /// position; the connection process puts it on the wire ack only for a
286    /// connection that negotiated `PUBLISH_ACK_POSITION_VERSION`.
287    pub durable_position: Option<u64>,
288    /// Whom the publish reached, the console tap's word for it (SRV-012
289    /// R10): a subscriber, no subscriber, only subscribers that refused it,
290    /// or nobody because dedup suppressed it.
291    pub reach: Reach,
292}
293
294/// Whom a publish reached (SRV-012 R10).
295#[derive(Clone, Copy, Debug, PartialEq, Eq)]
296pub enum Reach {
297    /// At least one live subscriber took it.
298    Delivered,
299    /// The channel had no matching subscriber.
300    NoSubscriber,
301    /// Every matching subscriber refused it.
302    NoneAccepted,
303    /// A dedup-suppressed re-publish, sent to nobody.
304    Suppressed,
305}
306
307impl Reach {
308    /// The reach of a library delivery.
309    #[must_use]
310    pub const fn of(delivery: &ChannelDelivery) -> Self {
311        if delivery.is_delivered() {
312            Self::Delivered
313        } else if delivery.matched_count() == 0 {
314            Self::NoSubscriber
315        } else {
316            Self::NoneAccepted
317        }
318    }
319}
320
321/// Which channel operation is asking the roster for admission.
322///
323/// `Copy`, so the hot path allocates nothing to ask. The contrast is
324/// [`ServerError::UnsupportedOperation`]'s owned `operation: String`: that one is
325/// built on a refusal path and pays for its allocation once per refusal, while
326/// this one is consulted on every publish and every subscribe frame that reaches
327/// the connection process.
328#[derive(Clone, Copy, Debug, PartialEq, Eq)]
329pub enum ChannelOperation {
330    /// A `Publish` frame is asking.
331    Publish,
332    /// A `Subscribe` frame is asking.
333    Subscribe,
334}
335
336/// Operations that adapt wire frames to liminal library calls.
337pub trait ConnectionServices: std::fmt::Debug + Send + Sync {
338    /// Construction built-ins are dispatched before ordinary worker routing.
339    fn construction_request(
340        &self,
341        _: Option<&dyn crate::auth_lys::session::LysSession>,
342        subject: &str,
343        _: &[u8],
344        _: u64,
345    ) -> Option<Result<serde_json::Value, ServerError>> {
346        is_site_route(subject).then(|| {
347            Err(site_services::site_error(
348                crate::services::lifecycle::site::SiteError::Invalid("construction not installed"),
349            ))
350        })
351    }
352    /// Reserves the completion barrier before storing a trial registration.
353    ///
354    /// # Errors
355    /// The trial's identity, contracts or serve decision refused.
356    fn prepare_site_trial(
357        &self,
358        _: u64,
359        _: Option<&dyn crate::auth_lys::session::LysSession>,
360        _: &liminal::protocol::WorkerRegistration,
361        _: Option<&crate::services::lifecycle::ProcessGrant>,
362        _: u64,
363    ) -> Result<Option<crate::services::lifecycle::trial::TrialPermit>, ServerError> {
364        Ok(None)
365    }
366    /// Checks a Building subject before routing and before releasing a reply.
367    ///
368    /// # Errors
369    /// A trial caller or its pass was refused.
370    fn admit_site_call(
371        &self,
372        _: Option<&dyn crate::auth_lys::session::LysSession>,
373        _: &str,
374        _: u64,
375    ) -> Result<bool, ServerError> {
376        Ok(false)
377    }
378    /// Removes trial barriers when the directory removes a connection.
379    ///
380    /// # Errors
381    /// The target store or console projection failed.
382    fn site_registration_closed(&self, _: u64) -> Result<(), ServerError> {
383        Ok(())
384    }
385    /// Publishes with the identity proved at Connect; it adds no authority.
386    ///
387    /// # Errors
388    /// Identity, rights, payload or publication admission refused.
389    fn publish_identified(
390        &self,
391        connection: u64,
392        session: Option<&dyn crate::auth_lys::session::LysSession>,
393        channel: &str,
394        envelope: &MessageEnvelope,
395        key: Option<&str>,
396    ) -> Result<PublishOutcome, ServerError> {
397        crate::auth_membership::check_agent_binding(
398            session,
399            crate::auth_lys::HotAction::Publish,
400            channel,
401        )?;
402        self.publish(connection, channel, envelope, key)
403    }
404    /// Agent station state, when configured.
405    fn agents(&self) -> Option<&Arc<crate::agents::runtime::Agents>> {
406        None
407    }
408
409    /// Subscribes using the identity proved at Connect; it adds no authority.
410    ///
411    /// # Errors
412    /// Identity, rights or subscription admission refused.
413    fn subscribe_identified(
414        &self,
415        connection: u64,
416        session: Option<&dyn crate::auth_lys::session::LysSession>,
417        channel: &str,
418        schemas: &[ProtocolSchemaId],
419        install: Option<liminal::channel::InboxInstall>,
420    ) -> Result<ConnectionSubscription, ServerError> {
421        if connection == 0 {
422            return Err(crate::agents::error::AgentError::Invalid("connection id").into());
423        }
424        crate::auth_membership::check_agent_binding(
425            session,
426            crate::auth_lys::HotAction::Subscribe,
427            channel,
428        )?;
429        self.subscribe(channel, schemas, install)
430    }
431    /// Returns the complete participant service installed on this adapter.
432    ///
433    /// `None` keeps participant capability disabled even when the adapter owns a
434    /// durable store for unrelated channel traffic. The returned token is
435    /// server-sealed and atomically carries declared semantics plus durability,
436    /// making a handler-without-store activation impossible by construction.
437    fn participant_service(&self) -> Option<InstalledParticipantService> {
438        None
439    }
440
441    /// Delegates a publish request to the liminal library.
442    ///
443    /// `connection` is the publishing connection's id (its process pid, as
444    /// `apply_frame` knows it), carried as the publish origin so the console
445    /// tap attributes the message to it (SRV-012). `idempotency_key`, when `Some`, drives dedup-on-delivery: a re-publish with
446    /// the same key is delivered to subscribers at most once. The returned
447    /// [`PublishOutcome`] carries the genuine delivery ack.
448    ///
449    /// # Errors
450    /// Returns [`ServerError`] when the liminal publish operation fails.
451    fn publish(
452        &self,
453        connection: u64,
454        channel: &str,
455        envelope: &MessageEnvelope,
456        idempotency_key: Option<&str>,
457    ) -> Result<PublishOutcome, ServerError>;
458
459    /// Delegates a subscribe request to the liminal library.
460    ///
461    /// `install`, when `Some`, carries the connection's §5 shared inbox byte
462    /// budget, per-inbox fairness cap, and R3 wake notifier. The implementation
463    /// MUST install it on the subscription's inbox BEFORE the registration is
464    /// published to the channel actor (i.e. before any envelope can be
465    /// delivered), so no envelope is ever admitted uncharged, past the depth
466    /// cap, or without a wake. Implementations with no real inbox (test
467    /// stand-ins, capability-scoped profiles that refuse subscribe) may ignore
468    /// it.
469    ///
470    /// # Errors
471    /// Returns [`ServerError`] when the liminal subscribe operation fails.
472    fn subscribe(
473        &self,
474        channel: &str,
475        accepted_schemas: &[ProtocolSchemaId],
476        install: Option<liminal::channel::InboxInstall>,
477    ) -> Result<ConnectionSubscription, ServerError>;
478
479    /// Delegates unsubscribe to the liminal library.
480    ///
481    /// # Errors
482    /// Returns [`ServerError`] when the liminal unsubscribe operation fails.
483    fn unsubscribe(&self, subscription: ConnectionSubscription) -> Result<(), ServerError>;
484
485    /// Delegates conversation open to the liminal library.
486    ///
487    /// # Errors
488    /// Returns [`ServerError`] when the liminal conversation open operation fails.
489    fn open_conversation(
490        &self,
491        conversation_id: u64,
492        subject: &str,
493    ) -> Result<ConnectionConversation, ServerError>;
494
495    /// Delegates a conversation message to the liminal library.
496    ///
497    /// # Errors
498    /// Returns [`ServerError`] when the liminal conversation message operation fails.
499    fn conversation_message(
500        &self,
501        conversation: &ConnectionConversation,
502        envelope: &MessageEnvelope,
503        op_id: Option<u64>,
504    ) -> Result<(), ServerError>;
505
506    /// Delegates conversation close to the liminal library.
507    ///
508    /// # Errors
509    /// Returns [`ServerError`] when the liminal conversation close operation fails.
510    fn close_conversation(&self, conversation: ConnectionConversation) -> Result<(), ServerError>;
511
512    /// Flushes the shared durable store after channel publications have settled.
513    ///
514    /// # Errors
515    /// Returns [`ServerError`] when the shared store flush operation fails.
516    fn flush_durable_state(&self) -> Result<(), ServerError>;
517
518    /// Stops the participant warm and records the clean stop's capacity
519    /// snapshot (DUR-020 R3, R5). Called once, by the shutdown sequence,
520    /// before the final flush; an adapter with no participant does nothing.
521    fn stop_participant_warm(&self) {}
522
523    /// Joins background work that can still write to the store.
524    ///
525    /// # Errors
526    /// Returns a named shutdown failure when an owned worker cannot be joined.
527    fn finish_background_store_work(&self) -> Result<(), ServerError> {
528        Ok(())
529    }
530
531    /// Stops store-owning service workers after the final flush.
532    ///
533    /// # Errors
534    /// Returns a named shutdown failure when a service worker cannot be joined.
535    fn release_store_holders(&self) -> Result<(), ServerError> {
536        Ok(())
537    }
538
539    /// The listener is bound: whatever the start reads that no admission
540    /// needs before it is judged runs from here, behind the listening door.
541    /// An adapter with nothing to read does nothing. Answers the thread it
542    /// started, which the listener leaves to run and a test may join.
543    fn listening(self: Arc<Self>) -> Option<std::thread::JoinHandle<()>> {
544        None
545    }
546
547    /// Whether this adapter backs ordinary channel and conversation operations.
548    ///
549    /// The default is `true` — the full-service adapter serves publish, subscribe,
550    /// and conversation frames, so full mode is byte-for-byte unchanged. The
551    /// capability-scoped worker front door overrides this to `false`, letting
552    /// [`super::apply`] reject the channel/conversation frames it short-circuits on
553    /// empty connection state (`Unsubscribe`, `ConversationMessage`,
554    /// `ConversationClose`) with a typed error frame instead of silently swallowing
555    /// an operation for a resource that could never have been created in this
556    /// profile. Frames that always reach a service method (`Publish`, `Subscribe`,
557    /// `ConversationOpen`) are rejected by the front door's own method bodies and do
558    /// not consult this flag.
559    fn supports_channel_operations(&self) -> bool {
560        true
561    }
562
563    /// Whether `channel` admits `operation` right now.
564    ///
565    /// Consulted by the connection process BEFORE the operation is delegated, so
566    /// a roster refusal is typed at the moment of the decision, by the component
567    /// that made it, from the value it decided on. The alternative — classifying
568    /// an opaque failure afterwards by re-reading the roster in the error arm —
569    /// cannot tell "the roster refused this" from "something else failed while
570    /// the roster happened to change", and would put a confident wrong cause on
571    /// the wire.
572    ///
573    /// It does NOT replace the adapter's own inner check. Admission here is the
574    /// caller's guard; a service method is public and callable without a frame,
575    /// so it keeps its own.
576    ///
577    /// The default ADMITS. An adapter with no roster has nothing to say here and
578    /// its refusals travel as service errors exactly as they do today; only the
579    /// roster-owning adapter overrides this. A default body is also what keeps
580    /// this addition inside "minor": a method added without one breaks every
581    /// downstream implementor of a public trait.
582    ///
583    /// # Errors
584    /// Returns [`ChannelAccessError`] when the roster refuses the operation.
585    fn admit_channel(
586        &self,
587        operation: ChannelOperation,
588        channel: &str,
589    ) -> Result<(), ChannelAccessError> {
590        let _ = (operation, channel);
591        Ok(())
592    }
593}
594
595/// Default adapter from server wire frames to liminal channel/conversation APIs.
596#[derive(Debug)]
597pub struct LiminalConnectionServices {
598    publication_identity: Arc<publication_identity::ServerPublicationIdentity>,
599    agents: Option<Arc<crate::agents::runtime::Agents>>,
600    approval_agents: Option<Arc<crate::agents::waiting::durable::ApprovalAgents>>,
601    agent_delivery: Option<Arc<crate::agents::waiting::AgentDelivery>>,
602    site_lifecycle: OnceLock<Arc<crate::services::lifecycle::Lifecycle>>,
603    site_drafts: OnceLock<Arc<dyn SiteDraftHolder>>,
604    host_config: OnceLock<(crate::config::types::LysConfig, tokio::runtime::Handle)>,
605    draft_hook: OnceLock<std::sync::Weak<dyn crate::auth_lys::drafts::DraftDecisionHook>>,
606
607    /// The channel roster, keyed by channel name.
608    ///
609    /// Behind an [`RwLock`] over `Arc` values so a reader clones one pointer out
610    /// under the guard and works with it after the guard is released: no roster
611    /// read is ever held across a library call, and an entry handed out can
612    /// outlive a concurrent mutation of the map itself.
613    channels: RwLock<HashMap<String, Arc<ConfiguredChannel>>>,
614    export_subscriptions:
615        Mutex<Vec<std::sync::Weak<crate::link_bridge::subscriptions::ExportSubscriptions>>>,
616    /// The operator-declared bound on RUNTIME-registered channels
617    /// (`limits.max_channels`), carried verbatim from config.
618    ///
619    /// `None` is "the operator declared no bound", which refuses every runtime
620    /// registration rather than admitting an unbounded one — the roster's
621    /// aggregate size is otherwise unbounded the moment a registration API
622    /// exists. Boot-configured channels are never counted against it: they are
623    /// the bound the operator already wrote, in the file they wrote it in.
624    max_channels: Option<usize>,
625    cluster: ChannelCluster,
626    durable_store: Arc<dyn DurableStore>,
627    head_checkpoint_stop: Arc<liminal::durability::head_checkpoint_stop::HeadCheckpointStop>,
628    /// Reads the counters of every store these services built: the durable
629    /// store and, when it is a separate store, the ephemeral dedup store.
630    store_counts: StoreCountsHandle,
631    ephemeral_sweep: Option<EphemeralSweepCounts>,
632    signals: ServerSignals,
633    /// Complete participant service, installed only when semantic lifecycle
634    /// handling and its durable aggregate store are both ready.
635    participant_service: Option<InstalledParticipantService>,
636    /// The production handler's refused-load record, captured at construction.
637    ///
638    /// [`InstalledParticipantService`] holds its handler behind
639    /// `dyn ParticipantSemanticHandler`, so the concrete production handler is
640    /// no longer reachable once it is installed; the record is taken here, in
641    /// the one place the concrete handler exists, and carried out to the
642    /// startup path that publishes it onto the health endpoint. `None` is "no
643    /// participant is configured", which the operator surface reports as such
644    /// rather than as an empty refusal set.
645    unloadable_conversations: Option<UnloadableConversationRecord>,
646    /// The production handler as the A7 operator credential-re-issue authority
647    /// (§0.18), captured at construction for exactly the reason above: the
648    /// concrete handler is unreachable once erased behind
649    /// `dyn ParticipantSemanticHandler`, so the operator surface's handle on it
650    /// has to be taken in the one place it still exists. `None` is "no
651    /// participant is configured", which the route reports as such.
652    credential_reissuer: Option<Arc<dyn OperatorCredentialReissuer>>,
653    /// The participant warm's stop handle (DUR-020 R4, R5), captured at
654    /// construction for the same reason as the two above. The shutdown flush
655    /// stops the warm first, so no conversation begins restoring after the
656    /// server has begun to stop. `None` is "no participant is configured".
657    participant_warm: Option<ParticipantWarm>,
658    /// Dedup-on-delivery cache for publishes to **durable** channels. Keyed by
659    /// the per-message idempotency key carried on the publish frame; a duplicate
660    /// key is suppressed before fan-out so a subscriber receives it at most once.
661    ///
662    /// The namespace is stable across restarts by design (A1 §5): on a durable
663    /// channel the MESSAGE survives a crash too, so a surviving receipt is
664    /// truthful.
665    dedup: DedupCache,
666    /// Dedup-on-delivery cache for publishes to **ephemeral** channels
667    /// (A1 §5, graft §0.2 — the panel's sharpest crash hole).
668    ///
669    /// Identical machinery, different namespace: this one carries a per-boot
670    /// incarnation id, so a receipt and the message it vouches for share a
671    /// lifetime. Without it, `ephemeral + keyed publish -> Defer (buffered in
672    /// memory) -> complete_receipt (persisted) -> crash before the consumer
673    /// pops` loses the message while the receipt survives to suppress the
674    /// producer's retry — silent permanent loss of something the producer was
675    /// told was admitted. With it, the post-crash retry re-claims and
676    /// re-delivers, which is exactly ephemeral's at-least-once contract.
677    ///
678    /// This changes no dedup API. It changes only the namespace string the
679    /// server composes.
680    dedup_ephemeral: DedupCache,
681    /// Durable-channel idempotency keys whose publish outcome is not yet
682    /// known, held until their row is looked up by its key. A cache of the
683    /// durable in-flight claims, rebuilt from them when the services start.
684    held_keys: unresolved::HeldKeys,
685    /// The store history whose in-flight claims are still to be resolved:
686    /// some until the resolve has run, behind the listening door or at the
687    /// first keyed durable admission, whichever comes first (`unresolved_start.rs`).
688    claims_start: Mutex<Option<StoreHistory>>,
689    conversation_supervisor: Arc<ConversationSupervisor>,
690    /// Registered custom conversation responders, keyed by conversation subject.
691    ///
692    /// When a conversation is opened (`open_conversation`), the subject is looked
693    /// up here: a registered [`ParticipantBehaviour`] becomes the conversation's
694    /// participant; with no registration the local route refuses by name.
695    /// This is the seam aion #13 plugs a remote worker responder into. Interior
696    /// mutability is required because the services are shared behind `&self`.
697    responders: Mutex<ResponderRegistry>,
698    next_subscription_id: AtomicU64,
699    /// SRV-012: the console's held topology and tap.
700    console: Arc<crate::console::ConsoleHub>,
701}
702
703impl LiminalConnectionServices {
704    /// Builds library-backed services from validated server configuration.
705    ///
706    /// Durable-mode channels are backed by a shared haematite event store so
707    /// their publishes are persisted and survive the graceful-shutdown flush;
708    /// ephemeral channels carry no store.
709    ///
710    /// Full-only: a config selecting the worker-front-door profile is rejected at
711    /// entry (before any store is built), so this constructor can never build full
712    /// services for a profile that forbids them. Profile-aware callers go through
713    /// [`build_connection_services`] instead.
714    ///
715    /// # Errors
716    /// Returns [`ServerError`] when the config selects a non-full profile or a
717    /// configured channel cannot be initialized.
718    pub fn from_config(config: &ServerConfig) -> Result<Self, ServerError> {
719        Self::from_config_with_runtime(config, None)
720    }
721
722    pub(in crate::server) fn from_config_with_runtime(
723        config: &ServerConfig,
724        runtime: Option<tokio::runtime::Handle>,
725    ) -> Result<Self, ServerError> {
726        require_full_profile(config)?;
727        let path = required_persistence_path(config)?;
728        let history = StoreHistory::of(path);
729        let subsystems = RootedProductionSubsystems::new(path)?;
730        let store = subsystems.durable_store(path)?;
731        Self::from_config_with_store_and_runtime(config, store, history, &subsystems, runtime)
732    }
733
734    /// Builds services over a caller-provided durable store.
735    ///
736    /// Used by tests that need to inspect persisted state through the same store
737    /// handle the durable channels write to.
738    ///
739    /// Full-only: rejects a worker-front-door profile at entry, exactly like
740    /// [`Self::from_config`].
741    ///
742    /// # Errors
743    /// Returns [`ServerError`] when the config selects a non-full profile or a
744    /// configured channel cannot be initialized.
745    pub fn from_config_with_store(
746        config: &ServerConfig,
747        durable_store: Arc<dyn DurableStore>,
748    ) -> Result<Self, ServerError> {
749        require_full_profile(config)?;
750        #[cfg(any(test, feature = "test-support"))]
751        if config.persistence_path.is_none() {
752            return Self::from_config_with_store_via(
753                config,
754                durable_store,
755                StoreHistory::Recorded,
756                &ProductionSubsystems,
757            );
758        }
759        let subsystems = RootedProductionSubsystems::new(required_persistence_path(config)?)?;
760        Self::from_config_with_store_via(config, durable_store, StoreHistory::Recorded, &subsystems)
761    }
762
763    /// [`Self::from_config_with_store`] with the subsystem factory injected.
764    ///
765    /// The channel supervisor and conversation supervisor are constructed ONLY
766    /// through `subsystems` — there is no direct constructor call in this body —
767    /// so a factory that records as a side effect of constructing cannot have its
768    /// recording omitted (§9 D2 seam census, record-by-construction). No profile
769    /// check here: the caller (the public wrapper or the profile dispatch in
770    /// [`build_connection_services`]) has already established the full profile.
771    /// `history` says whether `durable_store` can already hold in-flight
772    /// claims for the services to look up before they judge a publish.
773    fn from_config_with_store_via(
774        config: &ServerConfig,
775        durable_store: Arc<dyn DurableStore>,
776        history: StoreHistory,
777        subsystems: &dyn SubsystemFactory,
778    ) -> Result<Self, ServerError> {
779        Self::from_config_with_store_and_runtime(config, durable_store, history, subsystems, None)
780    }
781
782    fn from_config_with_store_and_runtime(
783        config: &ServerConfig,
784        durable_store: Arc<dyn DurableStore>,
785        history: StoreHistory,
786        subsystems: &dyn SubsystemFactory,
787        runtime: Option<tokio::runtime::Handle>,
788    ) -> Result<Self, ServerError> {
789        let head_checkpoint_stop = Arc::new(
790            liminal::durability::head_checkpoint_stop::HeadCheckpointStop::new(durable_store),
791        );
792        let durable_store: Arc<dyn DurableStore> = head_checkpoint_stop.clone();
793        // Build ONE shared channel supervisor for the whole server. When a
794        // [cluster] section is present it is distribution-enabled, so every
795        // channel actor and subscriber shares the clustered scheduler the cluster
796        // attaches its process-group transport to (SRV-005, Constraint B).
797        let cluster = subsystems.channel_cluster(config.cluster.as_ref())?;
798        let mut channels = HashMap::new();
799        for channel in &config.channels {
800            // Resolve the channel's real JSON Schema (loaded from `schema_ref`
801            // during config validation) or the permissive empty schema when the
802            // channel declared none. The protocol schema id advertised at
803            // subscribe time is derived from the SAME schema bytes so an SDK
804            // deriving ids from schema bytes converges on it.
805            let resolved = resolve_channel_schema(channel);
806            let configured = build_configured_channel(
807                &channel.name,
808                resolved,
809                channel.durable,
810                ChannelOrigin::BootConfigured,
811                &durable_store,
812                cluster.supervisor(),
813            )
814            .map_err(|error| ServerError::ConfigValidation {
815                message: error.boot_message(&channel.name),
816            })?;
817            channels.insert(channel.name.clone(), Arc::new(configured));
818        }
819        let conversation_supervisor = subsystems.conversation_supervisor()?;
820        let dedup = history.delivery_dedup(Arc::clone(&durable_store));
821        // A1 §5: the ephemeral namespace is scoped to THIS incarnation, so an
822        // ephemeral receipt cannot outlive the in-memory message it vouches for.
823        // No earlier process used it, so it starts empty.
824        let ephemeral_store = ephemeral_dedup_store(&durable_store, history, subsystems);
825        let store_counts = combined_store_counts(&durable_store, &ephemeral_store);
826        let dedup_ephemeral = DedupCache::new_empty(ephemeral_store, ephemeral_dedup_namespace());
827        // Production participant activation (LP gap closure, Part B): the
828        // deployment's [participant] section installs the ONE production
829        // semantic handler, sealed together with the same durable store the
830        // conversation logs live in, under the configured wire-frame limit.
831        // No section, no service — the capability bit stays off and the
832        // connection path is byte-identical to the pre-activation build.
833        let signals = ServerSignals::default();
834        let installed_participant = config
835            .participant
836            .as_ref()
837            .map(|participant| install_participant(&durable_store, participant, &signals))
838            .transpose()?;
839        let device = config
840            .cluster
841            .as_ref()
842            .map_or("local", |cluster| cluster.node_name.as_str());
843        let console = Arc::new(console_hub(
844            device,
845            &channels,
846            config.persistence_path.is_some(),
847            config.auth.as_ref().and_then(|auth| auth.lys.as_ref()),
848        )?);
849        signals.install_console(Arc::clone(&console))?;
850        cluster.observer().set_console(Arc::clone(&console))?;
851        let (participant_service, unloadable_conversations, credential_reissuer, participant_warm) =
852            installed_participant.map_or((None, None, None, None), |installed| {
853                (
854                    Some(installed.service),
855                    Some(installed.unloadable_conversations),
856                    Some(installed.credential_reissuer),
857                    Some(installed.warm),
858                )
859            });
860        let (agents, approval_agents, agent_delivery) = Self::configured_agents(
861            config,
862            Arc::clone(&durable_store),
863            cluster.supervisor().clone(),
864            &console,
865            runtime,
866        )?;
867        let services = Self {
868            channels: RwLock::new(channels),
869            export_subscriptions: Mutex::new(Vec::new()),
870            max_channels: config.limits.max_channels,
871            cluster,
872            durable_store,
873            head_checkpoint_stop,
874            store_counts,
875            ephemeral_sweep: subsystems.ephemeral_sweep_counts(),
876            signals,
877            participant_service,
878            unloadable_conversations,
879            credential_reissuer,
880            participant_warm,
881            dedup,
882            dedup_ephemeral,
883            held_keys: unresolved::HeldKeys::default(),
884            // Every durable in-flight claim is looked up by its key before the
885            // services judge a keyed durable publish: behind the listening
886            // door, or by that publish itself when it comes first.
887            claims_start: Mutex::new(Some(history)),
888            conversation_supervisor,
889            responders: Mutex::new(HashMap::new()),
890            publication_identity: publication_identity::ServerPublicationIdentity::new()?,
891            next_subscription_id: AtomicU64::new(1),
892            console,
893            agents,
894            approval_agents,
895            agent_delivery,
896            site_lifecycle: OnceLock::new(),
897            site_drafts: OnceLock::new(),
898            host_config: OnceLock::new(),
899            draft_hook: OnceLock::new(),
900        };
901        services.install_roster_identity()?;
902        Ok(services)
903    }
904
905    /// Builds services with no configured channels.
906    ///
907    /// # Errors
908    /// Returns [`ServerError`] when the conversation supervisor scheduler cannot start.
909    #[cfg(any(test, feature = "test-support"))]
910    pub fn empty() -> Result<Self, ServerError> {
911        let conversation_supervisor = ProductionSubsystems.conversation_supervisor()?;
912        let store =
913            open_ephemeral(DEFAULT_SHARD_COUNT).map_err(|error| ServerError::ConfigValidation {
914                message: format!("failed to open test durable store: {error}"),
915            })?;
916        let head_checkpoint_stop = Arc::new(
917            liminal::durability::head_checkpoint_stop::HeadCheckpointStop::new(Arc::new(
918                CountingStore::new(store),
919            )),
920        );
921        let durable_store: Arc<dyn DurableStore> = head_checkpoint_stop.clone();
922        let store_counts = combined_store_counts(&durable_store, &durable_store);
923        let dedup = StoreHistory::Fresh.delivery_dedup(Arc::clone(&durable_store));
924        // A1 §5: the ephemeral namespace is scoped to THIS incarnation, so an
925        // ephemeral receipt cannot outlive the in-memory message it vouches for.
926        let dedup_ephemeral =
927            DedupCache::new_empty(Arc::clone(&durable_store), ephemeral_dedup_namespace());
928        let signals = ServerSignals::default();
929        let console = Arc::new(console_hub("local", &HashMap::new(), false, None)?);
930        signals.install_console(Arc::clone(&console))?;
931        let cluster = build_channel_cluster(None)?;
932        cluster.observer().set_console(Arc::clone(&console))?;
933        Ok(Self {
934            channels: RwLock::new(HashMap::new()),
935            export_subscriptions: Mutex::new(Vec::new()),
936            // No config, so no declared bound: this builder serves tests and
937            // callers with no channels at all, and a registration against it
938            // refuses `CapNotConfigured` exactly as an undeclared deployment's
939            // would.
940            max_channels: None,
941            cluster,
942            durable_store,
943            head_checkpoint_stop,
944            store_counts,
945            ephemeral_sweep: None,
946            signals,
947            participant_service: None,
948            unloadable_conversations: None,
949            credential_reissuer: None,
950            participant_warm: None,
951            dedup,
952            dedup_ephemeral,
953            held_keys: unresolved::HeldKeys::default(),
954            claims_start: Mutex::new(None),
955            conversation_supervisor,
956            responders: Mutex::new(HashMap::new()),
957            publication_identity: publication_identity::ServerPublicationIdentity::new()?,
958            next_subscription_id: AtomicU64::new(1),
959            console,
960            agents: None,
961            approval_agents: None,
962            agent_delivery: None,
963            site_lifecycle: OnceLock::new(),
964            site_drafts: OnceLock::new(),
965            host_config: OnceLock::new(),
966            draft_hook: OnceLock::new(),
967        })
968    }
969
970    /// The retained counts from the root's one startup sweep.
971    #[must_use]
972    pub const fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
973        self.ephemeral_sweep
974    }
975
976    /// The shared channel supervisor + cluster resolver backing this service.
977    ///
978    /// The server runtime uses this to attach the cluster to the channel
979    /// supervisor's clustered scheduler (SRV-005).
980    #[must_use]
981    pub const fn channel_cluster(&self) -> &ChannelCluster {
982        &self.cluster
983    }
984
985    /// Returns the shared durable store backing this service's durable channels.
986    #[must_use]
987    pub fn durable_store(&self) -> Arc<dyn DurableStore> {
988        Arc::clone(&self.durable_store)
989    }
990
991    /// The console's held topology and tap (SRV-012).
992    #[must_use]
993    pub fn console(&self) -> &crate::console::ConsoleHub {
994        &self.console
995    }
996
997    /// Applies a topology change; a poisoned model is counted by the hub
998    /// (`topology_emit_failures`) and logged, never recovered.
999    fn console_changed(&self, change: crate::console::TopologyChange) {
1000        self.signals.topology(change);
1001    }
1002
1003    /// Conversation subjects with a registered responder, sorted. Read from
1004    /// the in-memory registry; touches no store.
1005    ///
1006    /// # Errors
1007    /// Returns [`ServerError`] when the responder registry lock is poisoned.
1008    pub fn responder_subjects(&self) -> Result<Vec<String>, ServerError> {
1009        let responders = self.lock_responders()?;
1010        let mut subjects: Vec<String> = responders.keys().cloned().collect();
1011        drop(responders);
1012        subjects.sort();
1013        Ok(subjects)
1014    }
1015
1016    /// Restores every participant conversation the warm has not yet reached,
1017    /// on the caller's thread, and answers how many refused (DUR-020 R4).
1018    /// The store-count journeys call it so their windows do not race the
1019    /// warm thread; the server itself never waits for the warm.
1020    ///
1021    /// # Errors
1022    /// Returns [`ServerError`] when the pending-restore lock is poisoned.
1023    pub fn finish_participant_warm(&self) -> Result<usize, ServerError> {
1024        self.participant_warm.as_ref().map_or(Ok(0), |warm| {
1025            warm.finish()
1026                .map_err(|error| ServerError::ParticipantStartupRestore {
1027                    message: error.to_string(),
1028                })
1029        })
1030    }
1031
1032    /// Reads the store work these services have done since they were built,
1033    /// across every store they built.
1034    ///
1035    /// Empty (every snapshot zero) when the durable store was supplied by the
1036    /// caller and does not count.
1037    #[must_use]
1038    pub fn store_counts(&self) -> StoreCountsHandle {
1039        self.store_counts.clone()
1040    }
1041
1042    /// The production participant handler's refused-load record, when a
1043    /// participant is configured.
1044    ///
1045    /// The server's startup path publishes this onto the health endpoint so
1046    /// `GET /unloadable-conversations` answers from the same record the
1047    /// handler writes. `None` means no participant is configured at all, which
1048    /// the surface reports as a distinct state from "nothing was refused".
1049    #[must_use]
1050    pub fn unloadable_conversation_record(&self) -> Option<UnloadableConversationRecord> {
1051        self.unloadable_conversations.clone()
1052    }
1053
1054    /// The A7 operator credential-re-issue authority, when a participant is
1055    /// configured (§0.18).
1056    ///
1057    /// The startup path publishes this onto the health endpoint so
1058    /// `POST /operator/credential-reissue` reaches the SAME serialized
1059    /// participant-state point every wire operation does. `None` means no
1060    /// participant is configured, which the route reports as a distinct state
1061    /// from an unknown identity.
1062    #[must_use]
1063    pub fn credential_reissuer(&self) -> Option<Arc<dyn OperatorCredentialReissuer>> {
1064        self.credential_reissuer.clone()
1065    }
1066
1067    /// Installs a complete participant bundle in full-service supervisor tests.
1068    ///
1069    /// Production full services intentionally stay disabled until a concrete
1070    /// lifecycle handler exists. This consuming test builder exercises the real
1071    /// supervisor activation path without allowing an already-shared adapter to
1072    /// change capability posture.
1073    #[cfg(test)]
1074    #[must_use]
1075    pub(crate) fn with_participant_service(
1076        mut self,
1077        participant_service: InstalledParticipantService,
1078    ) -> Self {
1079        self.participant_service = Some(participant_service);
1080        self
1081    }
1082
1083    /// Returns the conversation supervisor backing supervised conversations.
1084    ///
1085    /// Tests use this to reach the underlying beamr scheduler so they can spawn
1086    /// or terminate participant processes and exercise crash detection.
1087    #[must_use]
1088    pub fn conversation_supervisor(&self) -> Arc<ConversationSupervisor> {
1089        Arc::clone(&self.conversation_supervisor)
1090    }
1091
1092    /// Registers a conversation responder for a routing `subject`.
1093    ///
1094    /// A later conversation on this exact subject runs the registered behaviour
1095    /// as a supervised linked participant. A registration replaces the previous
1096    /// behaviour and returns it to the caller.
1097    ///
1098    /// # Errors
1099    /// Returns [`ServerError`] when the responder registry lock is poisoned.
1100    pub fn register_responder(
1101        &self,
1102        subject: impl Into<String>,
1103        behaviour: Arc<dyn ParticipantBehaviour>,
1104    ) -> Result<Option<Arc<dyn ParticipantBehaviour>>, ServerError> {
1105        let subject = subject.into();
1106        let mut responders = self.lock_responders()?;
1107        let previous = responders.insert(subject.clone(), behaviour);
1108        drop(responders);
1109        self.console_changed(crate::console::TopologyChange::Service {
1110            subject,
1111            present: true,
1112        });
1113        Ok(previous)
1114    }
1115
1116    /// Removes the custom responder registered for `subject`, if any.
1117    ///
1118    /// After removal local calls on the subject are refused with `no_server`.
1119    /// Returns the removed behaviour when one was registered.
1120    ///
1121    /// # Errors
1122    /// Returns [`ServerError`] when the responder registry lock is poisoned.
1123    pub fn unregister_responder(
1124        &self,
1125        subject: &str,
1126    ) -> Result<Option<Arc<dyn ParticipantBehaviour>>, ServerError> {
1127        let mut responders = self.lock_responders()?;
1128        let removed = responders.remove(subject);
1129        drop(responders);
1130        if removed.is_some() {
1131            self.console_changed(crate::console::TopologyChange::Service {
1132                subject: subject.to_owned(),
1133                present: false,
1134            });
1135        }
1136        Ok(removed)
1137    }
1138
1139    /// Locks the responder registry, mapping a poisoned lock to a [`ServerError`]
1140    /// rather than panicking (the workspace denies `unwrap`/`expect`/`panic`).
1141    fn lock_responders(&self) -> Result<std::sync::MutexGuard<'_, ResponderRegistry>, ServerError> {
1142        self.responders
1143            .lock()
1144            .map_err(|_poisoned| ServerError::ListenerAccept {
1145                message: "responder registry lock poisoned".to_owned(),
1146            })
1147    }
1148
1149    /// Takes the channel roster's read guard, mapping a poisoned lock to a
1150    /// [`ServerError`] rather than panicking (the workspace denies
1151    /// `unwrap`/`expect`/`panic`), exactly as [`Self::lock_responders`] does for
1152    /// the responder registry.
1153    ///
1154    /// Every caller clones the `Arc` it needs out of the returned guard and drops
1155    /// the guard before doing anything else: the roster lock is never held across
1156    /// a call into the liminal library.
1157    fn read_channels(
1158        &self,
1159    ) -> Result<std::sync::RwLockReadGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>, ServerError>
1160    {
1161        self.channels
1162            .read()
1163            .map_err(|_poisoned| ServerError::ListenerAccept {
1164                message: "channel roster lock poisoned".to_owned(),
1165            })
1166    }
1167
1168    /// Subscribes to a configured channel and returns the raw library
1169    /// subscription handle so a test can drain the subscriber inbox directly and
1170    /// observe exactly which messages reached a subscriber.
1171    #[cfg(test)]
1172    pub(in crate::server) fn channel_handle_for_test(
1173        &self,
1174        channel: &str,
1175    ) -> Result<ChannelHandle, ServerError> {
1176        self.read_channels()?
1177            .get(channel)
1178            .map(|configured| configured.handle.clone())
1179            .ok_or_else(|| ServerError::ListenerAccept {
1180                message: format!("channel '{channel}' is not configured"),
1181            })
1182    }
1183
1184    #[cfg(test)]
1185    pub(crate) fn subscribe_handle_for_test(
1186        &self,
1187        channel: &str,
1188    ) -> Result<liminal::channel::SubscriptionHandle, ServerError> {
1189        self.subscribe_handle_for_test_with_install(channel, None)
1190    }
1191
1192    /// [`Self::subscribe_handle_for_test`] carrying an explicit install, so a
1193    /// test can put a subscriber under a REAL §5 regime (byte budget, fairness
1194    /// cap) and observe what a publish the bus dropped does to the dedup
1195    /// receipt.
1196    #[cfg(test)]
1197    pub(crate) fn subscribe_handle_for_test_with_install(
1198        &self,
1199        channel: &str,
1200        install: Option<liminal::channel::InboxInstall>,
1201    ) -> Result<liminal::channel::SubscriptionHandle, ServerError> {
1202        let channels = self.read_channels()?;
1203        let configured =
1204            channels
1205                .get(channel)
1206                .map(Arc::clone)
1207                .ok_or_else(|| ServerError::ListenerAccept {
1208                    message: format!("channel '{channel}' is not configured"),
1209                })?;
1210        drop(channels);
1211        install
1212            .map_or_else(
1213                || configured.handle.subscribe(),
1214                |install| configured.handle.subscribe_with_install(install),
1215            )
1216            .map_err(|error| ServerError::ListenerAccept {
1217                message: format!("liminal subscribe failed for channel '{channel}': {error}"),
1218            })
1219    }
1220
1221    /// Claims the delivery right for an idempotency key.
1222    ///
1223    /// Returns `Ok(true)` when this is the first publish for the key (the caller
1224    /// may deliver), and `Ok(false)` when the key was already claimed/completed (a
1225    /// duplicate the caller must suppress). The dedup cache is driven synchronously
1226    /// over the in-memory haematite store via the durable bridge.
1227    fn claim_delivery(cache: &DedupCache, key: &str) -> Result<bool, ServerError> {
1228        let decision = block_on(cache.claim_or_get(key, dedup_timestamp_millis()))
1229            .map_err(|error| ServerError::ListenerAccept {
1230                message: format!("dedup bridge failed for key '{key}': {error}"),
1231            })?
1232            .map_err(|error| ServerError::ListenerAccept {
1233                message: format!("dedup claim failed for key '{key}': {error}"),
1234            })?;
1235        Ok(matches!(decision, DedupDecision::Claimed))
1236    }
1237
1238    /// The dedup namespace a publish to `channel` belongs in (A1 §5).
1239    ///
1240    /// Durability decides, because durability is what determines whether a
1241    /// receipt can outlive the message it vouches for. A durable channel's
1242    /// message survives a crash, so its receipt may; an ephemeral channel's
1243    /// does not, so its receipt must not.
1244    const fn dedup_for(&self, channel: &ConfiguredChannel) -> &DedupCache {
1245        match channel.handle.config().mode {
1246            ChannelMode::Durable => &self.dedup,
1247            ChannelMode::Ephemeral => &self.dedup_ephemeral,
1248        }
1249    }
1250
1251    /// Releases a dangling in-flight dedup claim after a failed delivery.
1252    ///
1253    /// Best-effort: a release failure cannot mask the original publish error, so
1254    /// this returns nothing and logs at `error` level instead of surfacing. It is
1255    /// never silent — the leak (a permanently suppressed key) must be observable.
1256    /// `release_claim` itself never clobbers a stored receipt, so calling it on the
1257    /// failure path is safe even if a concurrent completion raced ahead.
1258    fn release_claim(cache: &DedupCache, key: &str) {
1259        match block_on(cache.release_claim(key)) {
1260            Ok(Ok(())) => {}
1261            Ok(Err(error)) => {
1262                tracing::error!(
1263                    idempotency_key = key,
1264                    %error,
1265                    "failed to release dedup claim after publish failure; key may stay suppressed"
1266                );
1267            }
1268            Err(error) => {
1269                tracing::error!(
1270                    idempotency_key = key,
1271                    %error,
1272                    "dedup release bridge failed after publish failure; key may stay suppressed"
1273                );
1274            }
1275        }
1276    }
1277
1278    /// The next connection-level message id.
1279    fn next_message_id(&self) -> Result<u64, ServerError> {
1280        self.publication_identity
1281            .next()
1282            .map(|message| message.id)
1283            .map_err(|error| ServerError::ListenerAccept {
1284                message: error.to_string(),
1285            })
1286    }
1287
1288    /// The outcome of a publish the channel took, recorded as accepted with
1289    /// its genuine subscriber deliveries. The delivered count (0 for an empty
1290    /// channel) is the same signal the delivery ack is derived from.
1291    fn delivered(delivery: &ChannelDelivery) -> Result<PublishOutcome, ServerError> {
1292        Self::record_delivery_counts(delivery);
1293        let message = delivery
1294            .delivery_id()
1295            .ok_or_else(|| ServerError::ListenerAccept {
1296                message: "publication identity missing after admission".into(),
1297            })?;
1298        Ok(PublishOutcome {
1299            message_id: message.id,
1300            delivered: delivery.is_delivered(),
1301            durable_position: delivery.durable_position(),
1302            reach: Reach::of(delivery),
1303        })
1304    }
1305}
1306
1307#[path = "deferred_store.rs"]
1308mod deferred_store;
1309#[path = "unresolved.rs"]
1310mod unresolved;
1311#[path = "unresolved_start.rs"]
1312mod unresolved_start;
1313#[cfg(test)]
1314#[path = "unresolved_start_tests.rs"]
1315mod unresolved_start_tests;
1316
1317use deferred_store::{DeferredStore, ThrowawayStoreOpener};
1318use unresolved_start::StoreHistory;
1319
1320/// The runtime channel-registration surface.
1321///
1322/// # Reachability: a test seam today, not a production capability
1323///
1324/// Production channels come exclusively from boot configuration; no production
1325/// code path calls `register_channel` (caller census at `c381f01`, G2 in
1326/// `docs/design/wire-durability-gaps-20260831.md` — the only callers are
1327/// `services_registry_tests.rs` and `apply_admission_tests.rs`). The method is
1328/// kept because its semantics — cap gating, typed identity refusals,
1329/// idempotent re-registration — are the contract a future runtime admission
1330/// path must satisfy, and the tests below pin that contract now. Wiring an
1331/// actual runtime admission route (an operator endpoint, a wire frame) is a
1332/// product decision that has NOT been taken; nothing here implies it. Reading
1333/// this method as an existing operator capability is the false-affordance
1334/// mistake this paragraph exists to prevent.
1335///
1336/// Inherent methods, not trait methods, and deliberately so: this is
1337/// authority-moving vocabulary that belongs to the ONE adapter owning a channel
1338/// roster. Putting `register`/`quiesce` on the public [`ConnectionServices`]
1339/// trait would break every external implementor and would hand
1340/// register/quiesce words to a profile that serves no channels at all.
1341///
1342/// The same seam as the existing runtime-mutation API on this type
1343/// (`register_responder`/`unregister_responder`): `&self`, interior mutability,
1344/// typed `Result`.
1345impl LiminalConnectionServices {
1346    /// Registers `spec` on the live roster.
1347    ///
1348    /// Test seam today: see the reachability note on this `impl` block — no
1349    /// production path reaches this method.
1350    ///
1351    /// Idempotent when an entry of that name already has an IDENTICAL
1352    /// configuration — mode, protocol schema id, and schema document, all three
1353    /// — and refuses typed, naming the first differing field, otherwise. An
1354    /// identical registration against a boot-configured entry answers
1355    /// [`Registered::AlreadyIdentical`] and leaves its origin alone: flipping it
1356    /// would make the entry lie about its restart fate and would move it into
1357    /// the counted population without a channel having been created.
1358    ///
1359    /// The cap is consulted first, in two steps with different reach. An absent
1360    /// `limits.max_channels` refuses EVERY call, identical or not, before the
1361    /// roster is read at all: a deployment that has declared no bound has not
1362    /// said what it admits, and unbounded-by-default is not a bound. A REACHED
1363    /// cap refuses only a call that would create an entry — it gates the insert,
1364    /// which is the population it bounds. An identical re-registration inserts
1365    /// nothing and so is answered `AlreadyIdentical` even at a full roster;
1366    /// refusing it would break idempotency at exactly the boundary a projector
1367    /// re-projecting its record crosses.
1368    ///
1369    /// # Errors
1370    /// Returns [`ChannelRegistryError`] when no cap is configured, the cap is
1371    /// reached, the name exists with a different configuration, the schema bytes
1372    /// do not parse or compile, durable initialization over the shared store
1373    /// fails, or the roster lock is poisoned.
1374    pub fn register_channel(
1375        &self,
1376        spec: &ChannelRegistration,
1377    ) -> Result<Registered, ChannelRegistryError> {
1378        // The cap is consulted before the roster is touched: an undeclared bound
1379        // refuses every runtime registration, identical or not, because the
1380        // deployment has not said what it admits.
1381        let Some(limit) = self.max_channels else {
1382            return Err(ChannelRegistryError::CapNotConfigured {
1383                cap: MAX_CHANNELS_KEY,
1384            });
1385        };
1386        // Schema resolution is pure (a JSON parse and a digest), so it runs with
1387        // no lock held and its result serves BOTH the identity comparison and
1388        // the construction below.
1389        let resolved = resolve_schema_bytes(spec.schema_bytes.as_deref()).map_err(|error| {
1390            ChannelRegistryError::SchemaRejected {
1391                name: spec.name.clone(),
1392                message: error.to_string(),
1393            }
1394        })?;
1395
1396        // Fast paths, under a READ lock: an already-identical registration
1397        // builds nothing, and a full roster refuses before paying for a
1398        // construction it would discard. Neither is the authority — the write
1399        // lock below re-decides both, because the roster can move in between.
1400        let (existing, registered_count) = {
1401            let channels = self.read_roster()?;
1402            let existing = channels.get(&spec.name).map(Arc::clone);
1403            let registered_count = runtime_registered_count(&channels);
1404            drop(channels);
1405            (existing, registered_count)
1406        };
1407        if let Some(existing) = existing {
1408            return compare_registration(&existing, spec, &resolved);
1409        }
1410        if registered_count >= limit {
1411            return Err(ChannelRegistryError::CapReached {
1412                cap: MAX_CHANNELS_KEY,
1413                limit,
1414            });
1415        }
1416
1417        // Construction runs with NO lock held. A durable channel recovers its
1418        // per-partition sequence counters from the store here, which is O(stream
1419        // length) in store reads; holding the roster lock across it would put a
1420        // slow store walk on every connection's publish and subscribe path.
1421        let configured = build_configured_channel(
1422            &spec.name,
1423            resolved,
1424            spec.durable,
1425            ChannelOrigin::RuntimeRegistered,
1426            &self.durable_store,
1427            self.cluster.supervisor(),
1428        )
1429        .map_err(|error| error.into_registry_error(&spec.name))?;
1430
1431        // The authoritative decision: identity, cap, and insert under ONE write
1432        // lock, so the count that admitted the entry and the insert that added
1433        // it cannot be separated by a concurrent registration.
1434        let mut exporters = self.export_subscriptions.lock().map_err(|_| {
1435            ChannelRegistryError::RosterUnavailable {
1436                message: "export subscription registry lock poisoned".to_owned(),
1437            }
1438        })?;
1439        exporters.retain(|exporter| exporter.strong_count() > 0);
1440        let mut channels = self.write_roster()?;
1441        let raced = channels.get(&spec.name).map(Arc::clone);
1442        let registered_count = runtime_registered_count(&channels);
1443        if raced.is_none() && registered_count < limit {
1444            let mut prepared = Vec::new();
1445            for exporter in exporters.iter().filter_map(std::sync::Weak::upgrade) {
1446                if let Some(binding) = exporter.binding(&spec.name) {
1447                    let refused = |error: ServerError| ChannelRegistryError::RosterUnavailable {
1448                        message: format!(
1449                            "export '{}' cannot subscribe to '{}': {error}",
1450                            exporter.link(),
1451                            spec.name
1452                        ),
1453                    };
1454                    let subscription = self
1455                        .build_subscription(
1456                            &spec.name,
1457                            &configured.handle,
1458                            configured.protocol_schema,
1459                            Some(binding.install),
1460                            Some(&binding.consumer),
1461                        )
1462                        .map_err(refused)?;
1463                    let subscription =
1464                        crate::link_bridge::subscriptions::PreparedSubscription::new(
1465                            binding.prefix,
1466                            spec.name.clone(),
1467                            subscription,
1468                        )
1469                        .map_err(refused)?;
1470                    prepared.push((exporter, subscription));
1471                }
1472            }
1473            // Bind every exporter before a publish can find the new roster entry.
1474            for (exporter, subscription) in prepared {
1475                exporter.attach(subscription);
1476            }
1477            self.identify_registered(&configured)?;
1478            channels.insert(spec.name.clone(), Arc::new(configured));
1479            drop(channels);
1480            drop(exporters);
1481            self.console_changed(crate::console::TopologyChange::Channel {
1482                name: spec.name.clone(),
1483                life: line_life(spec.durable),
1484                state: crate::console::records::LineState::Live,
1485            });
1486            return Ok(Registered::Created);
1487        }
1488        drop(channels);
1489        drop(exporters);
1490        if let Some(raced) = raced {
1491            // A racer registered this name while the channel above was being
1492            // built. The built entry is dropped unused: it owns no actor (the
1493            // actor is spawned lazily on first use) and its durable
1494            // construction only READ the store, so discarding it changes
1495            // nothing an observer could see.
1496            return compare_registration(&raced, spec, &schema_of(&configured));
1497        }
1498        Err(ChannelRegistryError::CapReached {
1499            cap: MAX_CHANNELS_KEY,
1500            limit,
1501        })
1502    }
1503
1504    /// Moves `name` from active to quiesced with a named `reason`. ONE-WAY.
1505    ///
1506    /// New publishes and new subscribes are refused afterwards, carrying the
1507    /// reason. Existing subscriptions are UNTOUCHED: nothing revokes a
1508    /// subscription handle, the actor's subscriber list is not walked, no EXIT
1509    /// is sent, and the channel actor keeps running. Quiesce is a roster-level
1510    /// admission decision, not an actor command.
1511    ///
1512    /// Re-quiescing under the IDENTICAL reason is `Ok(())`; a DIFFERENT reason
1513    /// refuses, carrying the reason already on record.
1514    ///
1515    /// The return does NOT mean "no new subscriber can appear". A subscribe that
1516    /// has already passed admission completes and gets its stream — the
1517    /// linearisation point is the admission read, not the subscribe's
1518    /// completion. A consumer that needs "nobody is attached" must observe
1519    /// attachment directly.
1520    ///
1521    /// # Errors
1522    /// Returns [`ChannelRegistryError`] when the name is not registered, is
1523    /// already quiesced under a different reason, or the roster lock is
1524    /// poisoned.
1525    pub fn quiesce_channel(
1526        &self,
1527        name: &str,
1528        reason: impl Into<String>,
1529    ) -> Result<(), ChannelRegistryError> {
1530        let reason = reason.into();
1531        // A READ lock: the state machine lives on the ENTRY, not in the map, so
1532        // the operation the design most wants to be safe never blocks a reader.
1533        let configured = {
1534            let channels = self.read_roster()?;
1535            channels.get(name).map(Arc::clone).ok_or_else(|| {
1536                ChannelRegistryError::NotRegistered {
1537                    name: name.to_owned(),
1538                }
1539            })?
1540        };
1541        configured.quiesce(name, &reason)?;
1542        self.console_changed(crate::console::TopologyChange::Channel {
1543            name: name.to_owned(),
1544            life: line_life(configured.durable),
1545            state: crate::console::records::LineState::Quiesced,
1546        });
1547        Ok(())
1548    }
1549
1550    /// Cheap typed probe: one roster read plus one atomic load.
1551    ///
1552    /// Touches no actor and therefore CANNOT spawn one. Every handle accessor
1553    /// that could answer a question about a channel's activity routes through
1554    /// the lazy-spawn path, so a probe built on one would materialise the actor
1555    /// of the idle channel it was asked about — turning a read into a side
1556    /// effect. This reads the roster entry's own recorded fields and nothing
1557    /// else.
1558    ///
1559    /// # Errors
1560    /// Returns [`ChannelRegistryError::RosterUnavailable`] only.
1561    pub fn channel_status(&self, name: &str) -> Result<ChannelStatus, ChannelRegistryError> {
1562        let configured = {
1563            let channels = self.read_roster()?;
1564            channels.get(name).map(Arc::clone)
1565        };
1566        Ok(
1567            configured.map_or(ChannelStatus::NotRegistered, |configured| {
1568                status_of(&configured)
1569            }),
1570        )
1571    }
1572
1573    /// Every roster entry with its status, sorted by name, from ONE roster
1574    /// read (SRV-012 R7: the console's topology snapshot).
1575    ///
1576    /// # Errors
1577    /// Returns [`ChannelRegistryError::RosterUnavailable`] when the roster
1578    /// lock is poisoned.
1579    pub fn channel_census(&self) -> Result<Vec<(String, ChannelStatus)>, ChannelRegistryError> {
1580        let mut census: Vec<(String, ChannelStatus)> = self
1581            .read_roster()?
1582            .iter()
1583            .map(|(name, configured)| (name.clone(), status_of(configured)))
1584            .collect();
1585        census.sort_by(|left, right| left.0.cmp(&right.0));
1586        Ok(census)
1587    }
1588
1589    /// The whole roster: one minimal descriptor per entry, sorted by name.
1590    ///
1591    /// The census companion to [`Self::channel_status`]. A by-name probe answers
1592    /// about a name the caller already suspects; only an enumeration can reveal
1593    /// a name the caller does not know to ask about, and a verification sweep
1594    /// with no population denominator is an instrument shape this estate
1595    /// forbids. Touches no actor, under the same constraint as the probe.
1596    ///
1597    /// # Errors
1598    /// Returns [`ChannelRegistryError::RosterUnavailable`] only.
1599    pub fn registered_channels(&self) -> Result<Vec<ChannelDescriptor>, ChannelRegistryError> {
1600        let entries: Vec<(String, Arc<ConfiguredChannel>)> = {
1601            let channels = self.read_roster()?;
1602            channels
1603                .iter()
1604                .map(|(name, configured)| (name.clone(), Arc::clone(configured)))
1605                .collect()
1606        };
1607        let mut descriptors: Vec<ChannelDescriptor> = entries
1608            .into_iter()
1609            .map(|(name, configured)| ChannelDescriptor {
1610                name,
1611                origin: configured.origin,
1612                state: configured.state(),
1613            })
1614            .collect();
1615        descriptors.sort_by(|left, right| left.name.cmp(&right.name));
1616        Ok(descriptors)
1617    }
1618
1619    /// The roster admission funnel: the ONE place a channel operation's
1620    /// permission is decided.
1621    ///
1622    /// Returns the admitted entry, so the caller works from the value the
1623    /// decision was made on rather than reading the roster a second time and
1624    /// risking a different answer. This read is the LINEARISATION POINT for the
1625    /// quiesce race: everything the caller does with the returned entry happens
1626    /// outside the lock and is not re-checked, which is exactly why a quiesce
1627    /// that commits after this returns does not stop the operation it admitted.
1628    ///
1629    /// # Errors
1630    /// Returns [`ChannelAccessError`] when the channel is absent, quiesced, or
1631    /// the roster lock is poisoned.
1632    fn admit_channel(&self, channel: &str) -> Result<Arc<ConfiguredChannel>, ChannelAccessError> {
1633        let configured = {
1634            let channels = self.channels.read().map_err(|_poisoned| {
1635                ChannelAccessError::RosterUnavailable {
1636                    message: ROSTER_POISONED.to_owned(),
1637                }
1638            })?;
1639            channels.get(channel).map(Arc::clone)
1640        };
1641        let configured = configured.ok_or_else(|| ChannelAccessError::NotRegistered {
1642            name: channel.to_owned(),
1643        })?;
1644        if configured.state.load(Ordering::Acquire) == STATE_QUIESCED {
1645            return Err(ChannelAccessError::Quiesced {
1646                name: channel.to_owned(),
1647                reason: configured.recorded_quiesce_reason(),
1648            });
1649        }
1650        Ok(configured)
1651    }
1652
1653    /// Takes the roster's READ guard for the registration surface, mapping a
1654    /// poisoned lock to a typed [`ChannelRegistryError`].
1655    ///
1656    /// Separate from [`Self::read_channels`] because the two surfaces answer
1657    /// different callers with different error types; recovering one from the
1658    /// other would mean inspecting a message.
1659    fn read_roster(
1660        &self,
1661    ) -> Result<
1662        std::sync::RwLockReadGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>,
1663        ChannelRegistryError,
1664    > {
1665        #[cfg(test)]
1666        ROSTER_READS.with(|reads| reads.set(reads.get() + 1));
1667        self.channels
1668            .read()
1669            .map_err(|_poisoned| ChannelRegistryError::RosterUnavailable {
1670                message: ROSTER_POISONED.to_owned(),
1671            })
1672    }
1673
1674    /// Takes the roster's WRITE guard — held only for the check-and-insert that
1675    /// must be atomic, never across a call into the liminal library.
1676    fn write_roster(
1677        &self,
1678    ) -> Result<
1679        std::sync::RwLockWriteGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>,
1680        ChannelRegistryError,
1681    > {
1682        self.channels
1683            .write()
1684            .map_err(|_poisoned| ChannelRegistryError::RosterUnavailable {
1685                message: ROSTER_POISONED.to_owned(),
1686            })
1687    }
1688
1689    /// Gives back the tap slot a subscribe to `channel` reserved at
1690    /// admission, taking its tap channel off the roster when no tap stands
1691    /// behind it (SRV-012 R6).
1692    fn give_back_tap(&self, channel: &str) {
1693        self.console.release_tap(channel, |subject| {
1694            crate::console::ServerPublisher::retire_console_channel(self, subject)
1695        });
1696    }
1697
1698    /// The trait's subscribe, from the console guard to the subscription;
1699    /// [`ConnectionServices::subscribe`] gives back the tap reservation when
1700    /// this fails.
1701    fn open_subscription(
1702        &self,
1703        channel: &str,
1704        accepted_schemas: &[ProtocolSchemaId],
1705        install: Option<liminal::channel::InboxInstall>,
1706        consumer: Option<&str>,
1707    ) -> Result<ConnectionSubscription, ServerError> {
1708        // The frame path refused this already at admission; this guard is for
1709        // a caller holding the trait directly.
1710        refuse_console_tap_target(channel).map_err(|error| access_to_server_error(&error))?;
1711        // As in `publish`: the admission funnel clones the entry's `Arc` out
1712        // under the read guard and the guard is released before schema
1713        // negotiation or the actor round-trip below. That release is what makes
1714        // this the linearisation point — a quiesce landing after it does not
1715        // stop the subscription this call is already building.
1716        let configured = self
1717            .admit_channel(channel)
1718            .map_err(|error| access_to_server_error(&error))?;
1719        let selected_schema = if accepted_schemas.is_empty() {
1720            configured.protocol_schema
1721        } else {
1722            liminal::protocol::negotiate_schema(configured.protocol_schema, accepted_schemas)
1723                .map_err(|error| server_error_from_protocol(&error))?
1724        };
1725        self.build_subscription(
1726            channel,
1727            &configured.handle,
1728            selected_schema,
1729            install,
1730            consumer,
1731        )
1732    }
1733
1734    fn build_subscription(
1735        &self,
1736        channel: &str,
1737        handle: &ChannelHandle,
1738        selected_schema: ProtocolSchemaId,
1739        install: Option<liminal::channel::InboxInstall>,
1740        consumer: Option<&str>,
1741    ) -> Result<ConnectionSubscription, ServerError> {
1742        // `subscribe_with_install` installs the §5 budget/fairness cap and the R3
1743        // wake notifier on the inbox at construction — strictly before the
1744        // registration is published to the channel actor — so there is no window
1745        // in which a publish can land uncharged or without a wake.
1746        let subscription = match (consumer, install) {
1747            (Some(consumer), Some(install)) => handle.subscribe_consumer(consumer, install),
1748            (Some(_), None) => {
1749                return Err(ServerError::ListenerAccept {
1750                    message: "a durable consumer requires a bounded inbox install".to_owned(),
1751                });
1752            }
1753            (None, Some(install)) => handle.subscribe_with_install(install),
1754            (None, None) => handle.subscribe(),
1755        }
1756        .map_err(|error| ServerError::ListenerAccept {
1757            message: format!("liminal subscribe failed for channel '{channel}': {error}"),
1758        })?;
1759        let id = self.next_subscription_id.fetch_add(1, Ordering::Relaxed);
1760        let mut connection_subscription = ConnectionSubscription::new(
1761            id,
1762            selected_schema,
1763            Box::new(LiminalSubscriptionResource { subscription }),
1764        );
1765        connection_subscription.set_channel(channel.to_owned());
1766        // SRV-012 R6: the console tap attaches and detaches through the
1767        // supervisor's observer, which hears every subscribe and every
1768        // departure once (ServerObserver).
1769        Ok(connection_subscription)
1770    }
1771
1772    pub(crate) fn subscribe_consumer(
1773        &self,
1774        channel: &str,
1775        consumer: &str,
1776        install: liminal::channel::InboxInstall,
1777    ) -> Result<ConnectionSubscription, ServerError> {
1778        self.open_subscription(channel, &[], Some(install), Some(consumer))
1779    }
1780
1781    pub(crate) fn install_exporter(
1782        &self,
1783        exporter: &Arc<crate::link_bridge::subscriptions::ExportSubscriptions>,
1784    ) -> Result<(), ServerError> {
1785        let mut exporters =
1786            self.export_subscriptions
1787                .lock()
1788                .map_err(|_| ServerError::ListenerAccept {
1789                    message: "export subscription registry lock poisoned".to_owned(),
1790                })?;
1791        exporters.retain(|exporter| exporter.strong_count() > 0);
1792        if exporters
1793            .iter()
1794            .filter_map(std::sync::Weak::upgrade)
1795            .any(|existing| existing.link() == exporter.link())
1796        {
1797            return Err(ServerError::ListenerAccept {
1798                message: format!(
1799                    "link '{}' already owns an export subscription",
1800                    exporter.link()
1801                ),
1802            });
1803        }
1804        let mut prepared = Vec::new();
1805        for channel in self
1806            .registered_channels()
1807            .map_err(|error| ServerError::ListenerAccept {
1808                message: error.to_string(),
1809            })?
1810        {
1811            if matches!(channel.state, ChannelState::Active) {
1812                if let Some(binding) = exporter.binding(&channel.name) {
1813                    let subscription =
1814                        self.subscribe_consumer(&channel.name, &binding.consumer, binding.install)?;
1815                    prepared.push(
1816                        crate::link_bridge::subscriptions::PreparedSubscription::new(
1817                            binding.prefix,
1818                            channel.name,
1819                            subscription,
1820                        )?,
1821                    );
1822                }
1823            }
1824        }
1825        for subscription in prepared {
1826            exporter.attach(subscription);
1827        }
1828        exporters.push(Arc::downgrade(exporter));
1829        drop(exporters);
1830        Ok(())
1831    }
1832
1833    /// Refuses a subscribe that would start one more live tap than
1834    /// [`crate::console::MAX_LIVE_TAPS`] (SRV-012 R6); a subscriber to a tap
1835    /// already live is not a new tap.
1836    ///
1837    /// A poisoned tap table does not refuse the subscribe (R8: a console
1838    /// bookkeeping failure never fails the client operation it watches); the
1839    /// attach that follows meets the poison and counts it.
1840    ///
1841    /// # Errors
1842    /// Returns [`ChannelAccessError::ConsoleTapLimit`] naming the channel.
1843    fn refuse_console_tap_limit(&self, channel: &str) -> Result<(), ChannelAccessError> {
1844        match self.console.reserve_tap(channel) {
1845            Ok(false) => Err(ChannelAccessError::ConsoleTapLimit {
1846                name: channel.to_owned(),
1847                limit: crate::console::MAX_LIVE_TAPS,
1848            }),
1849            Ok(true) | Err(_) => Ok(()),
1850        }
1851    }
1852}
1853
1854impl crate::console::ServerPublisher for LiminalConnectionServices {
1855    /// The server's own record on the one publish path, with a server origin:
1856    /// attributed to `liminal.<device>`, and never counted as a connection's
1857    /// work by the cost tick.
1858    fn publish_console(
1859        &self,
1860        subject: &str,
1861        payload: &[u8],
1862    ) -> Result<(), crate::console::ConsoleError> {
1863        let payload = self.agent_topology_payload(subject, payload)?;
1864        let payload = if subject == crate::console::TOPOLOGY_SUBJECT {
1865            if let Some(lifecycle) = self.site_lifecycle.get() {
1866                std::borrow::Cow::Owned(
1867                    crate::console::topology::with_sites(
1868                        &payload,
1869                        lifecycle,
1870                        self.console.device(),
1871                    )
1872                    .map_err(|error| crate::console::ConsoleError::Server {
1873                        subject: subject.into(),
1874                        source: site_services::site_error(error),
1875                    })?,
1876                )
1877            } else {
1878                payload
1879            }
1880        } else {
1881            payload
1882        };
1883        let envelope = MessageEnvelope::new(
1884            ProtocolSchemaId::new([0_u8; ProtocolSchemaId::WIRE_LEN]),
1885            liminal::protocol::CausalContext::independent(),
1886            payload.to_vec(),
1887        );
1888        self.publish_with_origin(
1889            subject,
1890            &envelope,
1891            None,
1892            PublishOrigin::Server {
1893                device: self.console.device(),
1894            },
1895        )
1896        .map(|_published| ())
1897        .map_err(|source| crate::console::ConsoleError::Server {
1898            subject: subject.to_owned(),
1899            source,
1900        })
1901    }
1902
1903    /// Takes a `console.tap.<target>` channel off the roster (SRV-012 R6).
1904    /// Only a console subject is ever removed here.
1905    fn retire_console_channel(&self, subject: &str) -> Result<(), crate::console::ConsoleError> {
1906        if !super::channel_registry::is_console_subject(subject) {
1907            return Ok(());
1908        }
1909        let mut channels =
1910            self.write_roster()
1911                .map_err(|error| crate::console::ConsoleError::Server {
1912                    subject: subject.to_owned(),
1913                    source: ServerError::ListenerAccept {
1914                        message: error.to_string(),
1915                    },
1916                })?;
1917        channels.remove(subject);
1918        drop(channels);
1919        Ok(())
1920    }
1921}
1922
1923/// The production participant service and the operator surfaces taken from
1924/// its one live handler before it is erased behind
1925/// `dyn ParticipantSemanticHandler`.
1926struct InstalledParticipant {
1927    service: InstalledParticipantService,
1928    unloadable_conversations: UnloadableConversationRecord,
1929    credential_reissuer: Arc<dyn OperatorCredentialReissuer>,
1930    warm: ParticipantWarm,
1931}
1932
1933/// Production participant activation (LP gap closure, Part B): the
1934/// deployment's [participant] section installs the ONE production semantic
1935/// handler, sealed together with the same durable store the conversation logs
1936/// live in, under the configured wire-frame limit.
1937///
1938/// # Errors
1939/// Returns [`ServerError::ParticipantStartupRestore`] when the handler cannot
1940/// be built or wired, and [`ServerError::ConfigValidation`] when the
1941/// wire-frame limit is below the protocol's minimum participant frame.
1942fn install_participant(
1943    durable_store: &Arc<dyn DurableStore>,
1944    participant: &ParticipantConfig,
1945    signals: &ServerSignals,
1946) -> Result<InstalledParticipant, ServerError> {
1947    let handler = ProductionParticipantHandler::new(Arc::clone(durable_store), *participant)
1948        .map_err(|error| ServerError::ParticipantStartupRestore {
1949            message: error.to_string(),
1950        })?;
1951    // Taken BEFORE the handler is erased behind
1952    // `dyn ParticipantSemanticHandler`. Construction restores no
1953    // conversation (DUR-020), so the record fills as conversations restore,
1954    // on first touch or in the warm, and is complete once the warm has
1955    // finished.
1956    handler.install_signals(signals.clone()).map_err(|error| {
1957        ServerError::ParticipantStartupRestore {
1958            message: error.to_string(),
1959        }
1960    })?;
1961    let unloadable_conversations = handler.unloadable_record();
1962    let handler = Arc::new(handler);
1963    let warm = handler
1964        .install_warm()
1965        .map_err(|error| ServerError::ParticipantStartupRestore {
1966            message: error.to_string(),
1967        })?;
1968    // A7 (§0.18): taken here for the same reason, and from the same one live
1969    // handler, so the operator surface and the participant wire cannot end up
1970    // talking to two different owners.
1971    let credential_reissuer: Arc<dyn OperatorCredentialReissuer> =
1972        Arc::<ProductionParticipantHandler>::clone(&handler);
1973    let service = InstalledParticipantService::new(
1974        handler,
1975        Arc::clone(durable_store),
1976        participant.wire_frame_limit,
1977    )
1978    .map_err(|error| ServerError::ConfigValidation {
1979        message: format!(
1980            "participant.wire_frame_limit: {} is below the protocol's minimum \
1981             complete participant frame ({error:?})",
1982            participant.wire_frame_limit
1983        ),
1984    })?;
1985    Ok(InstalledParticipant {
1986        service,
1987        unloadable_conversations,
1988        credential_reissuer,
1989        warm,
1990    })
1991}
1992
1993/// The console hub for `device`, its model built from the roster as it
1994/// stands (responders, connections and peers arrive as changes).
1995///
1996/// # Errors
1997/// Returns [`ServerError::ListenerAccept`] when the device is blank or the
1998/// wall clock cannot be read.
1999fn console_hub(
2000    device: &str,
2001    channels: &HashMap<String, Arc<ConfiguredChannel>>,
2002    depot: bool,
2003    lys: Option<&crate::config::types::LysConfig>,
2004) -> Result<crate::console::ConsoleHub, ServerError> {
2005    let this = crate::console::ThisDevice {
2006        id: device.to_owned(),
2007        liminal: env!("CARGO_PKG_VERSION").to_owned(),
2008        started: Instant::now(),
2009        depot,
2010    };
2011    let lines = channels.iter().map(|(name, configured)| {
2012        let state = match configured.state() {
2013            ChannelState::Active => crate::console::records::LineState::Live,
2014            ChannelState::Quiesced { .. } => crate::console::records::LineState::Quiesced,
2015        };
2016        (name.clone(), line_life(configured.durable), state)
2017    });
2018    let mut model = crate::console::TopologyModel::new(this, [], lines, [], []);
2019    if let Some(machine) = lys.and_then(|lys| lys.machine.as_ref()) {
2020        model = model.with_machine(machine);
2021    }
2022    crate::console::wall::WallBase::now()
2023        .and_then(|wall| crate::console::ConsoleHub::new(device, model, wall))
2024        .map_err(|error| ServerError::ListenerAccept {
2025            message: format!("console hub: {error}"),
2026        })
2027}
2028
2029/// A roster entry's status as [`LiminalConnectionServices::channel_status`]
2030/// answers it.
2031fn status_of(configured: &ConfiguredChannel) -> ChannelStatus {
2032    let mode = configured.handle.config().mode;
2033    match configured.state() {
2034        ChannelState::Active => ChannelStatus::Active {
2035            origin: configured.origin,
2036            mode,
2037            schema: configured.protocol_schema,
2038        },
2039        ChannelState::Quiesced { reason } => ChannelStatus::Quiesced {
2040            reason,
2041            origin: configured.origin,
2042            mode,
2043        },
2044    }
2045}
2046
2047#[cfg(test)]
2048thread_local! {
2049    /// Roster read locks taken on this thread. Test-only.
2050    static ROSTER_READS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
2051}
2052
2053/// Roster read locks taken on the calling thread so far. Test-only.
2054#[cfg(test)]
2055pub(crate) fn roster_reads() -> u64 {
2056    ROSTER_READS.with(std::cell::Cell::get)
2057}
2058
2059/// A channel's line life: durable channels keep, ephemeral ones pass.
2060const fn line_life(durable: bool) -> crate::console::records::LineLife {
2061    if durable {
2062        crate::console::records::LineLife::Kept
2063    } else {
2064        crate::console::records::LineLife::Passing
2065    }
2066}
2067
2068/// The diagnostic carried when the roster lock is poisoned. One string, shared
2069/// by every surface that reports it, so the wording cannot drift between them.
2070const ROSTER_POISONED: &str = "channel roster lock poisoned";
2071
2072/// How many roster entries the registration cap counts.
2073///
2074/// Runtime-registered entries ONLY. Boot-configured channels are the operator's
2075/// own authored bound — they are in the file the operator wrote — and counting
2076/// them would make one number mean two different things depending on how the
2077/// deployment was configured. The origin never flips, so this population is
2078/// well defined over time.
2079fn runtime_registered_count(channels: &HashMap<String, Arc<ConfiguredChannel>>) -> usize {
2080    channels
2081        .values()
2082        .filter(|configured| configured.origin == ChannelOrigin::RuntimeRegistered)
2083        .count()
2084}
2085
2086/// The three-field identity comparison behind idempotent-if-identical
2087/// registration.
2088///
2089/// All three must match; the FIRST mismatch is reported by name. The name itself
2090/// is not compared — it is the roster key, a precondition of the comparison
2091/// rather than a member of it — and neither are the supervisor or the durable
2092/// store, which are one server-wide instance each and cannot differ between two
2093/// registrations in one process.
2094///
2095/// The schema is compared as BOTH its protocol id and its parsed document, on
2096/// purpose. The id is a 64-bit non-cryptographic digest, so the document guards
2097/// against a collision accepting a different schema as identical; and the two
2098/// fail in opposite directions, because two byte sequences that parse to the
2099/// same document but differ in whitespace produce different ids — which must
2100/// refuse, since the id is what every future subscriber negotiates.
2101///
2102/// The channel's own `Schema` is not compared: it is not `PartialEq`, and its
2103/// identifier is a fresh value per construction, so comparing it would refuse
2104/// every idempotent re-registration.
2105fn compare_registration(
2106    existing: &ConfiguredChannel,
2107    spec: &ChannelRegistration,
2108    resolved: &ChannelSchema,
2109) -> Result<Registered, ChannelRegistryError> {
2110    let requested_mode = if spec.durable {
2111        ChannelMode::Durable
2112    } else {
2113        ChannelMode::Ephemeral
2114    };
2115    let config = existing.handle.config();
2116    let mismatch = if config.mode == requested_mode {
2117        if existing.protocol_schema == resolved.protocol_id {
2118            if *config.schema.definition() == resolved.document {
2119                None
2120            } else {
2121                Some(ChannelConfigField::SchemaDocument)
2122            }
2123        } else {
2124            Some(ChannelConfigField::SchemaId)
2125        }
2126    } else {
2127        Some(ChannelConfigField::Mode)
2128    };
2129    mismatch.map_or(Ok(Registered::AlreadyIdentical), |field| {
2130        Err(ChannelRegistryError::AlreadyRegistered {
2131            name: spec.name.clone(),
2132            field,
2133        })
2134    })
2135}
2136
2137/// Recovers the resolved schema of an already-built entry, so the write-lock
2138/// re-check compares the SAME three fields against the same values the fast path
2139/// would have.
2140fn schema_of(configured: &ConfiguredChannel) -> ChannelSchema {
2141    ChannelSchema {
2142        document: configured.handle.config().schema.definition().clone(),
2143        protocol_id: configured.protocol_schema,
2144    }
2145}
2146
2147/// Builds one roster entry: the channel's validation engine, its library handle
2148/// (durable or ephemeral), and the protocol schema id advertised at subscribe
2149/// time.
2150///
2151/// This is the SOLE place a [`ConfiguredChannel`] is constructed. Keeping
2152/// construction in one function is what makes "every channel on this server is
2153/// the same kind of object" structural rather than a promise: there is no second
2154/// body a channel could be built by, so no channel can drift into a different
2155/// shape. The boot loop in [`LiminalConnectionServices::from_config_with_store_via`]
2156/// is its caller.
2157///
2158/// `resolved` is consumed because its JSON Schema document is moved into the
2159/// channel's [`Schema`]; the protocol id is carried onto the entry unchanged.
2160/// `origin` is stamped here and never written again.
2161///
2162/// The failure is typed ([`ChannelBuildError`]) rather than a [`ServerError`] so
2163/// BOTH callers can render it without inspecting a message: the boot loop maps
2164/// it back to the exact `ConfigValidation` strings it has always produced, and
2165/// `register_channel` maps it to the registry's own typed variants.
2166///
2167/// # Errors
2168/// Returns [`ChannelBuildError`] when the JSON Schema document does not compile
2169/// or durable initialization over the shared store fails.
2170fn build_configured_channel(
2171    name: &str,
2172    resolved: ChannelSchema,
2173    durable: bool,
2174    origin: ChannelOrigin,
2175    durable_store: &Arc<dyn DurableStore>,
2176    supervisor: &ChannelSupervisor,
2177) -> Result<ConfiguredChannel, ChannelBuildError> {
2178    let schema =
2179        Schema::new(resolved.document).map_err(|error| ChannelBuildError::SchemaRejected {
2180            message: error.to_string(),
2181        })?;
2182    let channel_config = if durable {
2183        ChannelConfig::new(name.to_owned(), schema, ChannelMode::Durable)
2184    } else {
2185        ChannelConfig::new(name.to_owned(), schema, ChannelMode::Ephemeral)
2186    };
2187    let handle = if durable {
2188        ChannelHandle::new_durable_with_supervisor(
2189            channel_config,
2190            Arc::clone(durable_store),
2191            supervisor.clone(),
2192        )
2193        .map_err(|error| ChannelBuildError::DurableInitFailed {
2194            message: error.to_string(),
2195        })?
2196    } else {
2197        ChannelHandle::with_supervisor(channel_config, supervisor.clone())
2198    };
2199    Ok(ConfiguredChannel {
2200        handle,
2201        protocol_schema: resolved.protocol_id,
2202        durable,
2203        origin,
2204        state: AtomicU8::new(STATE_ACTIVE),
2205        quiesce_reason: OnceLock::new(),
2206    })
2207}
2208
2209/// Returns the current epoch-millis timestamp used as the dedup entry anchor.
2210///
2211/// A clock error before the Unix epoch yields `0`: the timestamp is only a TTL
2212/// anchor for the in-memory cache and a zero anchor never breaks the at-most-once
2213/// claim semantics, so this avoids surfacing a clock fault on the publish path.
2214fn dedup_timestamp_millis() -> u64 {
2215    use std::time::{SystemTime, UNIX_EPOCH};
2216    SystemTime::now()
2217        .duration_since(UNIX_EPOCH)
2218        .ok()
2219        .and_then(|duration| u64::try_from(duration.as_millis()).ok())
2220        .unwrap_or(0)
2221}
2222
2223/// Default shard count for an on-disk durable store.
2224///
2225/// Haematite routes keys across this many single-threaded shard actors; a small
2226/// power of two gives parallelism across cursors/streams without spawning an
2227/// actor per core. The value is fixed (haematite has no silent default) and not
2228/// yet surfaced in server config.
2229const DEFAULT_SHARD_COUNT: usize = 8;
2230
2231/// Shard count of the throwaway store an incarnation's ephemeral dedup
2232/// namespace lives in: it holds one namespace, and dies with the process.
2233const THROWAWAY_SHARD_COUNT: usize = 1;
2234
2235/// Namespace prefix for the dedup-on-delivery cache streams. Keeps delivery dedup
2236/// keys from colliding with any other haematite streams in the shared store.
2237///
2238/// This is the **durable** namespace (A1 §5): stable across restarts, because a
2239/// durable channel's message outlives a restart too.
2240const DELIVERY_DEDUP_NAMESPACE: &str = "liminal:delivery-dedup";
2241
2242/// The **ephemeral** dedup namespace for this server incarnation (A1 §5,
2243/// graft §0.2).
2244///
2245/// Scoped to the process incarnation so an ephemeral channel's receipt shares a
2246/// lifetime with the in-memory message it vouches for. Composed from the
2247/// process id and the boot instant, which is enough to make two incarnations
2248/// disagree; nothing reads the components back, so no meaning is attached to
2249/// them beyond distinctness.
2250///
2251/// The namespace lives in [`ephemeral_dedup_store`], never in a store that
2252/// outlives the process, so no incarnation's records are left behind for a
2253/// later one to read past.
2254fn ephemeral_dedup_namespace() -> String {
2255    use std::time::{SystemTime, UNIX_EPOCH};
2256    let boot_nanos = SystemTime::now()
2257        .duration_since(UNIX_EPOCH)
2258        .map_or(0, |since| since.as_nanos());
2259    format!(
2260        "{DELIVERY_DEDUP_NAMESPACE}:incarnation:{}-{boot_nanos}",
2261        std::process::id()
2262    )
2263}
2264
2265/// One reader over the counters of `durable_store` and, when it is a separate
2266/// store, `ephemeral_store`; a store that is the same `Arc` is read once.
2267fn combined_store_counts(
2268    durable_store: &Arc<dyn DurableStore>,
2269    ephemeral_store: &Arc<dyn DurableStore>,
2270) -> StoreCountsHandle {
2271    let durable = durable_store.store_counts().unwrap_or_default();
2272    if Arc::ptr_eq(durable_store, ephemeral_store) {
2273        return durable;
2274    }
2275    durable.with(&ephemeral_store.store_counts().unwrap_or_default())
2276}
2277
2278/// The store an incarnation's ephemeral dedup namespace is written to: the
2279/// services' own store when it was created empty for this process and dies
2280/// with it, otherwise a store of its own that does, opened by the namespace's
2281/// first write rather than at start.
2282///
2283/// Written into a persistent store, every incarnation's namespace would stay
2284/// there for good, under the durable namespace's prefix, with nothing that
2285/// ever reads it again to prune it.
2286fn ephemeral_dedup_store(
2287    durable_store: &Arc<dyn DurableStore>,
2288    history: StoreHistory,
2289    subsystems: &dyn SubsystemFactory,
2290) -> Arc<dyn DurableStore> {
2291    match (history, durable_store) {
2292        #[cfg(any(test, feature = "test-support"))]
2293        (StoreHistory::Fresh, store) => Arc::clone(store),
2294        (StoreHistory::Created | StoreHistory::Recorded, _) => Arc::new(CountingStore::new(
2295            DeferredStore::new(subsystems.throwaway_store()),
2296        )),
2297    }
2298}
2299
2300/// Beamr-scheduler-owning subsystems the full-service construction path builds
2301/// beyond the connection supervisor's own scheduler. The §9 D2 seam census counts
2302/// these: the worker-front-door profile must construct NONE of them. Test-gated:
2303/// this is the recording vocabulary of the gate's instrument, not production state.
2304#[cfg(test)]
2305#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
2306pub(super) enum SchedulerSubsystem {
2307    /// The shared channel supervisor (its own beamr scheduler).
2308    ChannelSupervisor,
2309    /// The conversation supervisor (its own beamr scheduler).
2310    ConversationSupervisor,
2311    /// The haematite store's database (its shard-actor scheduler).
2312    HaematiteStore,
2313}
2314
2315/// Constructor seam for every scheduler-owning subsystem (`SchedulerSubsystem`)
2316/// the profile-aware construction path can create.
2317///
2318/// These methods are the ONLY route through which [`build_connection_services`]
2319/// and the [`LiminalConnectionServices`] config constructors reach
2320/// `build_channel_cluster`, `ConversationSupervisor::new`, and the durable-store
2321/// constructors — no direct constructor call exists in those bodies. The §9 D2
2322/// gate therefore injects a factory that records as a side effect of
2323/// constructing (the D3 store-seam ownership move applied to schedulers): a
2324/// recording cannot be omitted without also failing to construct the subsystem,
2325/// which closes the "hand-placed census call beside the constructor" gap where a
2326/// future subsystem could be built without its courtesy call.
2327pub(super) trait SubsystemFactory {
2328    /// Constructs the shared channel supervisor + cluster resolver (a beamr
2329    /// scheduler).
2330    ///
2331    /// # Errors
2332    /// Returns [`ServerError`] when the channel supervisor scheduler cannot start.
2333    fn channel_cluster(
2334        &self,
2335        cluster_config: Option<&ClusterConfig>,
2336    ) -> Result<ChannelCluster, ServerError>;
2337
2338    /// Constructs the conversation supervisor (a beamr scheduler).
2339    ///
2340    /// # Errors
2341    /// Returns [`ServerError`] when the conversation supervisor scheduler cannot
2342    /// start.
2343    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError>;
2344
2345    /// Constructs the persistent durable store under the explicit directory.
2346    ///
2347    /// # Errors
2348    /// Returns [`ServerError`] when the store cannot be opened.
2349    fn durable_store(&self, persistence_path: &Path) -> Result<Arc<dyn DurableStore>, ServerError>;
2350
2351    /// The opener of the throwaway store an incarnation's ephemeral dedup
2352    /// namespace lives in when the durable store outlives the process. The
2353    /// store is opened by the namespace's first write, never by this call.
2354    fn throwaway_store(&self) -> ThrowawayStoreOpener;
2355
2356    fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
2357        None
2358    }
2359}
2360
2361/// Test construction factory for callers that own their fixture directories.
2362#[cfg(any(test, feature = "test-support"))]
2363pub(super) struct ProductionSubsystems;
2364
2365#[cfg(any(test, feature = "test-support"))]
2366impl SubsystemFactory for ProductionSubsystems {
2367    fn channel_cluster(
2368        &self,
2369        cluster_config: Option<&ClusterConfig>,
2370    ) -> Result<ChannelCluster, ServerError> {
2371        build_channel_cluster(cluster_config)
2372    }
2373
2374    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2375        Ok(Arc::new(ConversationSupervisor::new().map_err(
2376            |error| ServerError::ConfigValidation {
2377                message: format!("failed to start conversation supervisor: {error}"),
2378            },
2379        )?))
2380    }
2381
2382    fn durable_store(&self, persistence_path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2383        build_durable_store(persistence_path)
2384    }
2385
2386    fn throwaway_store(&self) -> ThrowawayStoreOpener {
2387        Box::new(|| open_ephemeral(THROWAWAY_SHARD_COUNT))
2388    }
2389}
2390
2391struct RootedProductionSubsystems {
2392    root: Arc<EphemeralRoot>,
2393}
2394
2395impl RootedProductionSubsystems {
2396    fn new(path: &Path) -> Result<Self, ServerError> {
2397        let root = EphemeralRoot::open(path).map_err(|error| ServerError::ConfigValidation {
2398            message: error.to_string(),
2399        })?;
2400        Ok(Self { root })
2401    }
2402}
2403
2404impl SubsystemFactory for RootedProductionSubsystems {
2405    fn channel_cluster(
2406        &self,
2407        config: Option<&ClusterConfig>,
2408    ) -> Result<ChannelCluster, ServerError> {
2409        build_channel_cluster(config)
2410    }
2411
2412    fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2413        Ok(Arc::new(ConversationSupervisor::new().map_err(
2414            |error| ServerError::ConfigValidation {
2415                message: format!("failed to start conversation supervisor: {error}"),
2416            },
2417        )?))
2418    }
2419
2420    fn durable_store(&self, path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2421        build_durable_store(path)
2422    }
2423
2424    fn throwaway_store(&self) -> ThrowawayStoreOpener {
2425        let root = Arc::clone(&self.root);
2426        Box::new(move || root.open_store(THROWAWAY_SHARD_COUNT))
2427    }
2428
2429    fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
2430        Some(self.root.sweep_counts())
2431    }
2432}
2433
2434/// Test-only recording implementation of [`SubsystemFactory`] for the §9 D2
2435/// construction gate. Lives here (not in a test module's private scope) so the
2436/// supervisor-level gate test reuses the same instrument.
2437#[cfg(test)]
2438#[allow(clippy::expect_used)]
2439pub(super) mod subsystem_census {
2440    use std::path::{Path, PathBuf};
2441    use std::sync::{Arc, Mutex};
2442
2443    use liminal::conversation::ConversationSupervisor;
2444    use liminal::durability::{DurableStore, open_ephemeral_rooted};
2445
2446    use super::{
2447        ChannelCluster, ProductionSubsystems, SchedulerSubsystem, SubsystemFactory,
2448        THROWAWAY_SHARD_COUNT, ThrowawayStoreOpener, build_durable_store,
2449    };
2450    use crate::ServerError;
2451    use crate::config::types::ClusterConfig;
2452
2453    /// Records each [`SchedulerSubsystem`] AS A SIDE EFFECT of constructing it,
2454    /// then hands back the production-constructed subsystem (with ephemeral
2455    /// stores rooted in an isolated directory, so the fs half of the gate is a
2456    /// real negative assertion). Because [`SubsystemFactory`] is the only route
2457    /// the profile-aware construction path has to these constructors, a
2458    /// recording cannot be omitted without also failing to construct — the
2459    /// record-by-construction guarantee.
2460    pub struct RecordingSubsystems {
2461        census: Mutex<Vec<SchedulerSubsystem>>,
2462        ephemeral_root: PathBuf,
2463    }
2464
2465    impl RecordingSubsystems {
2466        /// A recording factory whose ephemeral stores live under `ephemeral_root`.
2467        pub fn rooted(ephemeral_root: &Path) -> Self {
2468            Self {
2469                census: Mutex::new(Vec::new()),
2470                ephemeral_root: ephemeral_root.to_path_buf(),
2471            }
2472        }
2473
2474        /// The recorded construction census, sorted for order-independent
2475        /// comparison.
2476        pub fn recorded(&self) -> Vec<SchedulerSubsystem> {
2477            let mut recorded = self
2478                .census
2479                .lock()
2480                .expect("subsystem census lock is never poisoned in tests")
2481                .clone();
2482            recorded.sort();
2483            recorded
2484        }
2485
2486        fn record(&self, subsystem: SchedulerSubsystem) {
2487            self.census
2488                .lock()
2489                .expect("subsystem census lock is never poisoned in tests")
2490                .push(subsystem);
2491        }
2492    }
2493
2494    impl SubsystemFactory for RecordingSubsystems {
2495        fn channel_cluster(
2496            &self,
2497            cluster_config: Option<&ClusterConfig>,
2498        ) -> Result<ChannelCluster, ServerError> {
2499            let cluster = ProductionSubsystems.channel_cluster(cluster_config)?;
2500            self.record(SchedulerSubsystem::ChannelSupervisor);
2501            Ok(cluster)
2502        }
2503
2504        fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2505            let supervisor = ProductionSubsystems.conversation_supervisor()?;
2506            self.record(SchedulerSubsystem::ConversationSupervisor);
2507            Ok(supervisor)
2508        }
2509
2510        fn durable_store(
2511            &self,
2512            persistence_path: &Path,
2513        ) -> Result<Arc<dyn DurableStore>, ServerError> {
2514            let store = build_durable_store(persistence_path)?;
2515            self.record(SchedulerSubsystem::HaematiteStore);
2516            Ok(store)
2517        }
2518
2519        fn throwaway_store(&self) -> ThrowawayStoreOpener {
2520            let root = self.ephemeral_root.clone();
2521            Box::new(move || open_ephemeral_rooted(&root, THROWAWAY_SHARD_COUNT))
2522        }
2523    }
2524}
2525
2526/// Full-only constructor guard: rejects a config whose profile is not `Full`.
2527///
2528/// [`LiminalConnectionServices`]' config-based constructors call this at entry so
2529/// the full service stack can never be built for a worker-front-door config —
2530/// profile enforcement holds on every public construction path, not only the
2531/// file-loading pipeline.
2532fn require_full_profile(config: &ServerConfig) -> Result<(), ServerError> {
2533    match config.services.profile()? {
2534        ServiceProfile::Full => Ok(()),
2535        ServiceProfile::WorkerFrontDoor => Err(ServerError::ConfigValidation {
2536            message: format!(
2537                "services.profile: \"{}\" cannot construct the full LiminalConnectionServices; \
2538                 build profile-selected services via build_connection_services",
2539                ServiceProfile::WORKER_FRONT_DOOR
2540            ),
2541        }),
2542    }
2543}
2544
2545/// Builds the connection-services adapter selected by `config`'s service profile.
2546///
2547/// `Full` builds [`LiminalConnectionServices`] (channels, conversations, durable
2548/// store, dedup cache) exactly as today. `WorkerFrontDoor` builds
2549/// [`WorkerFrontDoorServices`], which constructs none of that machinery. This is the
2550/// single profile-dispatch authority: [`super::supervisor::ConnectionSupervisor`]'s
2551/// config constructor and the standalone runtime's worker arm both route through it
2552/// (the runtime's full arm stays on the explicit
2553/// [`LiminalConnectionServices::from_config`] path because it also needs the shared
2554/// channel cluster, which the trait object does not expose — that path is guarded
2555/// full-only at entry).
2556///
2557/// # Errors
2558/// Returns [`ServerError`] when the selected adapter cannot be constructed, the
2559/// configured profile value is not recognised, or the worker-front-door profile is
2560/// combined with full-only config fields.
2561pub fn build_connection_services(
2562    config: &ServerConfig,
2563) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2564    match config.services.profile()? {
2565        ServiceProfile::Full => {
2566            let subsystems = RootedProductionSubsystems::new(required_persistence_path(config)?)?;
2567            build_connection_services_via(config, &subsystems)
2568        }
2569        ServiceProfile::WorkerFrontDoor => build_worker_front_door(config),
2570    }
2571}
2572
2573/// [`build_connection_services`] with the subsystem factory injected — the §9 D2
2574/// gate seam, both halves at once.
2575///
2576/// Every scheduler-owning subsystem the `Full` branch creates is constructed
2577/// through `subsystems` and nowhere else, so a recording factory observes exactly
2578/// what was built (thread half, record-by-construction); the gate's factory also
2579/// roots its ephemeral stores in an isolated directory, so "the root stays empty
2580/// on the worker branch" is a real negative assertion (fs half, the D3 pattern).
2581/// The `WorkerFrontDoor` branch never touches the factory — a regression that gave
2582/// the front door any subsystem would both record in the census and land a store
2583/// directory in the injected root.
2584///
2585/// The worker branch re-runs the cross-field checks here, not only in file-loading
2586/// validation, so a directly-constructed config cannot smuggle full-only machinery
2587/// past the profile.
2588pub(super) fn build_connection_services_via(
2589    config: &ServerConfig,
2590    subsystems: &dyn SubsystemFactory,
2591) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2592    match config.services.profile()? {
2593        ServiceProfile::Full => {
2594            let path = required_persistence_path(config)?;
2595            let history = StoreHistory::of(path);
2596            let store = subsystems.durable_store(path)?;
2597            Ok(Arc::new(
2598                LiminalConnectionServices::from_config_with_store_via(
2599                    config, store, history, subsystems,
2600                )?,
2601            ))
2602        }
2603        ServiceProfile::WorkerFrontDoor => build_worker_front_door(config),
2604    }
2605}
2606
2607fn build_worker_front_door(
2608    config: &ServerConfig,
2609) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2610    let errors = crate::config::validation::worker_front_door_field_errors(config);
2611    if !errors.is_empty() {
2612        return Err(ServerError::ConfigValidation {
2613            message: errors.join("; "),
2614        });
2615    }
2616    Ok(Arc::new(WorkerFrontDoorServices::new()))
2617}
2618
2619/// Opens the persistent durable store under the configured directory.
2620pub(crate) fn open_persistent_store(
2621    persistence_path: &Path,
2622) -> Result<Arc<dyn DurableStore>, ServerError> {
2623    build_durable_store(persistence_path)
2624}
2625
2626/// Every append is permanent through its WAL frames and one device flush
2627/// before it returns under either commit policy; what the policy decides is
2628/// when the shard folds its staged appends into a new tree version, the fold
2629/// that writes the leaf and every internal node above it as fresh files. Per
2630/// append, one keyed publish left 8 new node files (82,137 bytes) on the J3
2631/// store (NODE-FILES-PER-POST, check 15 on Dean at 68a08b35, 10 October
2632/// 2026), and the live spine reached 2,856,796 files that way. Buffered at
2633/// 1 MiB, Waffles' number (10 October 2026 11:19): the threshold haematite's
2634/// own `LeafLog` runs on, and the most an unclean restart replays from staged
2635/// frames per shard. The store's own reads merge the buffer, so a publish
2636/// reads back at once; a clean stop commits what is staged; no policy carries
2637/// a timer.
2638const SPINE_COMMIT_THRESHOLD_BYTES: NonZeroU64 = match NonZeroU64::new(1 << 20) {
2639    Some(threshold) => threshold,
2640    None => unreachable!(),
2641};
2642
2643/// The commit policy the spine's event store opens under.
2644const SPINE_COMMIT_POLICY: CommitPolicy = CommitPolicy::buffered(SPINE_COMMIT_THRESHOLD_BYTES);
2645
2646fn build_durable_store(path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2647    let data_dir = durable_data_dir(path);
2648    let database = open_or_create_database(&data_dir)?;
2649    let event_store = EventStore::with_commit_policy(database, SPINE_COMMIT_POLICY);
2650    Ok(Arc::new(CountingStore::new(HaematiteStore::new(Arc::new(
2651        event_store,
2652    )))))
2653}
2654
2655/// The directory the persistent database under `persistence_path` lives in.
2656pub(super) fn durable_data_dir(persistence_path: &Path) -> PathBuf {
2657    persistence_path.join("durability")
2658}
2659
2660/// Whether a haematite database already exists at `data_dir`: its
2661/// `config.json` is written when it is created.
2662pub(super) fn database_exists(data_dir: &Path) -> bool {
2663    database_config_file(data_dir).exists()
2664}
2665
2666/// What creating a haematite database leaves in its directory. A shard's
2667/// directory appears beside them the first time anything opens the shard, so
2668/// a database holding nothing else has never been read or written.
2669const UNTOUCHED_DATABASE_ENTRIES: [&str; 2] = ["config.json", "writer.lock"];
2670
2671/// Whether the database at `data_dir` holds only what its creation wrote.
2672/// Anything this cannot read, or does not recognise, answers `false`: the
2673/// database is then taken to hold what an earlier server wrote.
2674pub(super) fn database_untouched(data_dir: &Path) -> bool {
2675    std::fs::read_dir(data_dir).is_ok_and(|entries| {
2676        entries.into_iter().all(|entry| {
2677            entry.is_ok_and(|entry| {
2678                entry
2679                    .file_name()
2680                    .to_str()
2681                    .is_some_and(|name| UNTOUCHED_DATABASE_ENTRIES.contains(&name))
2682            })
2683        })
2684    })
2685}
2686
2687fn database_config_file(data_dir: &Path) -> PathBuf {
2688    data_dir.join("config.json")
2689}
2690
2691/// Every commit writes new copies of the nodes it changes; the durable store
2692/// deletes the copies its committed root no longer reaches while it runs, so it
2693/// holds its live records and not every version ever written. A store created
2694/// under `KeepAll` is moved onto this retention by its next open.
2695pub(super) const DURABLE_RETENTION: Retention = Retention::reclaim(0);
2696
2697/// Opens an existing haematite database at `data_dir`, or creates one.
2698fn open_or_create_database(data_dir: &Path) -> Result<Database, ServerError> {
2699    let config_file = database_config_file(data_dir);
2700    let result = if config_file.exists() {
2701        adopt_missing_config_keys(data_dir, &config_file)?;
2702        Database::open_with_retention(
2703            data_dir,
2704            haematite::RuntimeOptions {
2705                long_wait_interval: std::time::Duration::from_secs(5),
2706                // DUR-021 R1: no pass starts and no step runs until the
2707                // listeners are bound and runtime.rs releases it.
2708                reclaim_start: haematite::ReclaimStart::Held,
2709            },
2710            DURABLE_RETENTION,
2711        )
2712    } else {
2713        Database::create(
2714            DatabaseConfig {
2715                data_dir: data_dir.to_path_buf(),
2716                shard_count: DEFAULT_SHARD_COUNT,
2717                distributed: None,
2718                executor_threads: None,
2719                // haematite 0.8.3 requires this field and refuses `None` at
2720                // validation; `Unlimited` is the crate's explicit spelling of the
2721                // pre-budget (0.8.1) behaviour — the behaviour-preserving choice
2722                // for a dependency hop. A real byte ceiling is a deployment
2723                // decision for its own measured lane. A store whose `config.json`
2724                // predates the field is migrated by the open arm above
2725                // (`adopt_missing_config_keys`) — same value, written once, on
2726                // upgrade.
2727                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
2728                retention: DURABLE_RETENTION,
2729            },
2730            haematite::RuntimeOptions {
2731                long_wait_interval: std::time::Duration::from_secs(5),
2732                // DUR-021 R1: no pass starts and no step runs until the
2733                // listeners are bound and runtime.rs releases it.
2734                reclaim_start: haematite::ReclaimStart::Held,
2735            },
2736        )
2737    };
2738    let database = result.map_err(|error| ServerError::ConfigValidation {
2739        message: format!(
2740            "failed to open durable store at {}: {error}",
2741            data_dir.display()
2742        ),
2743    })?;
2744    liminal::durability::record_store_opened();
2745    super::reclaim_placement::arm_watch(&database);
2746    Ok(database)
2747}
2748
2749/// The stored keys this crate's create arm always writes, with the value it
2750/// writes, for a `config.json` written before haematite had the key.
2751///
2752/// - `node_cache_budget`: haematite 0.8.3 refuses a store without it
2753///   (`MissingNodeCacheBudget`); `"unlimited"` is the pre-budget behaviour.
2754/// - `retention`: `"keep_all"` is the 0.14.1 behaviour, written in so the
2755///   store's stored retention is explicit before the open. The open itself is
2756///   [`Database::open_with_retention`] under [`DURABLE_RETENTION`], which then
2757///   records reclaim in place of it, so an adopted store reclaims from that
2758///   same open onward (spine reclaim, Hermes's ruling of 6 October 2026:
2759///   `past_roots` 0).
2760const ADOPTED_CONFIG_KEYS: [(&str, &str); 2] = [
2761    ("node_cache_budget", "unlimited"),
2762    ("retention", "keep_all"),
2763];
2764
2765/// Writes the values of [`ADOPTED_CONFIG_KEYS`] into a `config.json` that
2766/// lacks any of them, so a store created before haematite had the key opens on
2767/// upgrade with no operator hand. The open that follows then moves its
2768/// retention onto [`DURABLE_RETENTION`].
2769///
2770/// This crate IS the caller that owns these decisions, and it already makes
2771/// them on every create, so answering them here, once, on upgrade, changes
2772/// nothing about how the store runs and takes the file off the operator's
2773/// desk (the store migrates itself or it doesn't ship).
2774///
2775/// Touches the file in exactly one case: a readable JSON object missing one of
2776/// those keys. Anything else — unreadable bytes, a non-object, an explicit
2777/// value however wrong — is left byte-identical so [`Database::open`] surfaces
2778/// haematite's own sharpest refusal instead of this function guessing over
2779/// it. The write is atomic and durable (tmp file beside the target, fsync,
2780/// rename, directory fsync): a crash mid-migration leaves the original bytes,
2781/// and the next boot migrates again.
2782fn adopt_missing_config_keys(data_dir: &Path, config_file: &Path) -> Result<(), ServerError> {
2783    let Ok(bytes) = std::fs::read(config_file) else {
2784        return Ok(());
2785    };
2786    let Ok(mut config) = serde_json::from_slice::<serde_json::Value>(&bytes) else {
2787        return Ok(());
2788    };
2789    let Some(object) = config.as_object_mut() else {
2790        return Ok(());
2791    };
2792    let mut adopted = Vec::new();
2793    for (key, value) in ADOPTED_CONFIG_KEYS {
2794        if !object.contains_key(key) {
2795            object.insert(key.to_owned(), serde_json::Value::String(value.to_owned()));
2796            adopted.push(key);
2797        }
2798    }
2799    if adopted.is_empty() {
2800        return Ok(());
2801    }
2802    let keys = adopted.join(", ");
2803    let migrated =
2804        serde_json::to_vec_pretty(&config).map_err(|error| ServerError::ConfigValidation {
2805            message: format!(
2806                "durable store config at {} predates `{keys}` and its migration could not be \
2807                 serialized: {error}",
2808                config_file.display()
2809            ),
2810        })?;
2811    let write_error = |step: &str, error: std::io::Error| ServerError::ConfigValidation {
2812        message: format!(
2813            "durable store config at {} predates `{keys}` and migrating it failed at {step}: \
2814             {error}",
2815            config_file.display()
2816        ),
2817    };
2818    let tmp_file = data_dir.join("config.json.migrate-tmp");
2819    let mut tmp = std::fs::File::create(&tmp_file)
2820        .map_err(|error| write_error("creating the temporary file", error))?;
2821    std::io::Write::write_all(&mut tmp, &migrated)
2822        .map_err(|error| write_error("writing the temporary file", error))?;
2823    tmp.sync_all()
2824        .map_err(|error| write_error("syncing the temporary file", error))?;
2825    drop(tmp);
2826    std::fs::rename(&tmp_file, config_file)
2827        .map_err(|error| write_error("renaming over config.json", error))?;
2828    std::fs::File::open(data_dir)
2829        .and_then(|dir| dir.sync_all())
2830        .map_err(|error| write_error("syncing the data directory", error))?;
2831    tracing::warn!(
2832        config = %config_file.display(),
2833        keys = %keys,
2834        "durable store config predates these keys -- wrote the create-arm values (the \
2835         behaviour-preserving ones) so the store opens with no operator action"
2836    );
2837    Ok(())
2838}
2839
2840pub(super) fn reserved_namespace_refusal(channel: &str) -> Option<ServerError> {
2841    if channel.starts_with("liminal.") {
2842        crate::metrics::self_stream_reserved_publication();
2843        Some(ServerError::ReservedNamespace {
2844            channel: channel.to_owned(),
2845        })
2846    } else {
2847        None
2848    }
2849}
2850
2851impl LiminalConnectionServices {
2852    pub(in crate::server) const fn signals(&self) -> &ServerSignals {
2853        &self.signals
2854    }
2855
2856    pub(in crate::server) fn self_stream_head(&self, name: &str) -> Result<u64, ServerError> {
2857        let channels = self.read_channels()?;
2858        let channel = channels
2859            .get(name)
2860            .ok_or_else(|| ServerError::ListenerAccept {
2861                message: format!("self stream '{name}' is absent after append"),
2862            })?;
2863        let heads =
2864            channel
2865                .handle
2866                .durable_positions()
2867                .map_err(|error| ServerError::ListenerAccept {
2868                    message: format!("self stream '{name}' head is unavailable: {error}"),
2869                })?;
2870        drop(channels);
2871        match heads.as_slice() {
2872            [head] => Ok(*head),
2873            _ => Err(ServerError::ListenerAccept {
2874                message: format!("self stream '{name}' must have exactly one durable partition"),
2875            }),
2876        }
2877    }
2878
2879    pub(in crate::server) fn publish_self_stream_batch(
2880        &self,
2881        payloads: &[Vec<u8>],
2882        publisher: &str,
2883    ) -> Result<u64, ServerError> {
2884        if publisher.trim().is_empty() || payloads.is_empty() {
2885            return Err(ServerError::ConfigValidation {
2886                message: "self-stream batch requires a publisher and rows".to_owned(),
2887            });
2888        }
2889        let configured = self
2890            .admit_channel("liminal.events")
2891            .map_err(|error| access_to_server_error(&error))?;
2892        let deliveries = configured
2893            .handle
2894            .publish_batch_with_delivery(
2895                payloads,
2896                &liminal::envelope::PublisherId::new(publisher),
2897                None,
2898            )
2899            .map_err(|error| match error {
2900                liminal::error::LiminalError::PublishUnresolved { append } => {
2901                    ServerError::PublishUnresolved {
2902                        channel: "liminal.events".to_owned(),
2903                        idempotency_key: None,
2904                        sequence: Some(append.sequence()),
2905                        message: append.to_string(),
2906                    }
2907                }
2908                error => ServerError::ListenerAccept {
2909                    message: format!("self-stream receipt batch failed: {error}"),
2910                },
2911            })?;
2912        if deliveries.len() != payloads.len()
2913            || deliveries
2914                .iter()
2915                .any(|delivery| delivery.durable_position().is_none())
2916        {
2917            return Err(ServerError::ListenerAccept {
2918                message: "self-stream receipt batch was refused before append".to_owned(),
2919            });
2920        }
2921        for pair in deliveries.windows(2) {
2922            if pair[0]
2923                .durable_position()
2924                .and_then(|position| position.checked_add(1))
2925                != pair[1].durable_position()
2926            {
2927                return Err(ServerError::ListenerAccept {
2928                    message: "self-stream receipt batch positions are not contiguous".to_owned(),
2929                });
2930            }
2931        }
2932        for delivery in &deliveries {
2933            Self::record_delivery_counts(delivery);
2934        }
2935        deliveries
2936            .last()
2937            .and_then(ChannelDelivery::durable_position)
2938            .and_then(|position| position.checked_add(1))
2939            .ok_or_else(|| ServerError::ListenerAccept {
2940                message: "self-stream receipt batch head is unavailable".to_owned(),
2941            })
2942    }
2943
2944    pub(in crate::server) fn checkpoint_self_stream_heads(
2945        &self,
2946        heads: [u64; 2],
2947    ) -> Result<(), ServerError> {
2948        for (name, head) in ["liminal.events:0", "liminal.cost:0"]
2949            .into_iter()
2950            .zip(heads)
2951        {
2952            block_on(liminal::durability::head_checkpoint::advance_head(
2953                self.durable_store.as_ref(),
2954                name,
2955                head,
2956            ))
2957            .map_err(|error| ServerError::ShutdownFlush {
2958                message: format!("self-stream head checkpoint bridge for '{name}' failed: {error}"),
2959            })?
2960            .map_err(|error| ServerError::ShutdownFlush {
2961                message: format!("self-stream head checkpoint for '{name}' failed: {error}"),
2962            })?;
2963        }
2964        Ok(())
2965    }
2966
2967    pub(in crate::server) fn install_self_stream_channels(&self) -> Result<(), ServerError> {
2968        let names = ["liminal.cost", "liminal.events"];
2969        {
2970            let channels = self.read_channels()?;
2971            for name in names {
2972                if channels.contains_key(name) {
2973                    return Err(ServerError::ConfigValidation {
2974                        message: format!("self-stream channel '{name}' is already configured"),
2975                    });
2976                }
2977            }
2978        }
2979        let mut built = Vec::with_capacity(names.len());
2980        for name in names {
2981            let schema =
2982                resolve_schema_bytes(None).map_err(|error| ServerError::ConfigValidation {
2983                    message: format!("self-stream channel '{name}' schema failed: {error}"),
2984                })?;
2985            let channel = build_configured_channel(
2986                name,
2987                schema,
2988                true,
2989                ChannelOrigin::BootConfigured,
2990                &self.durable_store,
2991                self.cluster.supervisor(),
2992            )
2993            .map_err(|error| ServerError::ConfigValidation {
2994                message: error.boot_message(name),
2995            })?;
2996            channel
2997                .handle
2998                .install_publication_identity(self.publication_identity.clone())
2999                .map_err(|error| ServerError::ConfigValidation {
3000                    message: error.to_string(),
3001                })?;
3002            built.push((name.to_owned(), Arc::new(channel)));
3003        }
3004        let mut channels =
3005            self.channels
3006                .write()
3007                .map_err(|error| ServerError::ConfigValidation {
3008                    message: format!("self-stream channel roster lock poisoned: {error}"),
3009                })?;
3010        for (name, _) in &built {
3011            if channels.contains_key(name) {
3012                return Err(ServerError::ConfigValidation {
3013                    message: format!("self-stream channel '{name}' was concurrently configured"),
3014                });
3015            }
3016        }
3017        channels.extend(built);
3018        drop(channels);
3019        // SRV-012 R7: every roster channel but `console.*` is a line, and
3020        // these join after the hub's model was built from the roster.
3021        for name in names {
3022            self.console_changed(crate::console::TopologyChange::Channel {
3023                name: name.to_owned(),
3024                life: line_life(true),
3025                state: crate::console::records::LineState::Live,
3026            });
3027        }
3028        Ok(())
3029    }
3030
3031    /// SRV-012 R4: registers `console.topology` as an ephemeral,
3032    /// server-owned channel (never a store write). A channel of that name
3033    /// already configured (ephemeral, since config refuses a durable one) is
3034    /// kept as it is.
3035    pub(in crate::server) fn install_console_channels(&self) -> Result<(), ServerError> {
3036        self.register_console_channel(crate::console::TOPOLOGY_SUBJECT)
3037            .map_err(|error| ServerError::ConfigValidation {
3038                message: format!("console channel registration failed: {error}"),
3039            })
3040    }
3041
3042    /// Puts the ephemeral console channel `name` on the roster unless it is
3043    /// there.
3044    fn register_console_channel(&self, name: &str) -> Result<(), ChannelRegistryError> {
3045        if self.read_roster()?.contains_key(name) {
3046            return Ok(());
3047        }
3048        let schema = resolve_schema_bytes(None).map_err(|error| {
3049            ChannelRegistryError::RosterUnavailable {
3050                message: format!("console channel '{name}' schema failed: {error}"),
3051            }
3052        })?;
3053        let channel = build_configured_channel(
3054            name,
3055            schema,
3056            false,
3057            ChannelOrigin::BootConfigured,
3058            &self.durable_store,
3059            self.cluster.supervisor(),
3060        )
3061        .map_err(|error| error.into_registry_error(name))?;
3062        channel
3063            .handle
3064            .install_publication_identity(self.publication_identity.clone())
3065            .map_err(|error| ChannelRegistryError::RosterUnavailable {
3066                message: error.to_string(),
3067            })?;
3068        self.write_roster()?
3069            .entry(name.to_owned())
3070            .or_insert_with(|| Arc::new(channel));
3071        Ok(())
3072    }
3073
3074    /// Publishes on an ordinary channel for an application embedding the server.
3075    ///
3076    /// The live and durable publisher identity is liminal.<host>. Reserved self
3077    /// streams and console channels retain the same refusal as connection publishes.
3078    /// An idempotency key has the same delivery semantics as a connection publish.
3079    ///
3080    /// # Errors
3081    ///
3082    /// Returns a configuration error for an empty or whitespace-only host name,
3083    /// a namespace refusal for a reserved channel, or the existing admission,
3084    /// durability, and publication error.
3085    pub fn publish_embedded(
3086        &self,
3087        host: &str,
3088        channel: &str,
3089        envelope: &MessageEnvelope,
3090        idempotency_key: Option<&str>,
3091    ) -> Result<PublishOutcome, ServerError> {
3092        self.publish_with_origin(
3093            channel,
3094            envelope,
3095            idempotency_key,
3096            PublishOrigin::Host { name: host },
3097        )
3098        .map(|published| published.outcome)
3099    }
3100
3101    pub(in crate::server) fn publish_with_origin(
3102        &self,
3103        channel: &str,
3104        envelope: &MessageEnvelope,
3105        idempotency_key: Option<&str>,
3106        origin: PublishOrigin<'_>,
3107    ) -> Result<OriginPublication, ServerError> {
3108        if !origin.is_server() {
3109            if plan_subject(channel).is_some() {
3110                return Err(site_services::site_error(
3111                    crate::services::lifecycle::site::SiteError::Invalid(
3112                        "plan subject is server-owned",
3113                    ),
3114                ));
3115            }
3116            crate::agents::binding::refuse_server_publish(channel)?;
3117        }
3118        validate_publication_origin(channel, idempotency_key, origin)?;
3119        // SRV-012 to_stations: only while a console tap is live does the
3120        // fan-out record whom the message reached, in a buffer the tap lends
3121        // and takes back; otherwise the publish is the plain one.
3122        let mut recipients = self.signals.lend_recipients();
3123        // Lane brief ruling 6: a tapped message is first seen as the server
3124        // receives it, before admission; the clock is read only while a tap
3125        // lent a record.
3126        let received_at = recipients.is_some().then(Instant::now);
3127        let result = (|| {
3128            // The roster read holds its guard only long enough to clone the entry's
3129            // `Arc` out; everything below — the dedup bridge and the publish into the
3130            // channel actor — runs with no roster lock held.
3131            //
3132            // This inner admission STAYS even once the connection process consults
3133            // the roster ahead of delegating: this method is public and callable
3134            // without going through a frame at all, and a guard that only exists in
3135            // the caller is not a guard.
3136            let configured = self
3137                .admit_channel(channel)
3138                .map_err(|error| access_to_server_error(&error))?;
3139
3140            // Dedup-on-delivery: a publish carrying an idempotency key is delivered to
3141            // subscribers AT MOST ONCE across re-publishes of the same key. Only a
3142            // fresh `Claimed` decision proceeds to fan-out; a `Completed`/`InFlight`
3143            // decision is a duplicate and is suppressed (no second delivery), which is
3144            // the at-most-once guarantee the aion outbox relies on.
3145            // A1 §5: the namespace is chosen by the channel's DURABILITY, so an
3146            // ephemeral receipt cannot outlive the message it vouches for.
3147            // A keyed publish to a durable channel claims its key WITH the channel
3148            // as evidence and holds the claim across an unresolved append
3149            // (`unresolved.rs`); the ephemeral namespace keeps the claim below.
3150            if let (Some(key), ChannelMode::Durable) =
3151                (idempotency_key, configured.handle.config().mode)
3152            {
3153                return self
3154                    .publish_durable_keyed(
3155                        &configured,
3156                        channel,
3157                        (envelope, origin),
3158                        key,
3159                        recipients.as_mut(),
3160                    )
3161                    .map(OriginPublication::connection);
3162            }
3163            let dedup = self.dedup_for(&configured);
3164            if let Some(key) = idempotency_key {
3165                if !Self::claim_delivery(dedup, key)? {
3166                    // A dedup-suppressed re-publish is still an accepted publish (it is
3167                    // assigned a message id), but it reaches no subscriber, so it counts
3168                    // toward publishes and not deliveries.
3169                    crate::metrics::publish_accepted();
3170                    return Ok(OriginPublication::connection(PublishOutcome {
3171                        message_id: self.next_message_id()?,
3172                        delivered: false,
3173                        durable_position: None,
3174                        reach: Reach::Suppressed,
3175                    }));
3176                }
3177            }
3178
3179            let delivery = fan_out(&configured, envelope, origin, recipients.as_mut());
3180            let delivery = match delivery {
3181                Ok(delivery) => delivery,
3182                // An unkeyed durable publish whose append outcome is not known is
3183                // answered as unresolved, never as a definite failure a producer
3184                // would retry into a second row.
3185                Err(liminal::error::LiminalError::PublishUnresolved { append }) => {
3186                    return Err(ServerError::PublishUnresolved {
3187                        channel: channel.to_owned(),
3188                        idempotency_key: None,
3189                        sequence: Some(append.sequence()),
3190                        message: append.to_string(),
3191                    });
3192                }
3193                Err(error) => {
3194                    // The claim above appended an `InFlight` entry but the delivery
3195                    // failed before `complete_receipt` could run. Release the claim so
3196                    // the key is re-claimable; otherwise every re-publish would see
3197                    // `InFlight` and be suppressed forever. Best-effort: surface the
3198                    // ORIGINAL publish error regardless, but never swallow a release
3199                    // failure silently (it leaves the leak intact).
3200                    if let Some(key) = idempotency_key {
3201                        Self::release_claim(dedup, key);
3202                    }
3203                    return Err(ServerError::ListenerAccept {
3204                        message: format!("liminal publish failed for channel '{channel}': {error}"),
3205                    });
3206                }
3207            };
3208
3209            // A1 §5, the two-way dedup mapping. Every publish outcome maps to
3210            // EXACTLY ONE of complete/release — the same shape as the C1 contract:
3211            //
3212            // * **Defer (or Accept) => `complete_receipt`.** The message is
3213            //   admitted and bus-held, so the key is done: a producer retrying it
3214            //   hits `Completed` and there is no duplicate delivery and no InFlight
3215            //   leak. This is the path that makes "re-publishing with the same key
3216            //   is safe and is suppressed while the bus owns the message" true.
3217            // * **Reject => `release_claim`.** Aggregation makes a producer-visible
3218            //   Reject mean delivered-to-zero, so the correct dedup state is "as if
3219            //   never published": tombstone it so a deliberate retry can re-claim.
3220            //   `release_claim` never clobbers a receipt, so the at-most-once guard
3221            //   survives a concurrent completion racing ahead.
3222            //
3223            // Mixed accept/reject reports Defer and so completes — consistent with
3224            // "at least one delivery happened".
3225            if let Some(key) = idempotency_key {
3226                if delivery.is_admitted() {
3227                    block_on(dedup.complete_receipt(key, ProcessingReceipt::new(Vec::new())))
3228                        .map_err(|error| ServerError::ListenerAccept {
3229                            message: format!(
3230                                "dedup receipt bridge failed for key '{key}': {error}"
3231                            ),
3232                        })?
3233                        .map_err(|error| ServerError::ListenerAccept {
3234                            message: format!("dedup receipt write failed for key '{key}': {error}"),
3235                        })?;
3236                } else {
3237                    Self::release_claim(dedup, key);
3238                }
3239            }
3240
3241            let self_row_at = delivery.durable_position().filter(|_| origin.is_server());
3242            Ok(OriginPublication {
3243                outcome: Self::delivered(&delivery)?,
3244                self_row_appended: self_row_at.is_some(),
3245                self_row_at,
3246            })
3247        })();
3248        self.signals.published(
3249            channel,
3250            origin,
3251            envelope,
3252            &result,
3253            recipients.as_ref().zip(received_at),
3254        );
3255        if let Some(buffer) = recipients {
3256            self.signals.give_back(buffer);
3257        }
3258        result
3259    }
3260}
3261
3262/// The library publish of an unkeyed message, recording what it saw in
3263/// `recipients` when the tap lent a record.
3264fn fan_out(
3265    configured: &ConfiguredChannel,
3266    envelope: &MessageEnvelope,
3267    origin: PublishOrigin<'_>,
3268    recipients: Option<&mut liminal::channel::PublishRecord>,
3269) -> Result<liminal::channel::ChannelDelivery, liminal::error::LiminalError> {
3270    let publisher = || publisher_for_origin(origin);
3271    recipients.map_or_else(
3272        || {
3273            configured
3274                .handle
3275                .publish_with_delivery(&envelope.payload, publisher(), None)
3276        },
3277        |recipients| {
3278            configured.handle.publish_recording(
3279                &envelope.payload,
3280                publisher(),
3281                None,
3282                None,
3283                recipients,
3284            )
3285        },
3286    )
3287}
3288
3289fn validate_publication_origin(
3290    channel: &str,
3291    idempotency_key: Option<&str>,
3292    origin: PublishOrigin<'_>,
3293) -> Result<(), ServerError> {
3294    if !origin.is_server() {
3295        refuse_console_namespace(channel).map_err(|error| access_to_server_error(&error))?;
3296        if let Some(error) = reserved_namespace_refusal(channel) {
3297            return Err(error);
3298        }
3299    }
3300    if let PublishOrigin::Host { name } = origin
3301        && name.trim().is_empty()
3302    {
3303        return Err(ServerError::ConfigValidation {
3304            message: "embedded publication requires a nonempty host name".to_owned(),
3305        });
3306    }
3307    if let PublishOrigin::Server { device } = origin {
3308        if device.trim().is_empty() || idempotency_key.is_some() {
3309            return Err(ServerError::ConfigValidation {
3310                message: "server-origin publication requires a device and no idempotency key"
3311                    .to_owned(),
3312            });
3313        }
3314    }
3315    Ok(())
3316}
3317
3318fn publisher_for_origin(origin: PublishOrigin<'_>) -> liminal::envelope::PublisherId {
3319    match origin {
3320        PublishOrigin::Connection { id } => liminal::envelope::PublisherId::new(id.to_string()),
3321        PublishOrigin::Host { name } => {
3322            liminal::envelope::PublisherId::new(format!("liminal.{name}"))
3323        }
3324        PublishOrigin::Server { device } => {
3325            liminal::envelope::PublisherId::new(format!("liminal.{device}"))
3326        }
3327    }
3328}
3329
3330impl ConnectionServices for LiminalConnectionServices {
3331    fn construction_request(
3332        &self,
3333        session: Option<&dyn crate::auth_lys::session::LysSession>,
3334        subject: &str,
3335        bytes: &[u8],
3336        at_ms: u64,
3337    ) -> Option<Result<serde_json::Value, ServerError>> {
3338        if let Some(answer) = self.agent_host_request(session, subject, bytes, at_ms) {
3339            return Some(answer);
3340        }
3341        is_site_route(subject).then(|| {
3342            self.serve_site_tool(session, subject, bytes, at_ms)
3343                .map_err(site_services::site_error)
3344        })
3345    }
3346    fn prepare_site_trial(
3347        &self,
3348        connection: u64,
3349        session: Option<&dyn crate::auth_lys::session::LysSession>,
3350        registration: &liminal::protocol::WorkerRegistration,
3351        process: Option<&crate::services::lifecycle::ProcessGrant>,
3352        at_ms: u64,
3353    ) -> Result<Option<crate::services::lifecycle::trial::TrialPermit>, ServerError> {
3354        self.site_lifecycle.get().map_or(Ok(None), |lifecycle| {
3355            lifecycle
3356                .prepare_trial(connection, session, registration, process, at_ms)
3357                .map_err(site_services::site_error)
3358        })
3359    }
3360    fn admit_site_call(
3361        &self,
3362        session: Option<&dyn crate::auth_lys::session::LysSession>,
3363        subject: &str,
3364        at_ms: u64,
3365    ) -> Result<bool, ServerError> {
3366        self.site_lifecycle.get().map_or(Ok(false), |lifecycle| {
3367            lifecycle
3368                .admit_trial_call(session, subject, at_ms)
3369                .map_err(site_services::site_error)
3370        })
3371    }
3372    fn site_registration_closed(&self, connection: u64) -> Result<(), ServerError> {
3373        if let Some(lifecycle) = self.site_lifecycle.get() {
3374            lifecycle
3375                .trial_closed(connection)
3376                .map_err(site_services::site_error)?;
3377            lifecycle
3378                .refresh_sites()
3379                .map_err(site_services::site_error)?;
3380        }
3381        Ok(())
3382    }
3383    fn publish_identified(
3384        &self,
3385        connection: u64,
3386        session: Option<&dyn crate::auth_lys::session::LysSession>,
3387        channel: &str,
3388        envelope: &MessageEnvelope,
3389        key: Option<&str>,
3390    ) -> Result<PublishOutcome, ServerError> {
3391        if plan_subject(channel).is_some() {
3392            return Err(site_services::site_error(
3393                crate::services::lifecycle::site::SiteError::Invalid(
3394                    "plan subject is server-owned",
3395                ),
3396            ));
3397        }
3398        self.publish_agent(connection, session, channel, envelope, key)
3399    }
3400    fn agents(&self) -> Option<&Arc<crate::agents::runtime::Agents>> {
3401        self.agents.as_ref()
3402    }
3403
3404    fn subscribe_identified(
3405        &self,
3406        connection: u64,
3407        session: Option<&dyn crate::auth_lys::session::LysSession>,
3408        channel: &str,
3409        schemas: &[ProtocolSchemaId],
3410        install: Option<liminal::channel::InboxInstall>,
3411    ) -> Result<ConnectionSubscription, ServerError> {
3412        if plan_subject(channel).is_some() {
3413            return self.subscribe_site_plan(session, channel, schemas, install);
3414        }
3415        self.subscribe_agent(connection, session, channel, schemas, install)
3416    }
3417    fn participant_service(&self) -> Option<InstalledParticipantService> {
3418        self.participant_service.clone()
3419    }
3420
3421    /// The roster's admission decision, exposed across the trait boundary as the
3422    /// permission alone.
3423    ///
3424    /// Two methods share the name `admit_channel` on this type: this trait
3425    /// method, and the private inherent funnel it delegates to. They are the
3426    /// same decision at two different boundaries — the funnel returns the ENTRY
3427    /// the decision was made on, because its in-crate callers go on to publish
3428    /// or subscribe through it, and this one returns `()`, because a caller
3429    /// outside the crate is asking whether it MAY, not for the thing itself.
3430    /// Handing the entry out here would make [`ConfiguredChannel`] public
3431    /// surface and hand a frame-level caller a channel handle it has no business
3432    /// holding. Rust resolves an inherent method ahead of a trait method of the
3433    /// same name, so the funnel's existing callers — and the `Self::` call below
3434    /// — reach the funnel, not this method; the trait form is reached only
3435    /// through a `dyn ConnectionServices`, which is exactly the caller it exists
3436    /// for. `an_admitted_operation_the_service_then_refuses_stays_undifferentiated`
3437    /// is the instrument that would catch this delegation turning into a
3438    /// recursion.
3439    ///
3440    /// `operation` is where the answers part: absence and quiesce refuse
3441    /// publishes and subscribes alike, but the console namespace refuses only
3442    /// a connection's publish (a console reads `console.*` as an ordinary
3443    /// subscriber, and only the server may write there), and a tap target
3444    /// refuses only a subscribe. Both are checked before the roster, so a
3445    /// client is told the same thing whether or not the subject is registered
3446    /// yet.
3447    fn admit_channel(
3448        &self,
3449        operation: ChannelOperation,
3450        channel: &str,
3451    ) -> Result<(), ChannelAccessError> {
3452        if let Some(name) = plan_subject(channel) {
3453            let refused = |message| ChannelAccessError::RosterUnavailable { message };
3454            if operation == ChannelOperation::Publish {
3455                return Err(refused("plan subject is server-owned".into()));
3456            }
3457            let lifecycle = self
3458                .site_lifecycle
3459                .get()
3460                .ok_or_else(|| refused("construction not installed".into()))?;
3461            lifecycle
3462                .record(name)
3463                .map_err(|error| refused(error.to_string()))?;
3464            return Ok(());
3465        }
3466        if let Some(admitted) = self.agent_channel_admission(channel) {
3467            return admitted;
3468        }
3469        match operation {
3470            ChannelOperation::Publish => refuse_console_namespace(channel)?,
3471            ChannelOperation::Subscribe => {
3472                refuse_console_tap_target(channel)?;
3473                self.refuse_console_tap_limit(channel)?;
3474                // SRV-012 R4/R6: a tap channel exists while it has a
3475                // subscriber, and only while the console streams run.
3476                let admitted = if self.signals.console_enabled()
3477                    && super::channel_registry::console_tap_target(channel).is_some()
3478                {
3479                    self.register_console_channel(channel).map_err(|error| {
3480                        ChannelAccessError::RosterUnavailable {
3481                            message: error.to_string(),
3482                        }
3483                    })
3484                } else {
3485                    Ok(())
3486                }
3487                .and_then(|()| Self::admit_channel(self, channel).map(|_| ()));
3488                if admitted.is_err() {
3489                    self.give_back_tap(channel);
3490                }
3491                return admitted;
3492            }
3493        }
3494        Self::admit_channel(self, channel)?;
3495        Ok(())
3496    }
3497
3498    fn publish(
3499        &self,
3500        connection: u64,
3501        channel: &str,
3502        envelope: &MessageEnvelope,
3503        idempotency_key: Option<&str>,
3504    ) -> Result<PublishOutcome, ServerError> {
3505        // The frame path refused this already at admission. This guard is for
3506        // a caller holding the trait directly: a guard that only exists in the
3507        // caller is not a guard.
3508        refuse_console_namespace(channel).map_err(|error| access_to_server_error(&error))?;
3509        if plan_subject(channel).is_some() {
3510            return Err(site_services::site_error(
3511                crate::services::lifecycle::site::SiteError::Invalid(
3512                    "plan subject is server-owned",
3513                ),
3514            ));
3515        }
3516        crate::agents::binding::refuse_server_publish(channel)?;
3517        if crate::agents::binding::subject(channel).is_some() {
3518            return Err(crate::agents::error::AgentError::Invalid(
3519                "agent publish requires the verified identity route",
3520            )
3521            .into());
3522        }
3523        self.publish_with_origin(
3524            channel,
3525            envelope,
3526            idempotency_key,
3527            PublishOrigin::Connection { id: connection },
3528        )
3529        .map(|published| published.outcome)
3530    }
3531
3532    fn subscribe(
3533        &self,
3534        channel: &str,
3535        accepted_schemas: &[ProtocolSchemaId],
3536        install: Option<liminal::channel::InboxInstall>,
3537    ) -> Result<ConnectionSubscription, ServerError> {
3538        if plan_subject(channel).is_some() {
3539            return Err(site_services::site_error(
3540                crate::services::lifecycle::site::SiteError::Invalid(
3541                    "plan subscription needs an admitted pass",
3542                ),
3543            ));
3544        }
3545        if crate::agents::binding::subject(channel).is_some() {
3546            return Err(crate::agents::error::AgentError::Invalid(
3547                "agent subscribe requires the verified identity route",
3548            )
3549            .into());
3550        }
3551        let subscribed = self.open_subscription(channel, accepted_schemas, install, None);
3552        if subscribed.is_err() {
3553            // SRV-012 R6: a tap slot reserved at admission for a subscribe
3554            // that did not happen is given back, and a tap channel it
3555            // registered with no tap behind it leaves the roster.
3556            self.give_back_tap(channel);
3557        }
3558        subscribed
3559    }
3560
3561    fn unsubscribe(&self, subscription: ConnectionSubscription) -> Result<(), ServerError> {
3562        subscription.unsubscribe()
3563    }
3564
3565    fn open_conversation(
3566        &self,
3567        conversation_id: u64,
3568        subject: &str,
3569    ) -> Result<ConnectionConversation, ServerError> {
3570        self.open_routed_conversation(conversation_id, subject)
3571    }
3572
3573    fn conversation_message(
3574        &self,
3575        conversation: &ConnectionConversation,
3576        envelope: &MessageEnvelope,
3577        op_id: Option<u64>,
3578    ) -> Result<(), ServerError> {
3579        self.routed_conversation_message(conversation, envelope, op_id)
3580    }
3581
3582    fn close_conversation(&self, conversation: ConnectionConversation) -> Result<(), ServerError> {
3583        conversation.close()
3584    }
3585
3586    fn listening(self: Arc<Self>) -> Option<std::thread::JoinHandle<()>> {
3587        self.resolve_claims_behind_listener()
3588    }
3589
3590    fn stop_participant_warm(&self) {
3591        // DUR-020 R5: the warm stops at its next conversation boundary and
3592        // the stop does not wait for it; whatever it had not restored
3593        // restores on the next start. Then the clean stop records the
3594        // capacity snapshot (R3), and the final flush makes it durable.
3595        if let Some(warm) = &self.participant_warm {
3596            warm.stop();
3597            warm.record_clean_stop();
3598        }
3599    }
3600
3601    fn finish_background_store_work(&self) -> Result<(), ServerError> {
3602        self.finish_background_work()
3603    }
3604
3605    fn release_store_holders(&self) -> Result<(), ServerError> {
3606        self.release_holders()
3607    }
3608
3609    fn flush_durable_state(&self) -> Result<(), ServerError> {
3610        // A flush observer publishes to a durable channel, so no channel append lock can be held.
3611        // Separate ephemeral dedup state dies with this incarnation; shared dedup state is covered here.
3612        self.signals.with_flush_barrier(|| {
3613            block_on(self.head_checkpoint_stop.checkpoint_heads())
3614                .map_err(|error| ServerError::ShutdownFlush {
3615                    message: format!("clean-stop head checkpoint bridge failed: {error}"),
3616                })?
3617                .map_err(|error| ServerError::ShutdownFlush {
3618                    message: format!("clean-stop head checkpoint failed: {error}"),
3619                })?;
3620            block_on(self.durable_store.flush())
3621                .map_err(|error| ServerError::ShutdownFlush {
3622                    message: format!("shared durable store flush bridge failed: {error}"),
3623                })?
3624                .map_err(|error| ServerError::ShutdownFlush {
3625                    message: format!("shared durable store flush failed: {error}"),
3626                })?;
3627            Ok(())
3628        })?;
3629        Ok(())
3630    }
3631}
3632
3633/// One roster entry: a channel's library handle, the protocol schema id
3634/// advertised for it, where it came from, and its admission state.
3635#[derive(Debug)]
3636pub(super) struct ConfiguredChannel {
3637    handle: ChannelHandle,
3638    protocol_schema: ProtocolSchemaId,
3639    /// Durable (stored) or ephemeral, as built; the console's line life.
3640    durable: bool,
3641    /// Boot-configured or runtime-registered. Written once at construction and
3642    /// NEVER flipped: it is the only field that predicts what a restart does to
3643    /// this entry, and it defines the population the registration cap counts.
3644    origin: ChannelOrigin,
3645    /// [`STATE_ACTIVE`] or [`STATE_QUIESCED`]. Written at most once, by one
3646    /// `compare_exchange`, so the transition is one-way and exactly one caller
3647    /// can perform it.
3648    state: AtomicU8,
3649    /// The quiesce cause, set STRICTLY BEFORE `state` flips so any reader that
3650    /// observes [`STATE_QUIESCED`] can read it. Written once (the `OnceLock`
3651    /// enforces that structurally), which is why a re-quiesce under a different
3652    /// reason must refuse rather than silently keep or replace one of them.
3653    quiesce_reason: OnceLock<String>,
3654}
3655
3656impl ConfiguredChannel {
3657    /// The entry's admission state as a value.
3658    ///
3659    /// One `Acquire` load and, when quiesced, one read of the already-written
3660    /// reason. Touches no actor: the whole point of recording state on the entry
3661    /// is that observing it cannot spawn the thing being observed.
3662    fn state(&self) -> ChannelState {
3663        if self.state.load(Ordering::Acquire) == STATE_QUIESCED {
3664            ChannelState::Quiesced {
3665                reason: self.recorded_quiesce_reason(),
3666            }
3667        } else {
3668            ChannelState::Active
3669        }
3670    }
3671
3672    /// The recorded quiesce reason, for a caller that has already observed
3673    /// [`STATE_QUIESCED`] with an `Acquire` load.
3674    fn recorded_quiesce_reason(&self) -> String {
3675        self.quiesce_reason
3676            .get()
3677            .map_or_else(|| UNRECORDED_QUIESCE_REASON.to_owned(), Clone::clone)
3678    }
3679
3680    /// Moves this entry from active to quiesced, recording `reason` first.
3681    ///
3682    /// The reason is written to the `OnceLock` BEFORE the `Release`
3683    /// `compare_exchange` that flips the state, so no reader can observe
3684    /// `QUIESCED` without being able to read why. Re-quiescing under the
3685    /// IDENTICAL reason is `Ok(())` — the caller's intent already holds — and a
3686    /// DIFFERENT reason refuses, because the recorded one cannot be replaced
3687    /// without losing a cause and cannot be kept without lying to the caller.
3688    fn quiesce(&self, name: &str, reason: &str) -> Result<(), ChannelRegistryError> {
3689        // Whoever wins the `OnceLock` writes the cause on record; every other
3690        // caller — a re-quiesce or a concurrent racer — is judged against THAT
3691        // reason, never against whether it happened to perform the flip itself.
3692        // Reporting success for having won the CAS would tell a racer whose
3693        // reason lost that its reason took effect.
3694        let recorded = match self.quiesce_reason.set(reason.to_owned()) {
3695            Ok(()) => reason.to_owned(),
3696            Err(_rejected) => self.recorded_quiesce_reason(),
3697        };
3698        // The reason is now readable, so the flip may become visible. The CAS
3699        // result is deliberately unused: the state is `QUIESCED` afterwards
3700        // whether this call or a racer performed the transition, and the answer
3701        // to the caller is governed by the recorded reason above.
3702        let _flipped_here = self.state.compare_exchange(
3703            STATE_ACTIVE,
3704            STATE_QUIESCED,
3705            Ordering::Release,
3706            Ordering::Acquire,
3707        );
3708        if recorded == reason {
3709            return Ok(());
3710        }
3711        Err(ChannelRegistryError::AlreadyQuiesced {
3712            name: name.to_owned(),
3713            reason: recorded,
3714        })
3715    }
3716}
3717
3718#[derive(Debug)]
3719struct LiminalSubscriptionResource {
3720    subscription: liminal::channel::SubscriptionHandle,
3721}
3722
3723impl SubscriptionResource for LiminalSubscriptionResource {
3724    fn try_next_positioned(
3725        &mut self,
3726    ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
3727        self.subscription
3728            .try_next_positioned()
3729            .map_err(|error| subscription_failure(&error))
3730    }
3731
3732    fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
3733        self.subscription
3734            .acknowledge_position(position)
3735            .map_err(|error| subscription_failure(&error))
3736    }
3737
3738    fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
3739        self.subscription
3740            .consumer_position()
3741            .map_err(|error| subscription_failure(&error))
3742    }
3743    fn unsubscribe(self: Box<Self>) -> Result<(), ServerError> {
3744        drop(self.subscription);
3745        Ok(())
3746    }
3747
3748    fn is_overflowed(&self) -> bool {
3749        self.subscription.is_overflowed()
3750    }
3751
3752    fn subscriber_pid(&self) -> Option<u64> {
3753        Some(self.subscription.subscriber_pid())
3754    }
3755
3756    fn has_pending(&self) -> bool {
3757        self.subscription.has_pending()
3758    }
3759
3760    fn try_next(&mut self) -> Option<liminal::envelope::Envelope> {
3761        match self.subscription.try_next() {
3762            Ok(envelope) => envelope,
3763            Err(error) => {
3764                tracing::error!(
3765                    %error,
3766                    "subscription could not receive its next envelope"
3767                );
3768                None
3769            }
3770        }
3771    }
3772}
3773
3774/// Preserves the library's subscription failure at the connection boundary.
3775fn subscription_failure(error: &liminal::error::LiminalError) -> ServerError {
3776    ServerError::ListenerAccept {
3777        message: format!("liminal subscription: {error}"),
3778    }
3779}
3780
3781/// Renders an admission refusal as the service error the trait's callers expect.
3782///
3783/// The two refusals that already existed keep their EXACT bytes: an absent
3784/// channel is still `channel '<name>' is not configured` and a poisoned roster
3785/// is still `channel roster lock poisoned`, so nothing that reads these messages
3786/// today sees a change. Discrimination lives in the reason code
3787/// ([`ChannelAccessError::reason_code`]), which the connection process carries to
3788/// the wire — this rendering is the degraded path for a caller that reaches the
3789/// service directly and has no code to carry.
3790///
3791/// `Quiesced` has no predecessor string to preserve; it carries the refusal's own
3792/// message, reason included.
3793///
3794/// The connection process renders its admission refusals through this SAME
3795/// function rather than through [`ChannelAccessError`]'s own `Display`. Two
3796/// consequences, both wanted. The bytes on the wire for an absent channel stay
3797/// exactly what they have always been, which is the whole of the semver promise
3798/// this lane makes. And an operation refused AT admission and one refused after
3799/// admission by the service produce the identical message, differing only in the
3800/// reason code — so the code really is the only discriminator, instead of the
3801/// message quietly becoming a second one.
3802pub(super) fn access_to_server_error(error: &ChannelAccessError) -> ServerError {
3803    let message = match error {
3804        ChannelAccessError::NotRegistered { name } => {
3805            format!("channel '{name}' is not configured")
3806        }
3807        ChannelAccessError::RosterUnavailable { message } => message.clone(),
3808        ChannelAccessError::Quiesced { .. }
3809        | ChannelAccessError::ConsoleNamespace { .. }
3810        | ChannelAccessError::ConsoleTapTarget { .. }
3811        | ChannelAccessError::ConsoleTapLimit { .. } => error.to_string(),
3812    };
3813    ServerError::ListenerAccept { message }
3814}
3815
3816/// Refuses a subscribe to `console.tap.<target>` whose target is empty or
3817/// under `console.`: the tap never taps the console's own records.
3818///
3819/// # Errors
3820/// Returns [`ChannelAccessError::ConsoleTapTarget`] naming the channel.
3821fn refuse_console_tap_target(channel: &str) -> Result<(), ChannelAccessError> {
3822    use super::channel_registry::{console_tap_target, is_console_subject};
3823    match console_tap_target(channel) {
3824        Some(target) if target.is_empty() || is_console_subject(target) => {
3825            Err(ChannelAccessError::ConsoleTapTarget {
3826                name: channel.to_owned(),
3827            })
3828        }
3829        _ => Ok(()),
3830    }
3831}
3832
3833/// Refuses a connection's publish under `console.`, the server's own namespace.
3834///
3835/// # Errors
3836/// Returns [`ChannelAccessError::ConsoleNamespace`] naming the channel.
3837fn refuse_console_namespace(channel: &str) -> Result<(), ChannelAccessError> {
3838    if super::channel_registry::is_console_subject(channel) {
3839        return Err(ChannelAccessError::ConsoleNamespace {
3840            name: channel.to_owned(),
3841        });
3842    }
3843    Ok(())
3844}
3845
3846pub(super) fn server_error_from_protocol(error: &ProtocolError) -> ServerError {
3847    ServerError::ListenerAccept {
3848        message: format!("protocol operation failed: {error}"),
3849    }
3850}
3851
3852/// The three pinned registration tests. A CHILD module of `services` because
3853/// test 2 must hold the exact entry the admission funnel handed out.
3854#[cfg(test)]
3855#[path = "services_registry_tests.rs"]
3856mod services_registry_tests;
3857
3858#[cfg(test)]
3859#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
3860mod durable_store_tests {
3861    use liminal::durability::open_ephemeral_rooted;
3862
3863    use super::subsystem_census::RecordingSubsystems;
3864    use super::{
3865        ConnectionServices, DEFAULT_SHARD_COUNT, LiminalConnectionServices, SchedulerSubsystem,
3866        build_connection_services, build_connection_services_via, build_durable_store,
3867    };
3868    use crate::ServerError;
3869    use crate::config::types::{LimitsConfig, ServerConfig, ServicesConfig};
3870
3871    /// Counts directory entries under `root`, for the empty/one-dir assertions
3872    /// on an injected ephemeral root.
3873    fn entry_count(root: &std::path::Path) -> usize {
3874        std::fs::read_dir(root)
3875            .expect("ephemeral root is readable")
3876            .count()
3877    }
3878
3879    /// A minimal channel-free config with the given service `profile`. No channels,
3880    /// routing, persistence, or cluster — the shape both profiles accept (the full
3881    /// profile simply builds an empty channel set; the worker-front-door profile
3882    /// requires exactly this shape).
3883    fn config_with_profile(profile: &str) -> ServerConfig {
3884        ServerConfig {
3885            listen_address: "127.0.0.1:0".parse().expect("valid socket addr"),
3886            extra_listen_addresses: Vec::new(),
3887            health_listen_address: "127.0.0.1:1".parse().expect("valid socket addr"),
3888            drain_timeout_ms: 30_000,
3889            channels: Vec::new(),
3890            routing_rules: Vec::new(),
3891            persistence_path: None,
3892            cluster: None,
3893            auth: None,
3894            services: ServicesConfig {
3895                profile: profile.to_owned(),
3896                ..ServicesConfig::default()
3897            },
3898            limits: LimitsConfig::default(),
3899            participant: None,
3900            websocket: None,
3901        }
3902    }
3903
3904    /// §9 D2 front-door construction gate (fs half): building the worker-front-door
3905    /// services creates NO haematite store and NO temp dir, while the full profile
3906    /// over an equally-rooted factory DOES create exactly one store directory.
3907    ///
3908    /// The injected root is the only place an ephemeral store directory can appear
3909    /// (the recording factory roots its stores there), so "the root stays empty on
3910    /// the front-door branch" is a real negative assertion: a regression that gave
3911    /// the front door a store would land its directory here and fail this test. The
3912    /// full-profile arm is the positive control proving the seam genuinely
3913    /// constructs a store when the profile asks for one.
3914    #[test]
3915    fn worker_front_door_builds_no_store_and_no_temp_dir() {
3916        let front_door_root = tempfile::tempdir().expect("test can create an ephemeral root");
3917        let full_root = tempfile::tempdir().expect("test can create an ephemeral root");
3918
3919        let front_door_subsystems = RecordingSubsystems::rooted(front_door_root.path());
3920        let front_door: std::sync::Arc<dyn ConnectionServices> = build_connection_services_via(
3921            &config_with_profile("worker-front-door"),
3922            &front_door_subsystems,
3923        )
3924        .expect("worker-front-door services build");
3925        assert!(
3926            !front_door.supports_channel_operations(),
3927            "the worker front door serves no channel operations"
3928        );
3929        assert_eq!(
3930            entry_count(front_door_root.path()),
3931            0,
3932            "the worker front door creates no ephemeral store directory (no haematite, no temp dir)"
3933        );
3934
3935        let full_subsystems = RecordingSubsystems::rooted(full_root.path());
3936        let mut config = config_with_profile("full");
3937        config.persistence_path = Some(full_root.path().to_path_buf());
3938        let full =
3939            build_connection_services_via(&config, &full_subsystems).expect("full services build");
3940        assert!(
3941            full.supports_channel_operations(),
3942            "full mode serves channel operations"
3943        );
3944        assert_eq!(
3945            entry_count(full_root.path()),
3946            1,
3947            "full mode builds its persistent database in the explicit directory"
3948        );
3949
3950        drop(front_door);
3951        drop(full);
3952    }
3953
3954    /// §9 D2 front-door construction gate (thread half — record-by-construction
3955    /// census): the worker profile constructs NO channel-supervisor,
3956    /// conversation-supervisor, or haematite scheduler, while the SAME instrument
3957    /// over the full profile records all three — the positive control proving the
3958    /// census detects the extra schedulers, so an empty census on the worker branch
3959    /// is a real observation, not a decoration.
3960    ///
3961    /// The instrument's boundary: recording happens INSIDE the [`SubsystemFactory`]
3962    /// methods that are the profile-aware path's only route to these constructors,
3963    /// so a recording cannot be silently omitted — a future subsystem added to this
3964    /// path either goes through the factory (and is recorded by construction) or
3965    /// bypasses it, which is a code-review-visible structural violation of the
3966    /// factory seam, not a silently-missing side call. The connection supervisor's
3967    /// own scheduler is the shared baseline of both profiles and is asserted at the
3968    /// supervisor level (`supervisor::tests`); an OS-level thread census upgrades
3969    /// this when the beamr composition lane's scheduler-inventory API (currently on
3970    /// their branch, not yet consumable from liminal) lands.
3971    #[test]
3972    fn worker_profile_census_is_empty_and_full_profile_records_all_schedulers() {
3973        let worker_root = tempfile::tempdir().expect("test can create an ephemeral root");
3974        let full_root = tempfile::tempdir().expect("test can create an ephemeral root");
3975
3976        let worker_subsystems = RecordingSubsystems::rooted(worker_root.path());
3977        let front_door = build_connection_services_via(
3978            &config_with_profile("worker-front-door"),
3979            &worker_subsystems,
3980        )
3981        .expect("worker-front-door services build");
3982        assert_eq!(
3983            worker_subsystems.recorded(),
3984            Vec::<SchedulerSubsystem>::new(),
3985            "the worker front door constructs no scheduler-owning subsystem"
3986        );
3987
3988        let full_subsystems = RecordingSubsystems::rooted(full_root.path());
3989        let mut config = config_with_profile("full");
3990        config.persistence_path = Some(full_root.path().to_path_buf());
3991        let full =
3992            build_connection_services_via(&config, &full_subsystems).expect("full services build");
3993        assert_eq!(
3994            full_subsystems.recorded(),
3995            vec![
3996                SchedulerSubsystem::ChannelSupervisor,
3997                SchedulerSubsystem::ConversationSupervisor,
3998                SchedulerSubsystem::HaematiteStore,
3999            ],
4000            "the full profile constructs every scheduler-owning subsystem, once each — \
4001             the positive control proving the census instrument detects them"
4002        );
4003
4004        drop(front_door);
4005        drop(full);
4006    }
4007
4008    /// MAJOR-1 regression: the full-only constructors reject a worker-front-door
4009    /// config with a typed `ConfigValidation` error AT ENTRY — no full service can
4010    /// be created through any public config-based constructor under that profile.
4011    #[test]
4012    fn full_only_constructors_reject_worker_profile() {
4013        let config = config_with_profile("worker-front-door");
4014
4015        let from_config = LiminalConnectionServices::from_config(&config);
4016        assert!(
4017            matches!(from_config, Err(ServerError::ConfigValidation { .. })),
4018            "from_config must reject a worker-front-door profile with ConfigValidation, got {from_config:?}"
4019        );
4020
4021        let root = tempfile::tempdir().expect("test can create an ephemeral root");
4022        let store = open_ephemeral_rooted(root.path(), DEFAULT_SHARD_COUNT)
4023            .expect("test store for the rejection check builds");
4024        let from_config_with_store =
4025            LiminalConnectionServices::from_config_with_store(&config, std::sync::Arc::new(store));
4026        assert!(
4027            matches!(
4028                from_config_with_store,
4029                Err(ServerError::ConfigValidation { .. })
4030            ),
4031            "from_config_with_store must reject a worker-front-door profile with ConfigValidation"
4032        );
4033    }
4034
4035    /// MAJOR-1 regression: the profile-aware factory itself re-runs the
4036    /// worker-front-door cross-field checks, so a directly-constructed config (one
4037    /// that never passed file-loading validation) combining the worker profile with
4038    /// full-only machinery is refused with the same typed `ConfigValidation` errors.
4039    #[test]
4040    fn build_connection_services_rejects_worker_profile_with_full_only_fields() {
4041        let mut config = config_with_profile("worker-front-door");
4042        config.channels = vec![crate::config::types::ChannelDef {
4043            name: "orders".to_owned(),
4044            schema_ref: None,
4045            durable: false,
4046            loaded_schema: None,
4047        }];
4048        config.persistence_path = Some(std::path::PathBuf::from("/tmp"));
4049
4050        let result = build_connection_services(&config);
4051        let Err(ServerError::ConfigValidation { message }) = result else {
4052            panic!("expected ConfigValidation for worker profile with full-only fields");
4053        };
4054        assert!(message.contains("builds no channels"), "got: {message}");
4055        assert!(
4056            message.contains("builds no durable store"),
4057            "got: {message}"
4058        );
4059    }
4060
4061    /// The required-path constructor creates the persistent database in its
4062    /// configured directory without populating the separate throwaway root.
4063    #[test]
4064    fn persistent_store_uses_configured_path_and_creates_no_temp_dir() {
4065        let home = tempfile::tempdir().expect("test can create a temp dir");
4066        let ephemeral_root = tempfile::tempdir().expect("test can create an ephemeral root");
4067
4068        let store = build_durable_store(home.path()).expect("persistent store builds");
4069
4070        assert!(
4071            home.path().join("durability").join("config.json").exists(),
4072            "the persistent database is created under the configured path"
4073        );
4074        assert_eq!(
4075            entry_count(ephemeral_root.path()),
4076            0,
4077            "the persistent branch creates no ephemeral guard directory"
4078        );
4079
4080        drop(store);
4081    }
4082
4083    #[test]
4084    fn missing_path_refuses_before_any_store_is_opened() -> Result<(), Box<dyn std::error::Error>> {
4085        let root = tempfile::tempdir()?;
4086        let factory = RecordingSubsystems::rooted(root.path());
4087        let result = build_connection_services_via(&config_with_profile("full"), &factory);
4088        assert!(
4089            matches!(result, Err(ServerError::ConfigValidation { message }) if message.contains("no store is opened"))
4090        );
4091        assert_eq!(entry_count(root.path()), 0);
4092        assert!(factory.recorded().is_empty());
4093        Ok(())
4094    }
4095
4096    /// The store migrates itself: a `config.json` written before haematite
4097    /// required `node_cache_budget` opens on the new server with no operator
4098    /// hand, and the file afterwards carries the key — spelled exactly as the
4099    /// create arm spells it. The pre-field store is fabricated by creating a
4100    /// current one and deleting the key, which is byte-faithful to what a
4101    /// 0.8.1-era store looks like to the parser: the key absent, everything
4102    /// else in place.
4103    #[test]
4104    fn a_store_whose_config_predates_the_budget_field_opens_with_no_operator_hand() {
4105        let home = tempfile::tempdir().expect("test can create a temp dir");
4106        let store = build_durable_store(home.path()).expect("persistent store builds");
4107        drop(store);
4108
4109        let config_file = home.path().join("durability").join("config.json");
4110        let bytes = std::fs::read(&config_file).expect("config.json is readable");
4111        let mut config: serde_json::Value =
4112            serde_json::from_slice(&bytes).expect("config.json parses");
4113        config
4114            .as_object_mut()
4115            .expect("config.json is an object")
4116            .remove("node_cache_budget")
4117            .expect("the created config carries the budget to remove");
4118        std::fs::write(
4119            &config_file,
4120            serde_json::to_vec_pretty(&config).expect("serializes"),
4121        )
4122        .expect("test can age the config");
4123
4124        let reopened = build_durable_store(home.path())
4125            .expect("a pre-budget store opens with no operator action");
4126        drop(reopened);
4127
4128        let after = std::fs::read(&config_file).expect("config.json is readable after");
4129        let migrated: serde_json::Value =
4130            serde_json::from_slice(&after).expect("migrated config parses");
4131        assert_eq!(
4132            migrated.get("node_cache_budget"),
4133            Some(&serde_json::Value::String("unlimited".to_owned())),
4134            "the migrated file carries the budget the create arm chooses"
4135        );
4136    }
4137
4138    /// The migration touches only the one migratable case. A corrupt
4139    /// `config.json` still refuses — through haematite's own parse refusal,
4140    /// which is sharper than anything this crate could say — and the bytes are
4141    /// left exactly as found, so the operator diagnoses the file that failed,
4142    /// not one the migration half-rewrote.
4143    #[test]
4144    fn a_corrupt_config_still_refuses_and_is_left_untouched() {
4145        let home = tempfile::tempdir().expect("test can create a temp dir");
4146        let data_dir = home.path().join("durability");
4147        std::fs::create_dir_all(&data_dir).expect("test can create the data dir");
4148        let config_file = data_dir.join("config.json");
4149        let garbage = b"{ not json at all";
4150        std::fs::write(&config_file, garbage).expect("test can write garbage");
4151
4152        let refused = build_durable_store(home.path());
4153        assert!(refused.is_err(), "a corrupt config refuses to open");
4154        assert_eq!(
4155            std::fs::read(&config_file)
4156                .expect("config.json is still readable")
4157                .as_slice(),
4158            garbage,
4159            "the refusing file is left byte-identical for the operator to read"
4160        );
4161    }
4162
4163    /// A config already carrying the budget is not rewritten: reopening leaves
4164    /// the file byte-identical, so the migration cannot churn mtimes, reorder
4165    /// keys, or otherwise touch a store that needs nothing.
4166    #[test]
4167    fn a_config_already_carrying_the_budget_is_left_byte_identical() {
4168        let home = tempfile::tempdir().expect("test can create a temp dir");
4169        let store = build_durable_store(home.path()).expect("persistent store builds");
4170        drop(store);
4171
4172        let config_file = home.path().join("durability").join("config.json");
4173        let before = std::fs::read(&config_file).expect("config.json is readable");
4174        let reopened = build_durable_store(home.path()).expect("a current store reopens");
4175        drop(reopened);
4176        assert_eq!(
4177            std::fs::read(&config_file).expect("config.json is readable after"),
4178            before,
4179            "a config that already carries the budget is left byte-identical"
4180        );
4181    }
4182
4183    /// A durable store created under `KeepAll` is moved onto reclaim by the
4184    /// server's next open: its `config.json` names `reclaim` afterwards, and the
4185    /// copies its old commits left are gone once reclaim has run.
4186    #[test]
4187    fn a_keep_all_durable_store_reclaims_after_the_next_open() {
4188        let dir = tempfile::tempdir().expect("test can create a store dir");
4189        let data_dir = dir.path().join("durability");
4190        let database = haematite::Database::create(
4191            haematite::DatabaseConfig {
4192                data_dir: data_dir.clone(),
4193                shard_count: 1,
4194                distributed: None,
4195                executor_threads: None,
4196                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
4197                retention: haematite::Retention::KeepAll,
4198            },
4199            haematite::RuntimeOptions {
4200                long_wait_interval: std::time::Duration::from_secs(5),
4201                reclaim_start: haematite::ReclaimStart::AtOpen,
4202            },
4203        )
4204        .expect("a keep-all store is created");
4205        for round in 0_u32..200 {
4206            let mut value = round.to_be_bytes().to_vec();
4207            value.resize(1_024, 0x5a);
4208            database.put(b"key".to_vec(), value).expect("put");
4209            database.commit().expect("commit");
4210        }
4211        drop(database);
4212        let grown =
4213            haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir.clone()))
4214                .expect("stats of the grown store")
4215                .totals
4216                .enumerated_bytes;
4217
4218        let reopened = super::open_or_create_database(&data_dir).expect("the server opens it");
4219        // The server holds reclaim at open and releases it once its listeners
4220        // bind (DUR-021 R1); the test releases it the same way.
4221        reopened
4222            .release_reclaim()
4223            .expect("the held reclaim is released");
4224        reopened.quiesce_reclaim().expect("reclaim runs to rest");
4225        drop(reopened);
4226
4227        let config: serde_json::Value = serde_json::from_slice(
4228            &std::fs::read(data_dir.join("config.json")).expect("config is readable"),
4229        )
4230        .expect("config is JSON");
4231        assert_eq!(
4232            config["retention"],
4233            serde_json::to_value(super::DURABLE_RETENTION).expect("the retention serializes")
4234        );
4235        let report = haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir))
4236            .expect("stats of the reclaimed store");
4237        assert!(
4238            report.totals.enumerated_bytes <= report.totals.marked_bytes.saturating_mul(3),
4239            "the reopened store holds about its live set: enumerated {} B, marked {} B",
4240            report.totals.enumerated_bytes,
4241            report.totals.marked_bytes
4242        );
4243        assert!(
4244            report.totals.enumerated_bytes.saturating_mul(10) < grown,
4245            "the old copies are gone: {} B now against {grown} B grown",
4246            report.totals.enumerated_bytes
4247        );
4248    }
4249
4250    /// `config.json` exactly as a store grown by an engine that never reclaimed
4251    /// carries it: `reclaim` with `past_roots` 0 is already written, so the
4252    /// open has no retention to adopt (it does move the format to version 3),
4253    /// and every superseded copy is still on disk.
4254    /// `DATA_DIR` stands for the one field the fixture rewrites.
4255    const STORED_RECLAIM_CONFIG: &str = r#"{
4256  "chunking_policy": {
4257    "chunking": "bytes-v2",
4258    "internal_target_bytes": 49152,
4259    "leaf_target_bytes": 65536
4260  },
4261  "data_dir": "DATA_DIR",
4262  "distributed": null,
4263  "executor_threads": null,
4264  "format_version": 2,
4265  "node_cache_budget": "unlimited",
4266  "retention": {
4267    "reclaim": {
4268      "past_roots": 0
4269    }
4270  },
4271  "shard_count": 8
4272}"#;
4273
4274    /// A store whose `config.json` already names reclaim, but whose history was
4275    /// grown with nothing reclaimed, is opened by the server and gives its old
4276    /// copies back with no write and no quiesce. A store with no reclaim
4277    /// record owes a startup pass, and a shard runs it on its first turn. The
4278    /// seed holds both shapes the pass must free: one row overwritten many
4279    /// times, and rows written and then deleted, as a prune leaves them.
4280    #[test]
4281    fn a_stored_reclaim_store_grown_without_reclaim_gives_its_old_copies_back_on_open() {
4282        let dir = tempfile::tempdir().expect("test can create a store dir");
4283        let data_dir = dir.path().join("durability");
4284        let database = haematite::Database::create(
4285            haematite::DatabaseConfig {
4286                data_dir: data_dir.clone(),
4287                shard_count: 8,
4288                distributed: None,
4289                executor_threads: None,
4290                node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
4291                retention: haematite::Retention::KeepAll,
4292            },
4293            haematite::RuntimeOptions {
4294                long_wait_interval: std::time::Duration::from_secs(5),
4295                reclaim_start: haematite::ReclaimStart::AtOpen,
4296            },
4297        )
4298        .expect("a keep-all store is created");
4299        for round in 0_u32..200 {
4300            let mut value = round.to_be_bytes().to_vec();
4301            value.resize(1_024, 0x5a);
4302            database.put(b"key".to_vec(), value).expect("put");
4303            database.commit().expect("commit");
4304        }
4305        for row in 0_u32..200 {
4306            let key = [b"pruned-".as_slice(), &row.to_be_bytes()].concat();
4307            database.put(key, vec![0xa5; 1_024]).expect("put");
4308            database.commit().expect("commit");
4309        }
4310        for row in 0_u32..200 {
4311            let key = [b"pruned-".as_slice(), &row.to_be_bytes()].concat();
4312            database.delete(key).expect("delete");
4313        }
4314        database.commit().expect("commit");
4315        drop(database);
4316        let grown_report =
4317            haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir.clone()))
4318                .expect("stats of the grown store");
4319        let grown = grown_report.totals;
4320        assert!(
4321            grown.unmarked_nodes > 0,
4322            "the seed leaves copies no root reaches"
4323        );
4324        let mut owed: std::collections::BTreeSet<usize> = grown_report
4325            .shards
4326            .iter()
4327            .filter(|shard| shard.nodes.unmarked_nodes > 0)
4328            .map(|shard| shard.shard_id)
4329            .collect();
4330
4331        let config_file = data_dir.join("config.json");
4332        let seeded = STORED_RECLAIM_CONFIG.replace(
4333            "DATA_DIR",
4334            data_dir.to_str().expect("the test dir is UTF-8"),
4335        );
4336        std::fs::write(&config_file, &seeded).expect("config.json is written");
4337
4338        let reopened = super::open_or_create_database(&data_dir).expect("the server opens it");
4339        let watch = reopened
4340            .watch_reclaim_passes(64)
4341            .expect("every shard accepts a pass watch");
4342        // Released as the server releases it once its listeners bind (DUR-021 R1).
4343        reopened
4344            .release_reclaim()
4345            .expect("the held reclaim is released");
4346        let mut unmarked_by_shard = std::collections::BTreeMap::new();
4347        while !owed.is_empty() {
4348            let report = watch
4349                .recv_timeout(std::time::Duration::from_secs(30))
4350                .expect("each pass the open owes ends without a write or a quiesce");
4351            assert_eq!(
4352                report.outcome,
4353                haematite::ReclaimPassOutcome::Passed,
4354                "the pass the open owed on shard {} finished",
4355                report.shard_id
4356            );
4357            if owed.remove(&report.shard_id) {
4358                unmarked_by_shard.insert(report.shard_id, report.unmarked_bytes);
4359            }
4360        }
4361        drop(reopened);
4362
4363        let after = haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir))
4364            .expect("stats of the reclaimed store")
4365            .totals;
4366        eprintln!(
4367            "grown {} B, after {} B, {} unreachable nodes seeded, unmarked bytes by shard {unmarked_by_shard:?}",
4368            grown.enumerated_bytes, after.enumerated_bytes, grown.unmarked_nodes
4369        );
4370        assert!(
4371            after.enumerated_bytes.saturating_mul(10) < grown.enumerated_bytes,
4372            "the old copies are gone: {} B now against {} B grown",
4373            after.enumerated_bytes,
4374            grown.enumerated_bytes
4375        );
4376        let config: serde_json::Value = serde_json::from_slice(
4377            &std::fs::read(&config_file).expect("config.json is readable after"),
4378        )
4379        .expect("config is JSON");
4380        assert_eq!(
4381            config["retention"],
4382            serde_json::to_value(super::DURABLE_RETENTION).expect("the retention serializes"),
4383            "config.json names reclaim after the open"
4384        );
4385    }
4386}