liminal_server/server/connection/services.rs
1use std::collections::HashMap;
2use std::path::{Path, PathBuf};
3use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
4use std::sync::{Arc, Mutex, OnceLock, RwLock};
5use std::time::Instant;
6
7use haematite::{CommitPolicy, Database, DatabaseConfig, EventStore, Retention};
8use liminal::channel::{
9 ChannelConfig, ChannelDelivery, ChannelHandle, ChannelMode, ChannelSupervisor, Schema,
10};
11use liminal::conversation::{ConversationSupervisor, ParticipantBehaviour};
12use liminal::durability::bridge::block_on;
13#[cfg(any(test, feature = "test-support"))]
14use liminal::durability::open_ephemeral;
15use liminal::durability::{
16 CountingStore, DedupCache, DedupDecision, DurableStore, EphemeralRoot, EphemeralSweepCounts,
17 HaematiteStore, ProcessingReceipt, StoreCountsHandle,
18};
19use liminal::protocol::{MessageEnvelope, ProtocolError, SchemaId as ProtocolSchemaId};
20use std::num::NonZeroU64;
21
22use super::channel_registry::{
23 ChannelAccessError, ChannelBuildError, ChannelConfigField, ChannelDescriptor, ChannelOrigin,
24 ChannelRegistration, ChannelRegistryError, ChannelState, ChannelStatus, MAX_CHANNELS_KEY,
25 Registered, STATE_ACTIVE, STATE_QUIESCED, UNRECORDED_QUIESCE_REASON,
26};
27use super::conversation::ConnectionConversation;
28use super::services_cluster::build_channel_cluster;
29use super::services_schema::{ChannelSchema, resolve_channel_schema, resolve_schema_bytes};
30use super::worker_front_door::WorkerFrontDoorServices;
31use crate::ServerError;
32use crate::config::types::{ClusterConfig, ParticipantConfig, ServerConfig, ServiceProfile};
33use crate::config::validation::required_persistence_path;
34use crate::health::reissue::OperatorCredentialReissuer;
35use crate::health::unloadable::UnloadableConversationRecord;
36use crate::server::participant::{
37 InstalledParticipantService, ParticipantWarm, ProductionParticipantHandler,
38};
39use crate::server::self_streams::events::PublishOrigin;
40use crate::server::self_streams::sink::OriginPublication;
41use crate::server::signals::ServerSignals;
42
43pub use super::services_cluster::ChannelCluster;
44
45#[path = "services_conversation.rs"]
46mod services_conversation;
47
48/// Registry of custom conversation responders, keyed by conversation subject.
49///
50/// A registered [`ParticipantBehaviour`] becomes the participant for any
51/// conversation opened on its subject; an unregistered subject has no local server.
52type ResponderRegistry = HashMap<String, Arc<dyn ParticipantBehaviour>>;
53
54/// Marker for resources retained by a connection process until unsubscribe.
55pub trait SubscriptionResource: std::fmt::Debug + Send {
56 /// Releases the library subscription resource.
57 ///
58 /// # Errors
59 /// Returns [`ServerError`] when the liminal library reports an unsubscribe failure.
60 fn unsubscribe(self: Box<Self>) -> Result<(), ServerError>;
61
62 /// Attempts to pull the next delivered envelope from the wrapped library
63 /// subscription without blocking.
64 ///
65 /// Returns `None` when the subscriber inbox is empty (or momentarily
66 /// unavailable): the connection process is the delivery pump, so a transient
67 /// empty read is simply "nothing to deliver this slice", never an error.
68 fn try_next(&mut self) -> Option<liminal::envelope::Envelope>;
69
70 /// Non-consuming availability query for the post-arm race barrier.
71 fn has_pending(&self) -> bool;
72
73 /// Whether an overflow has marked this subscription for shedding (§5). The
74 /// delivery pump sheds an overflowed subscription with a typed error frame.
75 /// Defaulted to `false`: a resource with no bounded inbox never overflows.
76 fn is_overflowed(&self) -> bool {
77 false
78 }
79
80 /// The library subscriber's pid, the id a recording publish names it by.
81 /// Defaulted to `None`: a resource with no library subscriber is never a
82 /// recipient.
83 fn subscriber_pid(&self) -> Option<u64> {
84 None
85 }
86
87 /// Reads a delivery and its durable position without acknowledging it.
88 ///
89 /// # Errors
90 /// The underlying subscription cannot be read.
91 fn try_next_positioned(
92 &mut self,
93 ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
94 Ok(self.try_next().map(|envelope| (envelope, None)))
95 }
96
97 /// Checkpoints a delivery after its consumer settles it.
98 ///
99 /// # Errors
100 /// The resource has no durable acknowledgement path, or checkpointing failed.
101 fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
102 Err(ServerError::ListenerAccept {
103 message: format!("subscription cannot acknowledge durable position {position}"),
104 })
105 }
106
107 /// Reads the subscription-owned durable checkpoint.
108 ///
109 /// # Errors
110 /// The checkpoint cannot be read.
111 fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
112 Ok(None)
113 }
114}
115
116/// Library subscription resource owned by a single connection process.
117#[derive(Debug)]
118pub struct ConnectionSubscription {
119 id: u64,
120 /// Channel this subscription was opened on, carried for DIAGNOSTICS only —
121 /// the shed warning has to name the channel or an operator reading it learns
122 /// that something was shed without learning what. Empty when unset (the
123 /// scheduler-free unit fixtures, which construct through `new` and never log).
124 channel: String,
125 /// Client-chosen application stream the server delivers this subscription's
126 /// messages on (echoed on `SubscribeAck`, carried on every `Deliver`). Set by
127 /// the connection process from the `Subscribe` frame before the subscription is
128 /// stored; `0` only while momentarily unset during construction.
129 stream_id: u32,
130 selected_schema: ProtocolSchemaId,
131 resource: Box<dyn SubscriptionResource>,
132 issued: super::state::IssuedDeliveries,
133}
134
135impl ConnectionSubscription {
136 /// Creates an owned subscription resource for one connection process.
137 #[must_use]
138 pub fn new(
139 id: u64,
140 selected_schema: ProtocolSchemaId,
141 resource: Box<dyn SubscriptionResource>,
142 ) -> Self {
143 Self {
144 id,
145 channel: String::new(),
146 stream_id: 0,
147 selected_schema,
148 resource,
149 issued: super::state::IssuedDeliveries::new(0),
150 }
151 }
152
153 /// Records the channel this subscription was opened on (diagnostics).
154 ///
155 /// Additive setter rather than a fourth `new` parameter: `new` is public API
156 /// and its existing callers keep compiling byte-for-byte.
157 pub(super) fn set_channel(&mut self, channel: String) {
158 self.channel = channel;
159 }
160
161 /// The channel this subscription was opened on, or `""` when unset.
162 #[must_use]
163 pub(super) fn channel(&self) -> &str {
164 &self.channel
165 }
166
167 /// Returns the protocol subscription id.
168 #[must_use]
169 pub const fn id(&self) -> u64 {
170 self.id
171 }
172
173 /// Records the client-chosen delivery stream id for this subscription.
174 pub(super) const fn set_stream_id(&mut self, stream_id: u32) {
175 self.stream_id = stream_id;
176 }
177
178 /// Returns the client-chosen application stream id deliveries ride on.
179 #[must_use]
180 pub(super) const fn stream_id(&self) -> u32 {
181 self.stream_id
182 }
183
184 pub(super) fn set_pressure_ledger_depth(&mut self, limit: usize) {
185 self.issued = super::state::IssuedDeliveries::new(limit);
186 }
187
188 pub(super) fn issued_delivery(&mut self, sequence: u64) {
189 self.issued.record(sequence);
190 }
191
192 pub(super) fn match_pressure(&mut self, reference: &str, retain: bool) -> bool {
193 self.issued.matches(reference, retain)
194 }
195
196 /// Returns the schema selected for this subscription stream.
197 #[must_use]
198 pub const fn selected_schema(&self) -> ProtocolSchemaId {
199 self.selected_schema
200 }
201
202 /// The library subscriber's pid, when the subscription has one.
203 #[must_use]
204 pub fn subscriber_pid(&self) -> Option<u64> {
205 self.resource.subscriber_pid()
206 }
207
208 /// Attempts to pull the next delivered envelope without blocking.
209 pub(crate) fn try_next(&mut self) -> Option<liminal::envelope::Envelope> {
210 self.resource.try_next()
211 }
212
213 pub(crate) fn has_pending(&self) -> bool {
214 self.resource.has_pending()
215 }
216
217 /// Whether this subscription has been shed by an inbox overflow (§5).
218 pub(crate) fn is_overflowed(&self) -> bool {
219 self.resource.is_overflowed()
220 }
221
222 pub(super) fn unsubscribe(self) -> Result<(), ServerError> {
223 self.resource.unsubscribe()
224 }
225
226 pub(crate) fn try_next_positioned(
227 &mut self,
228 ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
229 self.resource.try_next_positioned()
230 }
231
232 pub(crate) fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
233 self.resource.acknowledge_position(position)
234 }
235
236 pub(crate) fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
237 self.resource.consumer_position()
238 }
239}
240
241/// Outcome of a server publish.
242///
243/// Carries the assigned message id plus a genuine delivery ack (`delivered` = the
244/// message was accepted by at least one live subscriber on this publish, after any
245/// dedup-on-delivery suppression).
246#[derive(Clone, Copy, Debug, PartialEq, Eq)]
247pub struct PublishOutcome {
248 /// Transport-scoped message id assigned to the accepted publish.
249 ///
250 /// TRANSPORT-SCOPED, NOT A DURABLE POSITION (G1,
251 /// `docs/design/wire-durability-gaps-20260831.md`): this is a
252 /// process-local counter (`next_message_id`, reset to 1 on every
253 /// construction), so it restarts from 1 with the server and two servers
254 /// mint the same ids. It is monotonic only within one service instance's
255 /// lifetime and identifies the publish on THIS connection's wire, nothing
256 /// more. The durable position of a publish on a durable channel is the
257 /// store-assigned sequence the library now surfaces as
258 /// `ChannelDelivery::durable_position`; putting that on the wire ack is
259 /// the A1 resume lane's frame decision, not a reinterpretation of this
260 /// field.
261 pub message_id: u64,
262 /// Whether the message was genuinely delivered to a subscriber. `false` means
263 /// the publish was accepted but reached no subscriber (empty channel) or was
264 /// a duplicate suppressed by dedup-on-delivery.
265 pub delivered: bool,
266 /// Store-assigned position of the row this outcome answers for, when the
267 /// publish is known to have one. Unlike `message_id` this IS a durable
268 /// position; the connection process puts it on the wire ack only for a
269 /// connection that negotiated `PUBLISH_ACK_POSITION_VERSION`.
270 pub durable_position: Option<u64>,
271 /// Whom the publish reached, the console tap's word for it (SRV-012
272 /// R10): a subscriber, no subscriber, only subscribers that refused it,
273 /// or nobody because dedup suppressed it.
274 pub reach: Reach,
275}
276
277/// Whom a publish reached (SRV-012 R10).
278#[derive(Clone, Copy, Debug, PartialEq, Eq)]
279pub enum Reach {
280 /// At least one live subscriber took it.
281 Delivered,
282 /// The channel had no matching subscriber.
283 NoSubscriber,
284 /// Every matching subscriber refused it.
285 NoneAccepted,
286 /// A dedup-suppressed re-publish, sent to nobody.
287 Suppressed,
288}
289
290impl Reach {
291 /// The reach of a library delivery.
292 #[must_use]
293 pub const fn of(delivery: &ChannelDelivery) -> Self {
294 if delivery.is_delivered() {
295 Self::Delivered
296 } else if delivery.matched_count() == 0 {
297 Self::NoSubscriber
298 } else {
299 Self::NoneAccepted
300 }
301 }
302}
303
304/// Which channel operation is asking the roster for admission.
305///
306/// `Copy`, so the hot path allocates nothing to ask. The contrast is
307/// [`ServerError::UnsupportedOperation`]'s owned `operation: String`: that one is
308/// built on a refusal path and pays for its allocation once per refusal, while
309/// this one is consulted on every publish and every subscribe frame that reaches
310/// the connection process.
311#[derive(Clone, Copy, Debug, PartialEq, Eq)]
312pub enum ChannelOperation {
313 /// A `Publish` frame is asking.
314 Publish,
315 /// A `Subscribe` frame is asking.
316 Subscribe,
317}
318
319/// Operations that adapt wire frames to liminal library calls.
320pub trait ConnectionServices: std::fmt::Debug + Send + Sync {
321 /// Returns the complete participant service installed on this adapter.
322 ///
323 /// `None` keeps participant capability disabled even when the adapter owns a
324 /// durable store for unrelated channel traffic. The returned token is
325 /// server-sealed and atomically carries declared semantics plus durability,
326 /// making a handler-without-store activation impossible by construction.
327 fn participant_service(&self) -> Option<InstalledParticipantService> {
328 None
329 }
330
331 /// Delegates a publish request to the liminal library.
332 ///
333 /// `connection` is the publishing connection's id (its process pid, as
334 /// `apply_frame` knows it), carried as the publish origin so the console
335 /// tap attributes the message to it (SRV-012). `idempotency_key`, when `Some`, drives dedup-on-delivery: a re-publish with
336 /// the same key is delivered to subscribers at most once. The returned
337 /// [`PublishOutcome`] carries the genuine delivery ack.
338 ///
339 /// # Errors
340 /// Returns [`ServerError`] when the liminal publish operation fails.
341 fn publish(
342 &self,
343 connection: u64,
344 channel: &str,
345 envelope: &MessageEnvelope,
346 idempotency_key: Option<&str>,
347 ) -> Result<PublishOutcome, ServerError>;
348
349 /// Delegates a subscribe request to the liminal library.
350 ///
351 /// `install`, when `Some`, carries the connection's §5 shared inbox byte
352 /// budget, per-inbox fairness cap, and R3 wake notifier. The implementation
353 /// MUST install it on the subscription's inbox BEFORE the registration is
354 /// published to the channel actor (i.e. before any envelope can be
355 /// delivered), so no envelope is ever admitted uncharged, past the depth
356 /// cap, or without a wake. Implementations with no real inbox (test
357 /// stand-ins, capability-scoped profiles that refuse subscribe) may ignore
358 /// it.
359 ///
360 /// # Errors
361 /// Returns [`ServerError`] when the liminal subscribe operation fails.
362 fn subscribe(
363 &self,
364 channel: &str,
365 accepted_schemas: &[ProtocolSchemaId],
366 install: Option<liminal::channel::InboxInstall>,
367 ) -> Result<ConnectionSubscription, ServerError>;
368
369 /// Delegates unsubscribe to the liminal library.
370 ///
371 /// # Errors
372 /// Returns [`ServerError`] when the liminal unsubscribe operation fails.
373 fn unsubscribe(&self, subscription: ConnectionSubscription) -> Result<(), ServerError>;
374
375 /// Delegates conversation open to the liminal library.
376 ///
377 /// # Errors
378 /// Returns [`ServerError`] when the liminal conversation open operation fails.
379 fn open_conversation(
380 &self,
381 conversation_id: u64,
382 subject: &str,
383 ) -> Result<ConnectionConversation, ServerError>;
384
385 /// Delegates a conversation message to the liminal library.
386 ///
387 /// # Errors
388 /// Returns [`ServerError`] when the liminal conversation message operation fails.
389 fn conversation_message(
390 &self,
391 conversation: &ConnectionConversation,
392 envelope: &MessageEnvelope,
393 op_id: Option<u64>,
394 ) -> Result<(), ServerError>;
395
396 /// Delegates conversation close to the liminal library.
397 ///
398 /// # Errors
399 /// Returns [`ServerError`] when the liminal conversation close operation fails.
400 fn close_conversation(&self, conversation: ConnectionConversation) -> Result<(), ServerError>;
401
402 /// Flushes the shared durable store after channel publications have settled.
403 ///
404 /// # Errors
405 /// Returns [`ServerError`] when the shared store flush operation fails.
406 fn flush_durable_state(&self) -> Result<(), ServerError>;
407
408 /// Stops the participant warm and records the clean stop's capacity
409 /// snapshot (DUR-020 R3, R5). Called once, by the shutdown sequence,
410 /// before the final flush; an adapter with no participant does nothing.
411 fn stop_participant_warm(&self) {}
412
413 /// The listener is bound: whatever the start reads that no admission
414 /// needs before it is judged runs from here, behind the listening door.
415 /// An adapter with nothing to read does nothing. Answers the thread it
416 /// started, which the listener leaves to run and a test may join.
417 fn listening(self: Arc<Self>) -> Option<std::thread::JoinHandle<()>> {
418 None
419 }
420
421 /// Whether this adapter backs ordinary channel and conversation operations.
422 ///
423 /// The default is `true` — the full-service adapter serves publish, subscribe,
424 /// and conversation frames, so full mode is byte-for-byte unchanged. The
425 /// capability-scoped worker front door overrides this to `false`, letting
426 /// [`super::apply`] reject the channel/conversation frames it short-circuits on
427 /// empty connection state (`Unsubscribe`, `ConversationMessage`,
428 /// `ConversationClose`) with a typed error frame instead of silently swallowing
429 /// an operation for a resource that could never have been created in this
430 /// profile. Frames that always reach a service method (`Publish`, `Subscribe`,
431 /// `ConversationOpen`) are rejected by the front door's own method bodies and do
432 /// not consult this flag.
433 fn supports_channel_operations(&self) -> bool {
434 true
435 }
436
437 /// Whether `channel` admits `operation` right now.
438 ///
439 /// Consulted by the connection process BEFORE the operation is delegated, so
440 /// a roster refusal is typed at the moment of the decision, by the component
441 /// that made it, from the value it decided on. The alternative — classifying
442 /// an opaque failure afterwards by re-reading the roster in the error arm —
443 /// cannot tell "the roster refused this" from "something else failed while
444 /// the roster happened to change", and would put a confident wrong cause on
445 /// the wire.
446 ///
447 /// It does NOT replace the adapter's own inner check. Admission here is the
448 /// caller's guard; a service method is public and callable without a frame,
449 /// so it keeps its own.
450 ///
451 /// The default ADMITS. An adapter with no roster has nothing to say here and
452 /// its refusals travel as service errors exactly as they do today; only the
453 /// roster-owning adapter overrides this. A default body is also what keeps
454 /// this addition inside "minor": a method added without one breaks every
455 /// downstream implementor of a public trait.
456 ///
457 /// # Errors
458 /// Returns [`ChannelAccessError`] when the roster refuses the operation.
459 fn admit_channel(
460 &self,
461 operation: ChannelOperation,
462 channel: &str,
463 ) -> Result<(), ChannelAccessError> {
464 let _ = (operation, channel);
465 Ok(())
466 }
467}
468
469/// Default adapter from server wire frames to liminal channel/conversation APIs.
470#[derive(Debug)]
471pub struct LiminalConnectionServices {
472 /// The channel roster, keyed by channel name.
473 ///
474 /// Behind an [`RwLock`] over `Arc` values so a reader clones one pointer out
475 /// under the guard and works with it after the guard is released: no roster
476 /// read is ever held across a library call, and an entry handed out can
477 /// outlive a concurrent mutation of the map itself.
478 channels: RwLock<HashMap<String, Arc<ConfiguredChannel>>>,
479 export_subscriptions:
480 Mutex<Vec<std::sync::Weak<crate::link_bridge::subscriptions::ExportSubscriptions>>>,
481 /// The operator-declared bound on RUNTIME-registered channels
482 /// (`limits.max_channels`), carried verbatim from config.
483 ///
484 /// `None` is "the operator declared no bound", which refuses every runtime
485 /// registration rather than admitting an unbounded one — the roster's
486 /// aggregate size is otherwise unbounded the moment a registration API
487 /// exists. Boot-configured channels are never counted against it: they are
488 /// the bound the operator already wrote, in the file they wrote it in.
489 max_channels: Option<usize>,
490 cluster: ChannelCluster,
491 durable_store: Arc<dyn DurableStore>,
492 /// Reads the counters of every store these services built: the durable
493 /// store and, when it is a separate store, the ephemeral dedup store.
494 store_counts: StoreCountsHandle,
495 ephemeral_sweep: Option<EphemeralSweepCounts>,
496 signals: ServerSignals,
497 /// Complete participant service, installed only when semantic lifecycle
498 /// handling and its durable aggregate store are both ready.
499 participant_service: Option<InstalledParticipantService>,
500 /// The production handler's refused-load record, captured at construction.
501 ///
502 /// [`InstalledParticipantService`] holds its handler behind
503 /// `dyn ParticipantSemanticHandler`, so the concrete production handler is
504 /// no longer reachable once it is installed; the record is taken here, in
505 /// the one place the concrete handler exists, and carried out to the
506 /// startup path that publishes it onto the health endpoint. `None` is "no
507 /// participant is configured", which the operator surface reports as such
508 /// rather than as an empty refusal set.
509 unloadable_conversations: Option<UnloadableConversationRecord>,
510 /// The production handler as the A7 operator credential-re-issue authority
511 /// (§0.18), captured at construction for exactly the reason above: the
512 /// concrete handler is unreachable once erased behind
513 /// `dyn ParticipantSemanticHandler`, so the operator surface's handle on it
514 /// has to be taken in the one place it still exists. `None` is "no
515 /// participant is configured", which the route reports as such.
516 credential_reissuer: Option<Arc<dyn OperatorCredentialReissuer>>,
517 /// The participant warm's stop handle (DUR-020 R4, R5), captured at
518 /// construction for the same reason as the two above. The shutdown flush
519 /// stops the warm first, so no conversation begins restoring after the
520 /// server has begun to stop. `None` is "no participant is configured".
521 participant_warm: Option<ParticipantWarm>,
522 /// Dedup-on-delivery cache for publishes to **durable** channels. Keyed by
523 /// the per-message idempotency key carried on the publish frame; a duplicate
524 /// key is suppressed before fan-out so a subscriber receives it at most once.
525 ///
526 /// The namespace is stable across restarts by design (A1 §5): on a durable
527 /// channel the MESSAGE survives a crash too, so a surviving receipt is
528 /// truthful.
529 dedup: DedupCache,
530 /// Dedup-on-delivery cache for publishes to **ephemeral** channels
531 /// (A1 §5, graft §0.2 — the panel's sharpest crash hole).
532 ///
533 /// Identical machinery, different namespace: this one carries a per-boot
534 /// incarnation id, so a receipt and the message it vouches for share a
535 /// lifetime. Without it, `ephemeral + keyed publish -> Defer (buffered in
536 /// memory) -> complete_receipt (persisted) -> crash before the consumer
537 /// pops` loses the message while the receipt survives to suppress the
538 /// producer's retry — silent permanent loss of something the producer was
539 /// told was admitted. With it, the post-crash retry re-claims and
540 /// re-delivers, which is exactly ephemeral's at-least-once contract.
541 ///
542 /// This changes no dedup API. It changes only the namespace string the
543 /// server composes.
544 dedup_ephemeral: DedupCache,
545 /// Durable-channel idempotency keys whose publish outcome is not yet
546 /// known, held until their row is looked up by its key. A cache of the
547 /// durable in-flight claims, rebuilt from them when the services start.
548 held_keys: unresolved::HeldKeys,
549 /// The store history whose in-flight claims are still to be resolved:
550 /// some until the resolve has run, behind the listening door or at the
551 /// first keyed durable admission, whichever comes first (`unresolved_start.rs`).
552 claims_start: Mutex<Option<StoreHistory>>,
553 conversation_supervisor: Arc<ConversationSupervisor>,
554 /// Registered custom conversation responders, keyed by conversation subject.
555 ///
556 /// When a conversation is opened (`open_conversation`), the subject is looked
557 /// up here: a registered [`ParticipantBehaviour`] becomes the conversation's
558 /// participant; with no registration the local route refuses by name.
559 /// This is the seam aion #13 plugs a remote worker responder into. Interior
560 /// mutability is required because the services are shared behind `&self`.
561 responders: Mutex<ResponderRegistry>,
562 next_message_id: AtomicU64,
563 next_subscription_id: AtomicU64,
564 /// SRV-012: the console's held topology and tap.
565 console: Arc<crate::console::ConsoleHub>,
566}
567
568impl LiminalConnectionServices {
569 /// Builds library-backed services from validated server configuration.
570 ///
571 /// Durable-mode channels are backed by a shared haematite event store so
572 /// their publishes are persisted and survive the graceful-shutdown flush;
573 /// ephemeral channels carry no store.
574 ///
575 /// Full-only: a config selecting the worker-front-door profile is rejected at
576 /// entry (before any store is built), so this constructor can never build full
577 /// services for a profile that forbids them. Profile-aware callers go through
578 /// [`build_connection_services`] instead.
579 ///
580 /// # Errors
581 /// Returns [`ServerError`] when the config selects a non-full profile or a
582 /// configured channel cannot be initialized.
583 pub fn from_config(config: &ServerConfig) -> Result<Self, ServerError> {
584 require_full_profile(config)?;
585 let path = required_persistence_path(config)?;
586 let history = StoreHistory::of(path);
587 let subsystems = RootedProductionSubsystems::new(path)?;
588 let store = subsystems.durable_store(path)?;
589 Self::from_config_with_store_via(config, store, history, &subsystems)
590 }
591
592 /// Builds services over a caller-provided durable store.
593 ///
594 /// Used by tests that need to inspect persisted state through the same store
595 /// handle the durable channels write to.
596 ///
597 /// Full-only: rejects a worker-front-door profile at entry, exactly like
598 /// [`Self::from_config`].
599 ///
600 /// # Errors
601 /// Returns [`ServerError`] when the config selects a non-full profile or a
602 /// configured channel cannot be initialized.
603 pub fn from_config_with_store(
604 config: &ServerConfig,
605 durable_store: Arc<dyn DurableStore>,
606 ) -> Result<Self, ServerError> {
607 require_full_profile(config)?;
608 #[cfg(any(test, feature = "test-support"))]
609 if config.persistence_path.is_none() {
610 return Self::from_config_with_store_via(
611 config,
612 durable_store,
613 StoreHistory::Recorded,
614 &ProductionSubsystems,
615 );
616 }
617 let subsystems = RootedProductionSubsystems::new(required_persistence_path(config)?)?;
618 Self::from_config_with_store_via(config, durable_store, StoreHistory::Recorded, &subsystems)
619 }
620
621 /// [`Self::from_config_with_store`] with the subsystem factory injected.
622 ///
623 /// The channel supervisor and conversation supervisor are constructed ONLY
624 /// through `subsystems` — there is no direct constructor call in this body —
625 /// so a factory that records as a side effect of constructing cannot have its
626 /// recording omitted (§9 D2 seam census, record-by-construction). No profile
627 /// check here: the caller (the public wrapper or the profile dispatch in
628 /// [`build_connection_services`]) has already established the full profile.
629 /// `history` says whether `durable_store` can already hold in-flight
630 /// claims for the services to look up before they judge a publish.
631 fn from_config_with_store_via(
632 config: &ServerConfig,
633 durable_store: Arc<dyn DurableStore>,
634 history: StoreHistory,
635 subsystems: &dyn SubsystemFactory,
636 ) -> Result<Self, ServerError> {
637 // Build ONE shared channel supervisor for the whole server. When a
638 // [cluster] section is present it is distribution-enabled, so every
639 // channel actor and subscriber shares the clustered scheduler the cluster
640 // attaches its process-group transport to (SRV-005, Constraint B).
641 let cluster = subsystems.channel_cluster(config.cluster.as_ref())?;
642 let mut channels = HashMap::new();
643 for channel in &config.channels {
644 // Resolve the channel's real JSON Schema (loaded from `schema_ref`
645 // during config validation) or the permissive empty schema when the
646 // channel declared none. The protocol schema id advertised at
647 // subscribe time is derived from the SAME schema bytes so an SDK
648 // deriving ids from schema bytes converges on it.
649 let resolved = resolve_channel_schema(channel);
650 let configured = build_configured_channel(
651 &channel.name,
652 resolved,
653 channel.durable,
654 ChannelOrigin::BootConfigured,
655 &durable_store,
656 cluster.supervisor(),
657 )
658 .map_err(|error| ServerError::ConfigValidation {
659 message: error.boot_message(&channel.name),
660 })?;
661 channels.insert(channel.name.clone(), Arc::new(configured));
662 }
663 let conversation_supervisor = subsystems.conversation_supervisor()?;
664 let dedup = history.delivery_dedup(Arc::clone(&durable_store));
665 // A1 §5: the ephemeral namespace is scoped to THIS incarnation, so an
666 // ephemeral receipt cannot outlive the in-memory message it vouches for.
667 // No earlier process used it, so it starts empty.
668 let ephemeral_store = ephemeral_dedup_store(&durable_store, history, subsystems);
669 let store_counts = combined_store_counts(&durable_store, &ephemeral_store);
670 let dedup_ephemeral = DedupCache::new_empty(ephemeral_store, ephemeral_dedup_namespace());
671 // Production participant activation (LP gap closure, Part B): the
672 // deployment's [participant] section installs the ONE production
673 // semantic handler, sealed together with the same durable store the
674 // conversation logs live in, under the configured wire-frame limit.
675 // No section, no service — the capability bit stays off and the
676 // connection path is byte-identical to the pre-activation build.
677 let signals = ServerSignals::default();
678 let installed_participant = config
679 .participant
680 .as_ref()
681 .map(|participant| install_participant(&durable_store, participant, &signals))
682 .transpose()?;
683 let device = config
684 .cluster
685 .as_ref()
686 .map_or("local", |cluster| cluster.node_name.as_str());
687 let console = Arc::new(console_hub(
688 device,
689 &channels,
690 config.persistence_path.is_some(),
691 config.auth.as_ref().and_then(|auth| auth.lys.as_ref()),
692 )?);
693 signals.install_console(Arc::clone(&console))?;
694 cluster.observer().set_console(Arc::clone(&console))?;
695 let (participant_service, unloadable_conversations, credential_reissuer, participant_warm) =
696 installed_participant.map_or((None, None, None, None), |installed| {
697 (
698 Some(installed.service),
699 Some(installed.unloadable_conversations),
700 Some(installed.credential_reissuer),
701 Some(installed.warm),
702 )
703 });
704 let services = Self {
705 channels: RwLock::new(channels),
706 export_subscriptions: Mutex::new(Vec::new()),
707 max_channels: config.limits.max_channels,
708 cluster,
709 durable_store,
710 store_counts,
711 ephemeral_sweep: subsystems.ephemeral_sweep_counts(),
712 signals,
713 participant_service,
714 unloadable_conversations,
715 credential_reissuer,
716 participant_warm,
717 dedup,
718 dedup_ephemeral,
719 held_keys: unresolved::HeldKeys::default(),
720 // Every durable in-flight claim is looked up by its key before the
721 // services judge a keyed durable publish: behind the listening
722 // door, or by that publish itself when it comes first.
723 claims_start: Mutex::new(Some(history)),
724 conversation_supervisor,
725 responders: Mutex::new(HashMap::new()),
726 next_message_id: AtomicU64::new(1),
727 next_subscription_id: AtomicU64::new(1),
728 console,
729 };
730 Ok(services)
731 }
732
733 /// Builds services with no configured channels.
734 ///
735 /// # Errors
736 /// Returns [`ServerError`] when the conversation supervisor scheduler cannot start.
737 #[cfg(any(test, feature = "test-support"))]
738 pub fn empty() -> Result<Self, ServerError> {
739 let conversation_supervisor = ProductionSubsystems.conversation_supervisor()?;
740 let store =
741 open_ephemeral(DEFAULT_SHARD_COUNT).map_err(|error| ServerError::ConfigValidation {
742 message: format!("failed to open test durable store: {error}"),
743 })?;
744 let durable_store: Arc<dyn DurableStore> = Arc::new(CountingStore::new(store));
745 let store_counts = combined_store_counts(&durable_store, &durable_store);
746 let dedup = StoreHistory::Fresh.delivery_dedup(Arc::clone(&durable_store));
747 // A1 §5: the ephemeral namespace is scoped to THIS incarnation, so an
748 // ephemeral receipt cannot outlive the in-memory message it vouches for.
749 let dedup_ephemeral =
750 DedupCache::new_empty(Arc::clone(&durable_store), ephemeral_dedup_namespace());
751 let signals = ServerSignals::default();
752 let console = Arc::new(console_hub("local", &HashMap::new(), false, None)?);
753 signals.install_console(Arc::clone(&console))?;
754 let cluster = build_channel_cluster(None)?;
755 cluster.observer().set_console(Arc::clone(&console))?;
756 Ok(Self {
757 channels: RwLock::new(HashMap::new()),
758 export_subscriptions: Mutex::new(Vec::new()),
759 // No config, so no declared bound: this builder serves tests and
760 // callers with no channels at all, and a registration against it
761 // refuses `CapNotConfigured` exactly as an undeclared deployment's
762 // would.
763 max_channels: None,
764 cluster,
765 durable_store,
766 store_counts,
767 ephemeral_sweep: None,
768 signals,
769 participant_service: None,
770 unloadable_conversations: None,
771 credential_reissuer: None,
772 participant_warm: None,
773 dedup,
774 dedup_ephemeral,
775 held_keys: unresolved::HeldKeys::default(),
776 claims_start: Mutex::new(None),
777 conversation_supervisor,
778 responders: Mutex::new(HashMap::new()),
779 next_message_id: AtomicU64::new(1),
780 next_subscription_id: AtomicU64::new(1),
781 console,
782 })
783 }
784
785 /// The retained counts from the root's one startup sweep.
786 #[must_use]
787 pub const fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
788 self.ephemeral_sweep
789 }
790
791 /// The shared channel supervisor + cluster resolver backing this service.
792 ///
793 /// The server runtime uses this to attach the cluster to the channel
794 /// supervisor's clustered scheduler (SRV-005).
795 #[must_use]
796 pub const fn channel_cluster(&self) -> &ChannelCluster {
797 &self.cluster
798 }
799
800 /// Returns the shared durable store backing this service's durable channels.
801 #[must_use]
802 pub fn durable_store(&self) -> Arc<dyn DurableStore> {
803 Arc::clone(&self.durable_store)
804 }
805
806 /// The console's held topology and tap (SRV-012).
807 #[must_use]
808 pub fn console(&self) -> &crate::console::ConsoleHub {
809 &self.console
810 }
811
812 /// Applies a topology change; a poisoned model is counted by the hub
813 /// (`topology_emit_failures`) and logged, never recovered.
814 fn console_changed(&self, change: crate::console::TopologyChange) {
815 self.signals.topology(change);
816 }
817
818 /// Conversation subjects with a registered responder, sorted. Read from
819 /// the in-memory registry; touches no store.
820 ///
821 /// # Errors
822 /// Returns [`ServerError`] when the responder registry lock is poisoned.
823 pub fn responder_subjects(&self) -> Result<Vec<String>, ServerError> {
824 let responders = self.lock_responders()?;
825 let mut subjects: Vec<String> = responders.keys().cloned().collect();
826 drop(responders);
827 subjects.sort();
828 Ok(subjects)
829 }
830
831 /// Restores every participant conversation the warm has not yet reached,
832 /// on the caller's thread, and answers how many refused (DUR-020 R4).
833 /// The store-count journeys call it so their windows do not race the
834 /// warm thread; the server itself never waits for the warm.
835 ///
836 /// # Errors
837 /// Returns [`ServerError`] when the pending-restore lock is poisoned.
838 pub fn finish_participant_warm(&self) -> Result<usize, ServerError> {
839 self.participant_warm.as_ref().map_or(Ok(0), |warm| {
840 warm.finish()
841 .map_err(|error| ServerError::ParticipantStartupRestore {
842 message: error.to_string(),
843 })
844 })
845 }
846
847 /// Reads the store work these services have done since they were built,
848 /// across every store they built.
849 ///
850 /// Empty (every snapshot zero) when the durable store was supplied by the
851 /// caller and does not count.
852 #[must_use]
853 pub fn store_counts(&self) -> StoreCountsHandle {
854 self.store_counts.clone()
855 }
856
857 /// The production participant handler's refused-load record, when a
858 /// participant is configured.
859 ///
860 /// The server's startup path publishes this onto the health endpoint so
861 /// `GET /unloadable-conversations` answers from the same record the
862 /// handler writes. `None` means no participant is configured at all, which
863 /// the surface reports as a distinct state from "nothing was refused".
864 #[must_use]
865 pub fn unloadable_conversation_record(&self) -> Option<UnloadableConversationRecord> {
866 self.unloadable_conversations.clone()
867 }
868
869 /// The A7 operator credential-re-issue authority, when a participant is
870 /// configured (§0.18).
871 ///
872 /// The startup path publishes this onto the health endpoint so
873 /// `POST /operator/credential-reissue` reaches the SAME serialized
874 /// participant-state point every wire operation does. `None` means no
875 /// participant is configured, which the route reports as a distinct state
876 /// from an unknown identity.
877 #[must_use]
878 pub fn credential_reissuer(&self) -> Option<Arc<dyn OperatorCredentialReissuer>> {
879 self.credential_reissuer.clone()
880 }
881
882 /// Installs a complete participant bundle in full-service supervisor tests.
883 ///
884 /// Production full services intentionally stay disabled until a concrete
885 /// lifecycle handler exists. This consuming test builder exercises the real
886 /// supervisor activation path without allowing an already-shared adapter to
887 /// change capability posture.
888 #[cfg(test)]
889 #[must_use]
890 pub(crate) fn with_participant_service(
891 mut self,
892 participant_service: InstalledParticipantService,
893 ) -> Self {
894 self.participant_service = Some(participant_service);
895 self
896 }
897
898 /// Returns the conversation supervisor backing supervised conversations.
899 ///
900 /// Tests use this to reach the underlying beamr scheduler so they can spawn
901 /// or terminate participant processes and exercise crash detection.
902 #[must_use]
903 pub fn conversation_supervisor(&self) -> Arc<ConversationSupervisor> {
904 Arc::clone(&self.conversation_supervisor)
905 }
906
907 /// Registers a conversation responder for a routing `subject`.
908 ///
909 /// A later conversation on this exact subject runs the registered behaviour
910 /// as a supervised linked participant. A registration replaces the previous
911 /// behaviour and returns it to the caller.
912 ///
913 /// # Errors
914 /// Returns [`ServerError`] when the responder registry lock is poisoned.
915 pub fn register_responder(
916 &self,
917 subject: impl Into<String>,
918 behaviour: Arc<dyn ParticipantBehaviour>,
919 ) -> Result<Option<Arc<dyn ParticipantBehaviour>>, ServerError> {
920 let subject = subject.into();
921 let mut responders = self.lock_responders()?;
922 let previous = responders.insert(subject.clone(), behaviour);
923 drop(responders);
924 self.console_changed(crate::console::TopologyChange::Service {
925 subject,
926 present: true,
927 });
928 Ok(previous)
929 }
930
931 /// Removes the custom responder registered for `subject`, if any.
932 ///
933 /// After removal local calls on the subject are refused with `no_server`.
934 /// Returns the removed behaviour when one was registered.
935 ///
936 /// # Errors
937 /// Returns [`ServerError`] when the responder registry lock is poisoned.
938 pub fn unregister_responder(
939 &self,
940 subject: &str,
941 ) -> Result<Option<Arc<dyn ParticipantBehaviour>>, ServerError> {
942 let mut responders = self.lock_responders()?;
943 let removed = responders.remove(subject);
944 drop(responders);
945 if removed.is_some() {
946 self.console_changed(crate::console::TopologyChange::Service {
947 subject: subject.to_owned(),
948 present: false,
949 });
950 }
951 Ok(removed)
952 }
953
954 /// Locks the responder registry, mapping a poisoned lock to a [`ServerError`]
955 /// rather than panicking (the workspace denies `unwrap`/`expect`/`panic`).
956 fn lock_responders(&self) -> Result<std::sync::MutexGuard<'_, ResponderRegistry>, ServerError> {
957 self.responders
958 .lock()
959 .map_err(|_poisoned| ServerError::ListenerAccept {
960 message: "responder registry lock poisoned".to_owned(),
961 })
962 }
963
964 /// Takes the channel roster's read guard, mapping a poisoned lock to a
965 /// [`ServerError`] rather than panicking (the workspace denies
966 /// `unwrap`/`expect`/`panic`), exactly as [`Self::lock_responders`] does for
967 /// the responder registry.
968 ///
969 /// Every caller clones the `Arc` it needs out of the returned guard and drops
970 /// the guard before doing anything else: the roster lock is never held across
971 /// a call into the liminal library.
972 fn read_channels(
973 &self,
974 ) -> Result<std::sync::RwLockReadGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>, ServerError>
975 {
976 self.channels
977 .read()
978 .map_err(|_poisoned| ServerError::ListenerAccept {
979 message: "channel roster lock poisoned".to_owned(),
980 })
981 }
982
983 /// Subscribes to a configured channel and returns the raw library
984 /// subscription handle so a test can drain the subscriber inbox directly and
985 /// observe exactly which messages reached a subscriber.
986 #[cfg(test)]
987 pub(in crate::server) fn channel_handle_for_test(
988 &self,
989 channel: &str,
990 ) -> Result<ChannelHandle, ServerError> {
991 self.read_channels()?
992 .get(channel)
993 .map(|configured| configured.handle.clone())
994 .ok_or_else(|| ServerError::ListenerAccept {
995 message: format!("channel '{channel}' is not configured"),
996 })
997 }
998
999 #[cfg(test)]
1000 pub(crate) fn subscribe_handle_for_test(
1001 &self,
1002 channel: &str,
1003 ) -> Result<liminal::channel::SubscriptionHandle, ServerError> {
1004 self.subscribe_handle_for_test_with_install(channel, None)
1005 }
1006
1007 /// [`Self::subscribe_handle_for_test`] carrying an explicit install, so a
1008 /// test can put a subscriber under a REAL §5 regime (byte budget, fairness
1009 /// cap) and observe what a publish the bus dropped does to the dedup
1010 /// receipt.
1011 #[cfg(test)]
1012 pub(crate) fn subscribe_handle_for_test_with_install(
1013 &self,
1014 channel: &str,
1015 install: Option<liminal::channel::InboxInstall>,
1016 ) -> Result<liminal::channel::SubscriptionHandle, ServerError> {
1017 let channels = self.read_channels()?;
1018 let configured =
1019 channels
1020 .get(channel)
1021 .map(Arc::clone)
1022 .ok_or_else(|| ServerError::ListenerAccept {
1023 message: format!("channel '{channel}' is not configured"),
1024 })?;
1025 drop(channels);
1026 install
1027 .map_or_else(
1028 || configured.handle.subscribe(),
1029 |install| configured.handle.subscribe_with_install(install),
1030 )
1031 .map_err(|error| ServerError::ListenerAccept {
1032 message: format!("liminal subscribe failed for channel '{channel}': {error}"),
1033 })
1034 }
1035
1036 /// Claims the delivery right for an idempotency key.
1037 ///
1038 /// Returns `Ok(true)` when this is the first publish for the key (the caller
1039 /// may deliver), and `Ok(false)` when the key was already claimed/completed (a
1040 /// duplicate the caller must suppress). The dedup cache is driven synchronously
1041 /// over the in-memory haematite store via the durable bridge.
1042 fn claim_delivery(cache: &DedupCache, key: &str) -> Result<bool, ServerError> {
1043 let decision = block_on(cache.claim_or_get(key, dedup_timestamp_millis()))
1044 .map_err(|error| ServerError::ListenerAccept {
1045 message: format!("dedup bridge failed for key '{key}': {error}"),
1046 })?
1047 .map_err(|error| ServerError::ListenerAccept {
1048 message: format!("dedup claim failed for key '{key}': {error}"),
1049 })?;
1050 Ok(matches!(decision, DedupDecision::Claimed))
1051 }
1052
1053 /// The dedup namespace a publish to `channel` belongs in (A1 §5).
1054 ///
1055 /// Durability decides, because durability is what determines whether a
1056 /// receipt can outlive the message it vouches for. A durable channel's
1057 /// message survives a crash, so its receipt may; an ephemeral channel's
1058 /// does not, so its receipt must not.
1059 const fn dedup_for(&self, channel: &ConfiguredChannel) -> &DedupCache {
1060 match channel.handle.config().mode {
1061 ChannelMode::Durable => &self.dedup,
1062 ChannelMode::Ephemeral => &self.dedup_ephemeral,
1063 }
1064 }
1065
1066 /// Releases a dangling in-flight dedup claim after a failed delivery.
1067 ///
1068 /// Best-effort: a release failure cannot mask the original publish error, so
1069 /// this returns nothing and logs at `error` level instead of surfacing. It is
1070 /// never silent — the leak (a permanently suppressed key) must be observable.
1071 /// `release_claim` itself never clobbers a stored receipt, so calling it on the
1072 /// failure path is safe even if a concurrent completion raced ahead.
1073 fn release_claim(cache: &DedupCache, key: &str) {
1074 match block_on(cache.release_claim(key)) {
1075 Ok(Ok(())) => {}
1076 Ok(Err(error)) => {
1077 tracing::error!(
1078 idempotency_key = key,
1079 %error,
1080 "failed to release dedup claim after publish failure; key may stay suppressed"
1081 );
1082 }
1083 Err(error) => {
1084 tracing::error!(
1085 idempotency_key = key,
1086 %error,
1087 "dedup release bridge failed after publish failure; key may stay suppressed"
1088 );
1089 }
1090 }
1091 }
1092
1093 /// The next connection-level message id.
1094 fn next_message_id(&self) -> u64 {
1095 self.next_message_id.fetch_add(1, Ordering::Relaxed)
1096 }
1097
1098 /// The outcome of a publish the channel took, recorded as accepted with
1099 /// its genuine subscriber deliveries. The delivered count (0 for an empty
1100 /// channel) is the same signal the delivery ack is derived from.
1101 fn delivered(&self, delivery: &ChannelDelivery) -> PublishOutcome {
1102 crate::metrics::publish_accepted();
1103 let delivered_count = u64::try_from(delivery.delivered_count()).unwrap_or(u64::MAX);
1104 crate::metrics::deliveries_recorded(delivered_count);
1105 PublishOutcome {
1106 message_id: self.next_message_id(),
1107 delivered: delivery.is_delivered(),
1108 durable_position: delivery.durable_position(),
1109 reach: Reach::of(delivery),
1110 }
1111 }
1112}
1113
1114#[path = "deferred_store.rs"]
1115mod deferred_store;
1116#[path = "unresolved.rs"]
1117mod unresolved;
1118#[path = "unresolved_start.rs"]
1119mod unresolved_start;
1120#[cfg(test)]
1121#[path = "unresolved_start_tests.rs"]
1122mod unresolved_start_tests;
1123
1124use deferred_store::{DeferredStore, ThrowawayStoreOpener};
1125use unresolved_start::StoreHistory;
1126
1127/// The runtime channel-registration surface.
1128///
1129/// # Reachability: a test seam today, not a production capability
1130///
1131/// Production channels come exclusively from boot configuration; no production
1132/// code path calls `register_channel` (caller census at `c381f01`, G2 in
1133/// `docs/design/wire-durability-gaps-20260831.md` — the only callers are
1134/// `services_registry_tests.rs` and `apply_admission_tests.rs`). The method is
1135/// kept because its semantics — cap gating, typed identity refusals,
1136/// idempotent re-registration — are the contract a future runtime admission
1137/// path must satisfy, and the tests below pin that contract now. Wiring an
1138/// actual runtime admission route (an operator endpoint, a wire frame) is a
1139/// product decision that has NOT been taken; nothing here implies it. Reading
1140/// this method as an existing operator capability is the false-affordance
1141/// mistake this paragraph exists to prevent.
1142///
1143/// Inherent methods, not trait methods, and deliberately so: this is
1144/// authority-moving vocabulary that belongs to the ONE adapter owning a channel
1145/// roster. Putting `register`/`quiesce` on the public [`ConnectionServices`]
1146/// trait would break every external implementor and would hand
1147/// register/quiesce words to a profile that serves no channels at all.
1148///
1149/// The same seam as the existing runtime-mutation API on this type
1150/// (`register_responder`/`unregister_responder`): `&self`, interior mutability,
1151/// typed `Result`.
1152impl LiminalConnectionServices {
1153 /// Registers `spec` on the live roster.
1154 ///
1155 /// Test seam today: see the reachability note on this `impl` block — no
1156 /// production path reaches this method.
1157 ///
1158 /// Idempotent when an entry of that name already has an IDENTICAL
1159 /// configuration — mode, protocol schema id, and schema document, all three
1160 /// — and refuses typed, naming the first differing field, otherwise. An
1161 /// identical registration against a boot-configured entry answers
1162 /// [`Registered::AlreadyIdentical`] and leaves its origin alone: flipping it
1163 /// would make the entry lie about its restart fate and would move it into
1164 /// the counted population without a channel having been created.
1165 ///
1166 /// The cap is consulted first, in two steps with different reach. An absent
1167 /// `limits.max_channels` refuses EVERY call, identical or not, before the
1168 /// roster is read at all: a deployment that has declared no bound has not
1169 /// said what it admits, and unbounded-by-default is not a bound. A REACHED
1170 /// cap refuses only a call that would create an entry — it gates the insert,
1171 /// which is the population it bounds. An identical re-registration inserts
1172 /// nothing and so is answered `AlreadyIdentical` even at a full roster;
1173 /// refusing it would break idempotency at exactly the boundary a projector
1174 /// re-projecting its record crosses.
1175 ///
1176 /// # Errors
1177 /// Returns [`ChannelRegistryError`] when no cap is configured, the cap is
1178 /// reached, the name exists with a different configuration, the schema bytes
1179 /// do not parse or compile, durable initialization over the shared store
1180 /// fails, or the roster lock is poisoned.
1181 pub fn register_channel(
1182 &self,
1183 spec: &ChannelRegistration,
1184 ) -> Result<Registered, ChannelRegistryError> {
1185 // The cap is consulted before the roster is touched: an undeclared bound
1186 // refuses every runtime registration, identical or not, because the
1187 // deployment has not said what it admits.
1188 let Some(limit) = self.max_channels else {
1189 return Err(ChannelRegistryError::CapNotConfigured {
1190 cap: MAX_CHANNELS_KEY,
1191 });
1192 };
1193 // Schema resolution is pure (a JSON parse and a digest), so it runs with
1194 // no lock held and its result serves BOTH the identity comparison and
1195 // the construction below.
1196 let resolved = resolve_schema_bytes(spec.schema_bytes.as_deref()).map_err(|error| {
1197 ChannelRegistryError::SchemaRejected {
1198 name: spec.name.clone(),
1199 message: error.to_string(),
1200 }
1201 })?;
1202
1203 // Fast paths, under a READ lock: an already-identical registration
1204 // builds nothing, and a full roster refuses before paying for a
1205 // construction it would discard. Neither is the authority — the write
1206 // lock below re-decides both, because the roster can move in between.
1207 let (existing, registered_count) = {
1208 let channels = self.read_roster()?;
1209 let existing = channels.get(&spec.name).map(Arc::clone);
1210 let registered_count = runtime_registered_count(&channels);
1211 drop(channels);
1212 (existing, registered_count)
1213 };
1214 if let Some(existing) = existing {
1215 return compare_registration(&existing, spec, &resolved);
1216 }
1217 if registered_count >= limit {
1218 return Err(ChannelRegistryError::CapReached {
1219 cap: MAX_CHANNELS_KEY,
1220 limit,
1221 });
1222 }
1223
1224 // Construction runs with NO lock held. A durable channel recovers its
1225 // per-partition sequence counters from the store here, which is O(stream
1226 // length) in store reads; holding the roster lock across it would put a
1227 // slow store walk on every connection's publish and subscribe path.
1228 let configured = build_configured_channel(
1229 &spec.name,
1230 resolved,
1231 spec.durable,
1232 ChannelOrigin::RuntimeRegistered,
1233 &self.durable_store,
1234 self.cluster.supervisor(),
1235 )
1236 .map_err(|error| error.into_registry_error(&spec.name))?;
1237
1238 // The authoritative decision: identity, cap, and insert under ONE write
1239 // lock, so the count that admitted the entry and the insert that added
1240 // it cannot be separated by a concurrent registration.
1241 let mut exporters = self.export_subscriptions.lock().map_err(|_| {
1242 ChannelRegistryError::RosterUnavailable {
1243 message: "export subscription registry lock poisoned".to_owned(),
1244 }
1245 })?;
1246 exporters.retain(|exporter| exporter.strong_count() > 0);
1247 let mut channels = self.write_roster()?;
1248 let raced = channels.get(&spec.name).map(Arc::clone);
1249 let registered_count = runtime_registered_count(&channels);
1250 if raced.is_none() && registered_count < limit {
1251 let mut prepared = Vec::new();
1252 for exporter in exporters.iter().filter_map(std::sync::Weak::upgrade) {
1253 if let Some(binding) = exporter.binding(&spec.name) {
1254 let refused = |error: ServerError| ChannelRegistryError::RosterUnavailable {
1255 message: format!(
1256 "export '{}' cannot subscribe to '{}': {error}",
1257 exporter.link(),
1258 spec.name
1259 ),
1260 };
1261 let subscription = self
1262 .build_subscription(
1263 &spec.name,
1264 &configured.handle,
1265 configured.protocol_schema,
1266 Some(binding.install),
1267 Some(&binding.consumer),
1268 )
1269 .map_err(refused)?;
1270 let subscription =
1271 crate::link_bridge::subscriptions::PreparedSubscription::new(
1272 binding.prefix,
1273 spec.name.clone(),
1274 subscription,
1275 )
1276 .map_err(refused)?;
1277 prepared.push((exporter, subscription));
1278 }
1279 }
1280 // Bind every exporter before a publish can find the new roster entry.
1281 for (exporter, subscription) in prepared {
1282 exporter.attach(subscription);
1283 }
1284 channels.insert(spec.name.clone(), Arc::new(configured));
1285 drop(channels);
1286 drop(exporters);
1287 self.console_changed(crate::console::TopologyChange::Channel {
1288 name: spec.name.clone(),
1289 life: line_life(spec.durable),
1290 state: crate::console::records::LineState::Live,
1291 });
1292 return Ok(Registered::Created);
1293 }
1294 drop(channels);
1295 drop(exporters);
1296 if let Some(raced) = raced {
1297 // A racer registered this name while the channel above was being
1298 // built. The built entry is dropped unused: it owns no actor (the
1299 // actor is spawned lazily on first use) and its durable
1300 // construction only READ the store, so discarding it changes
1301 // nothing an observer could see.
1302 return compare_registration(&raced, spec, &schema_of(&configured));
1303 }
1304 Err(ChannelRegistryError::CapReached {
1305 cap: MAX_CHANNELS_KEY,
1306 limit,
1307 })
1308 }
1309
1310 /// Moves `name` from active to quiesced with a named `reason`. ONE-WAY.
1311 ///
1312 /// New publishes and new subscribes are refused afterwards, carrying the
1313 /// reason. Existing subscriptions are UNTOUCHED: nothing revokes a
1314 /// subscription handle, the actor's subscriber list is not walked, no EXIT
1315 /// is sent, and the channel actor keeps running. Quiesce is a roster-level
1316 /// admission decision, not an actor command.
1317 ///
1318 /// Re-quiescing under the IDENTICAL reason is `Ok(())`; a DIFFERENT reason
1319 /// refuses, carrying the reason already on record.
1320 ///
1321 /// The return does NOT mean "no new subscriber can appear". A subscribe that
1322 /// has already passed admission completes and gets its stream — the
1323 /// linearisation point is the admission read, not the subscribe's
1324 /// completion. A consumer that needs "nobody is attached" must observe
1325 /// attachment directly.
1326 ///
1327 /// # Errors
1328 /// Returns [`ChannelRegistryError`] when the name is not registered, is
1329 /// already quiesced under a different reason, or the roster lock is
1330 /// poisoned.
1331 pub fn quiesce_channel(
1332 &self,
1333 name: &str,
1334 reason: impl Into<String>,
1335 ) -> Result<(), ChannelRegistryError> {
1336 let reason = reason.into();
1337 // A READ lock: the state machine lives on the ENTRY, not in the map, so
1338 // the operation the design most wants to be safe never blocks a reader.
1339 let configured = {
1340 let channels = self.read_roster()?;
1341 channels.get(name).map(Arc::clone).ok_or_else(|| {
1342 ChannelRegistryError::NotRegistered {
1343 name: name.to_owned(),
1344 }
1345 })?
1346 };
1347 configured.quiesce(name, &reason)?;
1348 self.console_changed(crate::console::TopologyChange::Channel {
1349 name: name.to_owned(),
1350 life: line_life(configured.durable),
1351 state: crate::console::records::LineState::Quiesced,
1352 });
1353 Ok(())
1354 }
1355
1356 /// Cheap typed probe: one roster read plus one atomic load.
1357 ///
1358 /// Touches no actor and therefore CANNOT spawn one. Every handle accessor
1359 /// that could answer a question about a channel's activity routes through
1360 /// the lazy-spawn path, so a probe built on one would materialise the actor
1361 /// of the idle channel it was asked about — turning a read into a side
1362 /// effect. This reads the roster entry's own recorded fields and nothing
1363 /// else.
1364 ///
1365 /// # Errors
1366 /// Returns [`ChannelRegistryError::RosterUnavailable`] only.
1367 pub fn channel_status(&self, name: &str) -> Result<ChannelStatus, ChannelRegistryError> {
1368 let configured = {
1369 let channels = self.read_roster()?;
1370 channels.get(name).map(Arc::clone)
1371 };
1372 Ok(
1373 configured.map_or(ChannelStatus::NotRegistered, |configured| {
1374 status_of(&configured)
1375 }),
1376 )
1377 }
1378
1379 /// Every roster entry with its status, sorted by name, from ONE roster
1380 /// read (SRV-012 R7: the console's topology snapshot).
1381 ///
1382 /// # Errors
1383 /// Returns [`ChannelRegistryError::RosterUnavailable`] when the roster
1384 /// lock is poisoned.
1385 pub fn channel_census(&self) -> Result<Vec<(String, ChannelStatus)>, ChannelRegistryError> {
1386 let mut census: Vec<(String, ChannelStatus)> = self
1387 .read_roster()?
1388 .iter()
1389 .map(|(name, configured)| (name.clone(), status_of(configured)))
1390 .collect();
1391 census.sort_by(|left, right| left.0.cmp(&right.0));
1392 Ok(census)
1393 }
1394
1395 /// The whole roster: one minimal descriptor per entry, sorted by name.
1396 ///
1397 /// The census companion to [`Self::channel_status`]. A by-name probe answers
1398 /// about a name the caller already suspects; only an enumeration can reveal
1399 /// a name the caller does not know to ask about, and a verification sweep
1400 /// with no population denominator is an instrument shape this estate
1401 /// forbids. Touches no actor, under the same constraint as the probe.
1402 ///
1403 /// # Errors
1404 /// Returns [`ChannelRegistryError::RosterUnavailable`] only.
1405 pub fn registered_channels(&self) -> Result<Vec<ChannelDescriptor>, ChannelRegistryError> {
1406 let entries: Vec<(String, Arc<ConfiguredChannel>)> = {
1407 let channels = self.read_roster()?;
1408 channels
1409 .iter()
1410 .map(|(name, configured)| (name.clone(), Arc::clone(configured)))
1411 .collect()
1412 };
1413 let mut descriptors: Vec<ChannelDescriptor> = entries
1414 .into_iter()
1415 .map(|(name, configured)| ChannelDescriptor {
1416 name,
1417 origin: configured.origin,
1418 state: configured.state(),
1419 })
1420 .collect();
1421 descriptors.sort_by(|left, right| left.name.cmp(&right.name));
1422 Ok(descriptors)
1423 }
1424
1425 /// The roster admission funnel: the ONE place a channel operation's
1426 /// permission is decided.
1427 ///
1428 /// Returns the admitted entry, so the caller works from the value the
1429 /// decision was made on rather than reading the roster a second time and
1430 /// risking a different answer. This read is the LINEARISATION POINT for the
1431 /// quiesce race: everything the caller does with the returned entry happens
1432 /// outside the lock and is not re-checked, which is exactly why a quiesce
1433 /// that commits after this returns does not stop the operation it admitted.
1434 ///
1435 /// # Errors
1436 /// Returns [`ChannelAccessError`] when the channel is absent, quiesced, or
1437 /// the roster lock is poisoned.
1438 fn admit_channel(&self, channel: &str) -> Result<Arc<ConfiguredChannel>, ChannelAccessError> {
1439 let configured = {
1440 let channels = self.channels.read().map_err(|_poisoned| {
1441 ChannelAccessError::RosterUnavailable {
1442 message: ROSTER_POISONED.to_owned(),
1443 }
1444 })?;
1445 channels.get(channel).map(Arc::clone)
1446 };
1447 let configured = configured.ok_or_else(|| ChannelAccessError::NotRegistered {
1448 name: channel.to_owned(),
1449 })?;
1450 if configured.state.load(Ordering::Acquire) == STATE_QUIESCED {
1451 return Err(ChannelAccessError::Quiesced {
1452 name: channel.to_owned(),
1453 reason: configured.recorded_quiesce_reason(),
1454 });
1455 }
1456 Ok(configured)
1457 }
1458
1459 /// Takes the roster's READ guard for the registration surface, mapping a
1460 /// poisoned lock to a typed [`ChannelRegistryError`].
1461 ///
1462 /// Separate from [`Self::read_channels`] because the two surfaces answer
1463 /// different callers with different error types; recovering one from the
1464 /// other would mean inspecting a message.
1465 fn read_roster(
1466 &self,
1467 ) -> Result<
1468 std::sync::RwLockReadGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>,
1469 ChannelRegistryError,
1470 > {
1471 #[cfg(test)]
1472 ROSTER_READS.with(|reads| reads.set(reads.get() + 1));
1473 self.channels
1474 .read()
1475 .map_err(|_poisoned| ChannelRegistryError::RosterUnavailable {
1476 message: ROSTER_POISONED.to_owned(),
1477 })
1478 }
1479
1480 /// Takes the roster's WRITE guard — held only for the check-and-insert that
1481 /// must be atomic, never across a call into the liminal library.
1482 fn write_roster(
1483 &self,
1484 ) -> Result<
1485 std::sync::RwLockWriteGuard<'_, HashMap<String, Arc<ConfiguredChannel>>>,
1486 ChannelRegistryError,
1487 > {
1488 self.channels
1489 .write()
1490 .map_err(|_poisoned| ChannelRegistryError::RosterUnavailable {
1491 message: ROSTER_POISONED.to_owned(),
1492 })
1493 }
1494
1495 /// Gives back the tap slot a subscribe to `channel` reserved at
1496 /// admission, taking its tap channel off the roster when no tap stands
1497 /// behind it (SRV-012 R6).
1498 fn give_back_tap(&self, channel: &str) {
1499 self.console.release_tap(channel, |subject| {
1500 crate::console::ServerPublisher::retire_console_channel(self, subject)
1501 });
1502 }
1503
1504 /// The trait's subscribe, from the console guard to the subscription;
1505 /// [`ConnectionServices::subscribe`] gives back the tap reservation when
1506 /// this fails.
1507 fn open_subscription(
1508 &self,
1509 channel: &str,
1510 accepted_schemas: &[ProtocolSchemaId],
1511 install: Option<liminal::channel::InboxInstall>,
1512 consumer: Option<&str>,
1513 ) -> Result<ConnectionSubscription, ServerError> {
1514 // The frame path refused this already at admission; this guard is for
1515 // a caller holding the trait directly.
1516 refuse_console_tap_target(channel).map_err(|error| access_to_server_error(&error))?;
1517 // As in `publish`: the admission funnel clones the entry's `Arc` out
1518 // under the read guard and the guard is released before schema
1519 // negotiation or the actor round-trip below. That release is what makes
1520 // this the linearisation point — a quiesce landing after it does not
1521 // stop the subscription this call is already building.
1522 let configured = self
1523 .admit_channel(channel)
1524 .map_err(|error| access_to_server_error(&error))?;
1525 let selected_schema = if accepted_schemas.is_empty() {
1526 configured.protocol_schema
1527 } else {
1528 liminal::protocol::negotiate_schema(configured.protocol_schema, accepted_schemas)
1529 .map_err(|error| server_error_from_protocol(&error))?
1530 };
1531 self.build_subscription(
1532 channel,
1533 &configured.handle,
1534 selected_schema,
1535 install,
1536 consumer,
1537 )
1538 }
1539
1540 fn build_subscription(
1541 &self,
1542 channel: &str,
1543 handle: &ChannelHandle,
1544 selected_schema: ProtocolSchemaId,
1545 install: Option<liminal::channel::InboxInstall>,
1546 consumer: Option<&str>,
1547 ) -> Result<ConnectionSubscription, ServerError> {
1548 // `subscribe_with_install` installs the §5 budget/fairness cap and the R3
1549 // wake notifier on the inbox at construction — strictly before the
1550 // registration is published to the channel actor — so there is no window
1551 // in which a publish can land uncharged or without a wake.
1552 let subscription = match (consumer, install) {
1553 (Some(consumer), Some(install)) => handle.subscribe_consumer(consumer, install),
1554 (Some(_), None) => {
1555 return Err(ServerError::ListenerAccept {
1556 message: "a durable consumer requires a bounded inbox install".to_owned(),
1557 });
1558 }
1559 (None, Some(install)) => handle.subscribe_with_install(install),
1560 (None, None) => handle.subscribe(),
1561 }
1562 .map_err(|error| ServerError::ListenerAccept {
1563 message: format!("liminal subscribe failed for channel '{channel}': {error}"),
1564 })?;
1565 let id = self.next_subscription_id.fetch_add(1, Ordering::Relaxed);
1566 let mut connection_subscription = ConnectionSubscription::new(
1567 id,
1568 selected_schema,
1569 Box::new(LiminalSubscriptionResource { subscription }),
1570 );
1571 connection_subscription.set_channel(channel.to_owned());
1572 // SRV-012 R6: the console tap attaches and detaches through the
1573 // supervisor's observer, which hears every subscribe and every
1574 // departure once (ServerObserver).
1575 Ok(connection_subscription)
1576 }
1577
1578 pub(crate) fn subscribe_consumer(
1579 &self,
1580 channel: &str,
1581 consumer: &str,
1582 install: liminal::channel::InboxInstall,
1583 ) -> Result<ConnectionSubscription, ServerError> {
1584 self.open_subscription(channel, &[], Some(install), Some(consumer))
1585 }
1586
1587 pub(crate) fn install_exporter(
1588 &self,
1589 exporter: &Arc<crate::link_bridge::subscriptions::ExportSubscriptions>,
1590 ) -> Result<(), ServerError> {
1591 let mut exporters =
1592 self.export_subscriptions
1593 .lock()
1594 .map_err(|_| ServerError::ListenerAccept {
1595 message: "export subscription registry lock poisoned".to_owned(),
1596 })?;
1597 exporters.retain(|exporter| exporter.strong_count() > 0);
1598 if exporters
1599 .iter()
1600 .filter_map(std::sync::Weak::upgrade)
1601 .any(|existing| existing.link() == exporter.link())
1602 {
1603 return Err(ServerError::ListenerAccept {
1604 message: format!(
1605 "link '{}' already owns an export subscription",
1606 exporter.link()
1607 ),
1608 });
1609 }
1610 let mut prepared = Vec::new();
1611 for channel in self
1612 .registered_channels()
1613 .map_err(|error| ServerError::ListenerAccept {
1614 message: error.to_string(),
1615 })?
1616 {
1617 if matches!(channel.state, ChannelState::Active) {
1618 if let Some(binding) = exporter.binding(&channel.name) {
1619 let subscription =
1620 self.subscribe_consumer(&channel.name, &binding.consumer, binding.install)?;
1621 prepared.push(
1622 crate::link_bridge::subscriptions::PreparedSubscription::new(
1623 binding.prefix,
1624 channel.name,
1625 subscription,
1626 )?,
1627 );
1628 }
1629 }
1630 }
1631 for subscription in prepared {
1632 exporter.attach(subscription);
1633 }
1634 exporters.push(Arc::downgrade(exporter));
1635 drop(exporters);
1636 Ok(())
1637 }
1638
1639 /// Refuses a subscribe that would start one more live tap than
1640 /// [`crate::console::MAX_LIVE_TAPS`] (SRV-012 R6); a subscriber to a tap
1641 /// already live is not a new tap.
1642 ///
1643 /// A poisoned tap table does not refuse the subscribe (R8: a console
1644 /// bookkeeping failure never fails the client operation it watches); the
1645 /// attach that follows meets the poison and counts it.
1646 ///
1647 /// # Errors
1648 /// Returns [`ChannelAccessError::ConsoleTapLimit`] naming the channel.
1649 fn refuse_console_tap_limit(&self, channel: &str) -> Result<(), ChannelAccessError> {
1650 match self.console.reserve_tap(channel) {
1651 Ok(false) => Err(ChannelAccessError::ConsoleTapLimit {
1652 name: channel.to_owned(),
1653 limit: crate::console::MAX_LIVE_TAPS,
1654 }),
1655 Ok(true) | Err(_) => Ok(()),
1656 }
1657 }
1658}
1659
1660impl crate::console::ServerPublisher for LiminalConnectionServices {
1661 /// The server's own record on the one publish path, with a server origin:
1662 /// attributed to `liminal.<device>`, and never counted as a connection's
1663 /// work by the cost tick.
1664 fn publish_console(
1665 &self,
1666 subject: &str,
1667 payload: &[u8],
1668 ) -> Result<(), crate::console::ConsoleError> {
1669 let envelope = MessageEnvelope::new(
1670 ProtocolSchemaId::new([0_u8; ProtocolSchemaId::WIRE_LEN]),
1671 liminal::protocol::CausalContext::independent(),
1672 payload.to_vec(),
1673 );
1674 self.publish_with_origin(
1675 subject,
1676 &envelope,
1677 None,
1678 PublishOrigin::Server {
1679 device: self.console.device(),
1680 },
1681 )
1682 .map(|_published| ())
1683 .map_err(|source| crate::console::ConsoleError::Server {
1684 subject: subject.to_owned(),
1685 source,
1686 })
1687 }
1688
1689 /// Takes a `console.tap.<target>` channel off the roster (SRV-012 R6).
1690 /// Only a console subject is ever removed here.
1691 fn retire_console_channel(&self, subject: &str) -> Result<(), crate::console::ConsoleError> {
1692 if !super::channel_registry::is_console_subject(subject) {
1693 return Ok(());
1694 }
1695 let mut channels =
1696 self.write_roster()
1697 .map_err(|error| crate::console::ConsoleError::Server {
1698 subject: subject.to_owned(),
1699 source: ServerError::ListenerAccept {
1700 message: error.to_string(),
1701 },
1702 })?;
1703 channels.remove(subject);
1704 drop(channels);
1705 Ok(())
1706 }
1707}
1708
1709/// The production participant service and the operator surfaces taken from
1710/// its one live handler before it is erased behind
1711/// `dyn ParticipantSemanticHandler`.
1712struct InstalledParticipant {
1713 service: InstalledParticipantService,
1714 unloadable_conversations: UnloadableConversationRecord,
1715 credential_reissuer: Arc<dyn OperatorCredentialReissuer>,
1716 warm: ParticipantWarm,
1717}
1718
1719/// Production participant activation (LP gap closure, Part B): the
1720/// deployment's [participant] section installs the ONE production semantic
1721/// handler, sealed together with the same durable store the conversation logs
1722/// live in, under the configured wire-frame limit.
1723///
1724/// # Errors
1725/// Returns [`ServerError::ParticipantStartupRestore`] when the handler cannot
1726/// be built or wired, and [`ServerError::ConfigValidation`] when the
1727/// wire-frame limit is below the protocol's minimum participant frame.
1728fn install_participant(
1729 durable_store: &Arc<dyn DurableStore>,
1730 participant: &ParticipantConfig,
1731 signals: &ServerSignals,
1732) -> Result<InstalledParticipant, ServerError> {
1733 let handler = ProductionParticipantHandler::new(Arc::clone(durable_store), *participant)
1734 .map_err(|error| ServerError::ParticipantStartupRestore {
1735 message: error.to_string(),
1736 })?;
1737 // Taken BEFORE the handler is erased behind
1738 // `dyn ParticipantSemanticHandler`. Construction restores no
1739 // conversation (DUR-020), so the record fills as conversations restore,
1740 // on first touch or in the warm, and is complete once the warm has
1741 // finished.
1742 handler.install_signals(signals.clone()).map_err(|error| {
1743 ServerError::ParticipantStartupRestore {
1744 message: error.to_string(),
1745 }
1746 })?;
1747 let unloadable_conversations = handler.unloadable_record();
1748 let handler = Arc::new(handler);
1749 let warm = handler
1750 .install_warm()
1751 .map_err(|error| ServerError::ParticipantStartupRestore {
1752 message: error.to_string(),
1753 })?;
1754 // A7 (§0.18): taken here for the same reason, and from the same one live
1755 // handler, so the operator surface and the participant wire cannot end up
1756 // talking to two different owners.
1757 let credential_reissuer: Arc<dyn OperatorCredentialReissuer> =
1758 Arc::<ProductionParticipantHandler>::clone(&handler);
1759 let service = InstalledParticipantService::new(
1760 handler,
1761 Arc::clone(durable_store),
1762 participant.wire_frame_limit,
1763 )
1764 .map_err(|error| ServerError::ConfigValidation {
1765 message: format!(
1766 "participant.wire_frame_limit: {} is below the protocol's minimum \
1767 complete participant frame ({error:?})",
1768 participant.wire_frame_limit
1769 ),
1770 })?;
1771 Ok(InstalledParticipant {
1772 service,
1773 unloadable_conversations,
1774 credential_reissuer,
1775 warm,
1776 })
1777}
1778
1779/// The console hub for `device`, its model built from the roster as it
1780/// stands (responders, connections and peers arrive as changes).
1781///
1782/// # Errors
1783/// Returns [`ServerError::ListenerAccept`] when the device is blank or the
1784/// wall clock cannot be read.
1785fn console_hub(
1786 device: &str,
1787 channels: &HashMap<String, Arc<ConfiguredChannel>>,
1788 depot: bool,
1789 lys: Option<&crate::config::types::LysConfig>,
1790) -> Result<crate::console::ConsoleHub, ServerError> {
1791 let this = crate::console::ThisDevice {
1792 id: device.to_owned(),
1793 liminal: env!("CARGO_PKG_VERSION").to_owned(),
1794 started: Instant::now(),
1795 depot,
1796 };
1797 let lines = channels.iter().map(|(name, configured)| {
1798 let state = match configured.state() {
1799 ChannelState::Active => crate::console::records::LineState::Live,
1800 ChannelState::Quiesced { .. } => crate::console::records::LineState::Quiesced,
1801 };
1802 (name.clone(), line_life(configured.durable), state)
1803 });
1804 let mut model = crate::console::TopologyModel::new(this, [], lines, [], []);
1805 if let Some(machine) = lys.and_then(|lys| lys.machine.as_ref()) {
1806 model = model.with_machine(machine);
1807 }
1808 crate::console::wall::WallBase::now()
1809 .and_then(|wall| crate::console::ConsoleHub::new(device, model, wall))
1810 .map_err(|error| ServerError::ListenerAccept {
1811 message: format!("console hub: {error}"),
1812 })
1813}
1814
1815/// A roster entry's status as [`LiminalConnectionServices::channel_status`]
1816/// answers it.
1817fn status_of(configured: &ConfiguredChannel) -> ChannelStatus {
1818 let mode = configured.handle.config().mode;
1819 match configured.state() {
1820 ChannelState::Active => ChannelStatus::Active {
1821 origin: configured.origin,
1822 mode,
1823 schema: configured.protocol_schema,
1824 },
1825 ChannelState::Quiesced { reason } => ChannelStatus::Quiesced {
1826 reason,
1827 origin: configured.origin,
1828 mode,
1829 },
1830 }
1831}
1832
1833#[cfg(test)]
1834thread_local! {
1835 /// Roster read locks taken on this thread. Test-only.
1836 static ROSTER_READS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
1837}
1838
1839/// Roster read locks taken on the calling thread so far. Test-only.
1840#[cfg(test)]
1841pub(crate) fn roster_reads() -> u64 {
1842 ROSTER_READS.with(std::cell::Cell::get)
1843}
1844
1845/// A channel's line life: durable channels keep, ephemeral ones pass.
1846const fn line_life(durable: bool) -> crate::console::records::LineLife {
1847 if durable {
1848 crate::console::records::LineLife::Kept
1849 } else {
1850 crate::console::records::LineLife::Passing
1851 }
1852}
1853
1854/// The diagnostic carried when the roster lock is poisoned. One string, shared
1855/// by every surface that reports it, so the wording cannot drift between them.
1856const ROSTER_POISONED: &str = "channel roster lock poisoned";
1857
1858/// How many roster entries the registration cap counts.
1859///
1860/// Runtime-registered entries ONLY. Boot-configured channels are the operator's
1861/// own authored bound — they are in the file the operator wrote — and counting
1862/// them would make one number mean two different things depending on how the
1863/// deployment was configured. The origin never flips, so this population is
1864/// well defined over time.
1865fn runtime_registered_count(channels: &HashMap<String, Arc<ConfiguredChannel>>) -> usize {
1866 channels
1867 .values()
1868 .filter(|configured| configured.origin == ChannelOrigin::RuntimeRegistered)
1869 .count()
1870}
1871
1872/// The three-field identity comparison behind idempotent-if-identical
1873/// registration.
1874///
1875/// All three must match; the FIRST mismatch is reported by name. The name itself
1876/// is not compared — it is the roster key, a precondition of the comparison
1877/// rather than a member of it — and neither are the supervisor or the durable
1878/// store, which are one server-wide instance each and cannot differ between two
1879/// registrations in one process.
1880///
1881/// The schema is compared as BOTH its protocol id and its parsed document, on
1882/// purpose. The id is a 64-bit non-cryptographic digest, so the document guards
1883/// against a collision accepting a different schema as identical; and the two
1884/// fail in opposite directions, because two byte sequences that parse to the
1885/// same document but differ in whitespace produce different ids — which must
1886/// refuse, since the id is what every future subscriber negotiates.
1887///
1888/// The channel's own `Schema` is not compared: it is not `PartialEq`, and its
1889/// identifier is a fresh value per construction, so comparing it would refuse
1890/// every idempotent re-registration.
1891fn compare_registration(
1892 existing: &ConfiguredChannel,
1893 spec: &ChannelRegistration,
1894 resolved: &ChannelSchema,
1895) -> Result<Registered, ChannelRegistryError> {
1896 let requested_mode = if spec.durable {
1897 ChannelMode::Durable
1898 } else {
1899 ChannelMode::Ephemeral
1900 };
1901 let config = existing.handle.config();
1902 let mismatch = if config.mode == requested_mode {
1903 if existing.protocol_schema == resolved.protocol_id {
1904 if *config.schema.definition() == resolved.document {
1905 None
1906 } else {
1907 Some(ChannelConfigField::SchemaDocument)
1908 }
1909 } else {
1910 Some(ChannelConfigField::SchemaId)
1911 }
1912 } else {
1913 Some(ChannelConfigField::Mode)
1914 };
1915 mismatch.map_or(Ok(Registered::AlreadyIdentical), |field| {
1916 Err(ChannelRegistryError::AlreadyRegistered {
1917 name: spec.name.clone(),
1918 field,
1919 })
1920 })
1921}
1922
1923/// Recovers the resolved schema of an already-built entry, so the write-lock
1924/// re-check compares the SAME three fields against the same values the fast path
1925/// would have.
1926fn schema_of(configured: &ConfiguredChannel) -> ChannelSchema {
1927 ChannelSchema {
1928 document: configured.handle.config().schema.definition().clone(),
1929 protocol_id: configured.protocol_schema,
1930 }
1931}
1932
1933/// Builds one roster entry: the channel's validation engine, its library handle
1934/// (durable or ephemeral), and the protocol schema id advertised at subscribe
1935/// time.
1936///
1937/// This is the SOLE place a [`ConfiguredChannel`] is constructed. Keeping
1938/// construction in one function is what makes "every channel on this server is
1939/// the same kind of object" structural rather than a promise: there is no second
1940/// body a channel could be built by, so no channel can drift into a different
1941/// shape. The boot loop in [`LiminalConnectionServices::from_config_with_store_via`]
1942/// is its caller.
1943///
1944/// `resolved` is consumed because its JSON Schema document is moved into the
1945/// channel's [`Schema`]; the protocol id is carried onto the entry unchanged.
1946/// `origin` is stamped here and never written again.
1947///
1948/// The failure is typed ([`ChannelBuildError`]) rather than a [`ServerError`] so
1949/// BOTH callers can render it without inspecting a message: the boot loop maps
1950/// it back to the exact `ConfigValidation` strings it has always produced, and
1951/// `register_channel` maps it to the registry's own typed variants.
1952///
1953/// # Errors
1954/// Returns [`ChannelBuildError`] when the JSON Schema document does not compile
1955/// or durable initialization over the shared store fails.
1956fn build_configured_channel(
1957 name: &str,
1958 resolved: ChannelSchema,
1959 durable: bool,
1960 origin: ChannelOrigin,
1961 durable_store: &Arc<dyn DurableStore>,
1962 supervisor: &ChannelSupervisor,
1963) -> Result<ConfiguredChannel, ChannelBuildError> {
1964 let schema =
1965 Schema::new(resolved.document).map_err(|error| ChannelBuildError::SchemaRejected {
1966 message: error.to_string(),
1967 })?;
1968 let channel_config = if durable {
1969 ChannelConfig::new(name.to_owned(), schema, ChannelMode::Durable)
1970 } else {
1971 ChannelConfig::new(name.to_owned(), schema, ChannelMode::Ephemeral)
1972 };
1973 let handle = if durable {
1974 ChannelHandle::new_durable_with_supervisor(
1975 channel_config,
1976 Arc::clone(durable_store),
1977 supervisor.clone(),
1978 )
1979 .map_err(|error| ChannelBuildError::DurableInitFailed {
1980 message: error.to_string(),
1981 })?
1982 } else {
1983 ChannelHandle::with_supervisor(channel_config, supervisor.clone())
1984 };
1985 Ok(ConfiguredChannel {
1986 handle,
1987 protocol_schema: resolved.protocol_id,
1988 durable,
1989 origin,
1990 state: AtomicU8::new(STATE_ACTIVE),
1991 quiesce_reason: OnceLock::new(),
1992 })
1993}
1994
1995/// Returns the current epoch-millis timestamp used as the dedup entry anchor.
1996///
1997/// A clock error before the Unix epoch yields `0`: the timestamp is only a TTL
1998/// anchor for the in-memory cache and a zero anchor never breaks the at-most-once
1999/// claim semantics, so this avoids surfacing a clock fault on the publish path.
2000fn dedup_timestamp_millis() -> u64 {
2001 use std::time::{SystemTime, UNIX_EPOCH};
2002 SystemTime::now()
2003 .duration_since(UNIX_EPOCH)
2004 .ok()
2005 .and_then(|duration| u64::try_from(duration.as_millis()).ok())
2006 .unwrap_or(0)
2007}
2008
2009/// Default shard count for an on-disk durable store.
2010///
2011/// Haematite routes keys across this many single-threaded shard actors; a small
2012/// power of two gives parallelism across cursors/streams without spawning an
2013/// actor per core. The value is fixed (haematite has no silent default) and not
2014/// yet surfaced in server config.
2015const DEFAULT_SHARD_COUNT: usize = 8;
2016
2017/// Shard count of the throwaway store an incarnation's ephemeral dedup
2018/// namespace lives in: it holds one namespace, and dies with the process.
2019const THROWAWAY_SHARD_COUNT: usize = 1;
2020
2021/// Namespace prefix for the dedup-on-delivery cache streams. Keeps delivery dedup
2022/// keys from colliding with any other haematite streams in the shared store.
2023///
2024/// This is the **durable** namespace (A1 §5): stable across restarts, because a
2025/// durable channel's message outlives a restart too.
2026const DELIVERY_DEDUP_NAMESPACE: &str = "liminal:delivery-dedup";
2027
2028/// The **ephemeral** dedup namespace for this server incarnation (A1 §5,
2029/// graft §0.2).
2030///
2031/// Scoped to the process incarnation so an ephemeral channel's receipt shares a
2032/// lifetime with the in-memory message it vouches for. Composed from the
2033/// process id and the boot instant, which is enough to make two incarnations
2034/// disagree; nothing reads the components back, so no meaning is attached to
2035/// them beyond distinctness.
2036///
2037/// The namespace lives in [`ephemeral_dedup_store`], never in a store that
2038/// outlives the process, so no incarnation's records are left behind for a
2039/// later one to read past.
2040fn ephemeral_dedup_namespace() -> String {
2041 use std::time::{SystemTime, UNIX_EPOCH};
2042 let boot_nanos = SystemTime::now()
2043 .duration_since(UNIX_EPOCH)
2044 .map_or(0, |since| since.as_nanos());
2045 format!(
2046 "{DELIVERY_DEDUP_NAMESPACE}:incarnation:{}-{boot_nanos}",
2047 std::process::id()
2048 )
2049}
2050
2051/// One reader over the counters of `durable_store` and, when it is a separate
2052/// store, `ephemeral_store`; a store that is the same `Arc` is read once.
2053fn combined_store_counts(
2054 durable_store: &Arc<dyn DurableStore>,
2055 ephemeral_store: &Arc<dyn DurableStore>,
2056) -> StoreCountsHandle {
2057 let durable = durable_store.store_counts().unwrap_or_default();
2058 if Arc::ptr_eq(durable_store, ephemeral_store) {
2059 return durable;
2060 }
2061 durable.with(&ephemeral_store.store_counts().unwrap_or_default())
2062}
2063
2064/// The store an incarnation's ephemeral dedup namespace is written to: the
2065/// services' own store when it was created empty for this process and dies
2066/// with it, otherwise a store of its own that does, opened by the namespace's
2067/// first write rather than at start.
2068///
2069/// Written into a persistent store, every incarnation's namespace would stay
2070/// there for good, under the durable namespace's prefix, with nothing that
2071/// ever reads it again to prune it.
2072fn ephemeral_dedup_store(
2073 durable_store: &Arc<dyn DurableStore>,
2074 history: StoreHistory,
2075 subsystems: &dyn SubsystemFactory,
2076) -> Arc<dyn DurableStore> {
2077 match (history, durable_store) {
2078 #[cfg(any(test, feature = "test-support"))]
2079 (StoreHistory::Fresh, store) => Arc::clone(store),
2080 (StoreHistory::Created | StoreHistory::Recorded, _) => Arc::new(CountingStore::new(
2081 DeferredStore::new(subsystems.throwaway_store()),
2082 )),
2083 }
2084}
2085
2086/// Beamr-scheduler-owning subsystems the full-service construction path builds
2087/// beyond the connection supervisor's own scheduler. The §9 D2 seam census counts
2088/// these: the worker-front-door profile must construct NONE of them. Test-gated:
2089/// this is the recording vocabulary of the gate's instrument, not production state.
2090#[cfg(test)]
2091#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
2092pub(super) enum SchedulerSubsystem {
2093 /// The shared channel supervisor (its own beamr scheduler).
2094 ChannelSupervisor,
2095 /// The conversation supervisor (its own beamr scheduler).
2096 ConversationSupervisor,
2097 /// The haematite store's database (its shard-actor scheduler).
2098 HaematiteStore,
2099}
2100
2101/// Constructor seam for every scheduler-owning subsystem (`SchedulerSubsystem`)
2102/// the profile-aware construction path can create.
2103///
2104/// These methods are the ONLY route through which [`build_connection_services`]
2105/// and the [`LiminalConnectionServices`] config constructors reach
2106/// `build_channel_cluster`, `ConversationSupervisor::new`, and the durable-store
2107/// constructors — no direct constructor call exists in those bodies. The §9 D2
2108/// gate therefore injects a factory that records as a side effect of
2109/// constructing (the D3 store-seam ownership move applied to schedulers): a
2110/// recording cannot be omitted without also failing to construct the subsystem,
2111/// which closes the "hand-placed census call beside the constructor" gap where a
2112/// future subsystem could be built without its courtesy call.
2113pub(super) trait SubsystemFactory {
2114 /// Constructs the shared channel supervisor + cluster resolver (a beamr
2115 /// scheduler).
2116 ///
2117 /// # Errors
2118 /// Returns [`ServerError`] when the channel supervisor scheduler cannot start.
2119 fn channel_cluster(
2120 &self,
2121 cluster_config: Option<&ClusterConfig>,
2122 ) -> Result<ChannelCluster, ServerError>;
2123
2124 /// Constructs the conversation supervisor (a beamr scheduler).
2125 ///
2126 /// # Errors
2127 /// Returns [`ServerError`] when the conversation supervisor scheduler cannot
2128 /// start.
2129 fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError>;
2130
2131 /// Constructs the persistent durable store under the explicit directory.
2132 ///
2133 /// # Errors
2134 /// Returns [`ServerError`] when the store cannot be opened.
2135 fn durable_store(&self, persistence_path: &Path) -> Result<Arc<dyn DurableStore>, ServerError>;
2136
2137 /// The opener of the throwaway store an incarnation's ephemeral dedup
2138 /// namespace lives in when the durable store outlives the process. The
2139 /// store is opened by the namespace's first write, never by this call.
2140 fn throwaway_store(&self) -> ThrowawayStoreOpener;
2141
2142 fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
2143 None
2144 }
2145}
2146
2147/// Test construction factory for callers that own their fixture directories.
2148#[cfg(any(test, feature = "test-support"))]
2149pub(super) struct ProductionSubsystems;
2150
2151#[cfg(any(test, feature = "test-support"))]
2152impl SubsystemFactory for ProductionSubsystems {
2153 fn channel_cluster(
2154 &self,
2155 cluster_config: Option<&ClusterConfig>,
2156 ) -> Result<ChannelCluster, ServerError> {
2157 build_channel_cluster(cluster_config)
2158 }
2159
2160 fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2161 Ok(Arc::new(ConversationSupervisor::new().map_err(
2162 |error| ServerError::ConfigValidation {
2163 message: format!("failed to start conversation supervisor: {error}"),
2164 },
2165 )?))
2166 }
2167
2168 fn durable_store(&self, persistence_path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2169 build_durable_store(persistence_path)
2170 }
2171
2172 fn throwaway_store(&self) -> ThrowawayStoreOpener {
2173 Box::new(|| open_ephemeral(THROWAWAY_SHARD_COUNT))
2174 }
2175}
2176
2177struct RootedProductionSubsystems {
2178 root: Arc<EphemeralRoot>,
2179}
2180
2181impl RootedProductionSubsystems {
2182 fn new(path: &Path) -> Result<Self, ServerError> {
2183 let root = EphemeralRoot::open(path).map_err(|error| ServerError::ConfigValidation {
2184 message: error.to_string(),
2185 })?;
2186 Ok(Self { root })
2187 }
2188}
2189
2190impl SubsystemFactory for RootedProductionSubsystems {
2191 fn channel_cluster(
2192 &self,
2193 config: Option<&ClusterConfig>,
2194 ) -> Result<ChannelCluster, ServerError> {
2195 build_channel_cluster(config)
2196 }
2197
2198 fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2199 Ok(Arc::new(ConversationSupervisor::new().map_err(
2200 |error| ServerError::ConfigValidation {
2201 message: format!("failed to start conversation supervisor: {error}"),
2202 },
2203 )?))
2204 }
2205
2206 fn durable_store(&self, path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2207 build_durable_store(path)
2208 }
2209
2210 fn throwaway_store(&self) -> ThrowawayStoreOpener {
2211 let root = Arc::clone(&self.root);
2212 Box::new(move || root.open_store(THROWAWAY_SHARD_COUNT))
2213 }
2214
2215 fn ephemeral_sweep_counts(&self) -> Option<EphemeralSweepCounts> {
2216 Some(self.root.sweep_counts())
2217 }
2218}
2219
2220/// Test-only recording implementation of [`SubsystemFactory`] for the §9 D2
2221/// construction gate. Lives here (not in a test module's private scope) so the
2222/// supervisor-level gate test reuses the same instrument.
2223#[cfg(test)]
2224#[allow(clippy::expect_used)]
2225pub(super) mod subsystem_census {
2226 use std::path::{Path, PathBuf};
2227 use std::sync::{Arc, Mutex};
2228
2229 use liminal::conversation::ConversationSupervisor;
2230 use liminal::durability::{DurableStore, open_ephemeral_rooted};
2231
2232 use super::{
2233 ChannelCluster, ProductionSubsystems, SchedulerSubsystem, SubsystemFactory,
2234 THROWAWAY_SHARD_COUNT, ThrowawayStoreOpener, build_durable_store,
2235 };
2236 use crate::ServerError;
2237 use crate::config::types::ClusterConfig;
2238
2239 /// Records each [`SchedulerSubsystem`] AS A SIDE EFFECT of constructing it,
2240 /// then hands back the production-constructed subsystem (with ephemeral
2241 /// stores rooted in an isolated directory, so the fs half of the gate is a
2242 /// real negative assertion). Because [`SubsystemFactory`] is the only route
2243 /// the profile-aware construction path has to these constructors, a
2244 /// recording cannot be omitted without also failing to construct — the
2245 /// record-by-construction guarantee.
2246 pub struct RecordingSubsystems {
2247 census: Mutex<Vec<SchedulerSubsystem>>,
2248 ephemeral_root: PathBuf,
2249 }
2250
2251 impl RecordingSubsystems {
2252 /// A recording factory whose ephemeral stores live under `ephemeral_root`.
2253 pub fn rooted(ephemeral_root: &Path) -> Self {
2254 Self {
2255 census: Mutex::new(Vec::new()),
2256 ephemeral_root: ephemeral_root.to_path_buf(),
2257 }
2258 }
2259
2260 /// The recorded construction census, sorted for order-independent
2261 /// comparison.
2262 pub fn recorded(&self) -> Vec<SchedulerSubsystem> {
2263 let mut recorded = self
2264 .census
2265 .lock()
2266 .expect("subsystem census lock is never poisoned in tests")
2267 .clone();
2268 recorded.sort();
2269 recorded
2270 }
2271
2272 fn record(&self, subsystem: SchedulerSubsystem) {
2273 self.census
2274 .lock()
2275 .expect("subsystem census lock is never poisoned in tests")
2276 .push(subsystem);
2277 }
2278 }
2279
2280 impl SubsystemFactory for RecordingSubsystems {
2281 fn channel_cluster(
2282 &self,
2283 cluster_config: Option<&ClusterConfig>,
2284 ) -> Result<ChannelCluster, ServerError> {
2285 let cluster = ProductionSubsystems.channel_cluster(cluster_config)?;
2286 self.record(SchedulerSubsystem::ChannelSupervisor);
2287 Ok(cluster)
2288 }
2289
2290 fn conversation_supervisor(&self) -> Result<Arc<ConversationSupervisor>, ServerError> {
2291 let supervisor = ProductionSubsystems.conversation_supervisor()?;
2292 self.record(SchedulerSubsystem::ConversationSupervisor);
2293 Ok(supervisor)
2294 }
2295
2296 fn durable_store(
2297 &self,
2298 persistence_path: &Path,
2299 ) -> Result<Arc<dyn DurableStore>, ServerError> {
2300 let store = build_durable_store(persistence_path)?;
2301 self.record(SchedulerSubsystem::HaematiteStore);
2302 Ok(store)
2303 }
2304
2305 fn throwaway_store(&self) -> ThrowawayStoreOpener {
2306 let root = self.ephemeral_root.clone();
2307 Box::new(move || open_ephemeral_rooted(&root, THROWAWAY_SHARD_COUNT))
2308 }
2309 }
2310}
2311
2312/// Full-only constructor guard: rejects a config whose profile is not `Full`.
2313///
2314/// [`LiminalConnectionServices`]' config-based constructors call this at entry so
2315/// the full service stack can never be built for a worker-front-door config —
2316/// profile enforcement holds on every public construction path, not only the
2317/// file-loading pipeline.
2318fn require_full_profile(config: &ServerConfig) -> Result<(), ServerError> {
2319 match config.services.profile()? {
2320 ServiceProfile::Full => Ok(()),
2321 ServiceProfile::WorkerFrontDoor => Err(ServerError::ConfigValidation {
2322 message: format!(
2323 "services.profile: \"{}\" cannot construct the full LiminalConnectionServices; \
2324 build profile-selected services via build_connection_services",
2325 ServiceProfile::WORKER_FRONT_DOOR
2326 ),
2327 }),
2328 }
2329}
2330
2331/// Builds the connection-services adapter selected by `config`'s service profile.
2332///
2333/// `Full` builds [`LiminalConnectionServices`] (channels, conversations, durable
2334/// store, dedup cache) exactly as today. `WorkerFrontDoor` builds
2335/// [`WorkerFrontDoorServices`], which constructs none of that machinery. This is the
2336/// single profile-dispatch authority: [`super::supervisor::ConnectionSupervisor`]'s
2337/// config constructor and the standalone runtime's worker arm both route through it
2338/// (the runtime's full arm stays on the explicit
2339/// [`LiminalConnectionServices::from_config`] path because it also needs the shared
2340/// channel cluster, which the trait object does not expose — that path is guarded
2341/// full-only at entry).
2342///
2343/// # Errors
2344/// Returns [`ServerError`] when the selected adapter cannot be constructed, the
2345/// configured profile value is not recognised, or the worker-front-door profile is
2346/// combined with full-only config fields.
2347pub fn build_connection_services(
2348 config: &ServerConfig,
2349) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2350 match config.services.profile()? {
2351 ServiceProfile::Full => {
2352 let subsystems = RootedProductionSubsystems::new(required_persistence_path(config)?)?;
2353 build_connection_services_via(config, &subsystems)
2354 }
2355 ServiceProfile::WorkerFrontDoor => build_worker_front_door(config),
2356 }
2357}
2358
2359/// [`build_connection_services`] with the subsystem factory injected — the §9 D2
2360/// gate seam, both halves at once.
2361///
2362/// Every scheduler-owning subsystem the `Full` branch creates is constructed
2363/// through `subsystems` and nowhere else, so a recording factory observes exactly
2364/// what was built (thread half, record-by-construction); the gate's factory also
2365/// roots its ephemeral stores in an isolated directory, so "the root stays empty
2366/// on the worker branch" is a real negative assertion (fs half, the D3 pattern).
2367/// The `WorkerFrontDoor` branch never touches the factory — a regression that gave
2368/// the front door any subsystem would both record in the census and land a store
2369/// directory in the injected root.
2370///
2371/// The worker branch re-runs the cross-field checks here, not only in file-loading
2372/// validation, so a directly-constructed config cannot smuggle full-only machinery
2373/// past the profile.
2374pub(super) fn build_connection_services_via(
2375 config: &ServerConfig,
2376 subsystems: &dyn SubsystemFactory,
2377) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2378 match config.services.profile()? {
2379 ServiceProfile::Full => {
2380 let path = required_persistence_path(config)?;
2381 let history = StoreHistory::of(path);
2382 let store = subsystems.durable_store(path)?;
2383 Ok(Arc::new(
2384 LiminalConnectionServices::from_config_with_store_via(
2385 config, store, history, subsystems,
2386 )?,
2387 ))
2388 }
2389 ServiceProfile::WorkerFrontDoor => build_worker_front_door(config),
2390 }
2391}
2392
2393fn build_worker_front_door(
2394 config: &ServerConfig,
2395) -> Result<Arc<dyn ConnectionServices>, ServerError> {
2396 let errors = crate::config::validation::worker_front_door_field_errors(config);
2397 if !errors.is_empty() {
2398 return Err(ServerError::ConfigValidation {
2399 message: errors.join("; "),
2400 });
2401 }
2402 Ok(Arc::new(WorkerFrontDoorServices::new()))
2403}
2404
2405/// Opens the persistent durable store under the configured directory.
2406pub(crate) fn open_persistent_store(
2407 persistence_path: &Path,
2408) -> Result<Arc<dyn DurableStore>, ServerError> {
2409 build_durable_store(persistence_path)
2410}
2411
2412/// Every append is permanent through its WAL frames and one device flush
2413/// before it returns under either commit policy; what the policy decides is
2414/// when the shard folds its staged appends into a new tree version, the fold
2415/// that writes the leaf and every internal node above it as fresh files. Per
2416/// append, one keyed publish left 8 new node files (82,137 bytes) on the J3
2417/// store (NODE-FILES-PER-POST, check 15 on Dean at 68a08b35, 10 October
2418/// 2026), and the live spine reached 2,856,796 files that way. Buffered at
2419/// 1 MiB, Waffles' number (10 October 2026 11:19): the threshold haematite's
2420/// own `LeafLog` runs on, and the most an unclean restart replays from staged
2421/// frames per shard. The store's own reads merge the buffer, so a publish
2422/// reads back at once; a clean stop commits what is staged; no policy carries
2423/// a timer.
2424const SPINE_COMMIT_THRESHOLD_BYTES: NonZeroU64 = match NonZeroU64::new(1 << 20) {
2425 Some(threshold) => threshold,
2426 None => unreachable!(),
2427};
2428
2429/// The commit policy the spine's event store opens under.
2430const SPINE_COMMIT_POLICY: CommitPolicy = CommitPolicy::buffered(SPINE_COMMIT_THRESHOLD_BYTES);
2431
2432fn build_durable_store(path: &Path) -> Result<Arc<dyn DurableStore>, ServerError> {
2433 let data_dir = durable_data_dir(path);
2434 let database = open_or_create_database(&data_dir)?;
2435 let event_store = EventStore::with_commit_policy(database, SPINE_COMMIT_POLICY);
2436 Ok(Arc::new(CountingStore::new(HaematiteStore::new(Arc::new(
2437 event_store,
2438 )))))
2439}
2440
2441/// The directory the persistent database under `persistence_path` lives in.
2442pub(super) fn durable_data_dir(persistence_path: &Path) -> PathBuf {
2443 persistence_path.join("durability")
2444}
2445
2446/// Whether a haematite database already exists at `data_dir`: its
2447/// `config.json` is written when it is created.
2448pub(super) fn database_exists(data_dir: &Path) -> bool {
2449 database_config_file(data_dir).exists()
2450}
2451
2452/// What creating a haematite database leaves in its directory. A shard's
2453/// directory appears beside them the first time anything opens the shard, so
2454/// a database holding nothing else has never been read or written.
2455const UNTOUCHED_DATABASE_ENTRIES: [&str; 2] = ["config.json", "writer.lock"];
2456
2457/// Whether the database at `data_dir` holds only what its creation wrote.
2458/// Anything this cannot read, or does not recognise, answers `false`: the
2459/// database is then taken to hold what an earlier server wrote.
2460pub(super) fn database_untouched(data_dir: &Path) -> bool {
2461 std::fs::read_dir(data_dir).is_ok_and(|entries| {
2462 entries.into_iter().all(|entry| {
2463 entry.is_ok_and(|entry| {
2464 entry
2465 .file_name()
2466 .to_str()
2467 .is_some_and(|name| UNTOUCHED_DATABASE_ENTRIES.contains(&name))
2468 })
2469 })
2470 })
2471}
2472
2473fn database_config_file(data_dir: &Path) -> PathBuf {
2474 data_dir.join("config.json")
2475}
2476
2477/// Every commit writes new copies of the nodes it changes; the durable store
2478/// deletes the copies its committed root no longer reaches while it runs, so it
2479/// holds its live records and not every version ever written. A store created
2480/// under `KeepAll` is moved onto this retention by its next open.
2481pub(super) const DURABLE_RETENTION: Retention = Retention::reclaim(0);
2482
2483/// Opens an existing haematite database at `data_dir`, or creates one.
2484fn open_or_create_database(data_dir: &Path) -> Result<Database, ServerError> {
2485 let config_file = database_config_file(data_dir);
2486 let result = if config_file.exists() {
2487 adopt_missing_config_keys(data_dir, &config_file)?;
2488 Database::open_with_retention(
2489 data_dir,
2490 haematite::RuntimeOptions {
2491 long_wait_interval: std::time::Duration::from_secs(5),
2492 // DUR-021 R1: no pass starts and no step runs until the
2493 // listeners are bound and runtime.rs releases it.
2494 reclaim_start: haematite::ReclaimStart::Held,
2495 },
2496 DURABLE_RETENTION,
2497 )
2498 } else {
2499 Database::create(
2500 DatabaseConfig {
2501 data_dir: data_dir.to_path_buf(),
2502 shard_count: DEFAULT_SHARD_COUNT,
2503 distributed: None,
2504 executor_threads: None,
2505 // haematite 0.8.3 requires this field and refuses `None` at
2506 // validation; `Unlimited` is the crate's explicit spelling of the
2507 // pre-budget (0.8.1) behaviour — the behaviour-preserving choice
2508 // for a dependency hop. A real byte ceiling is a deployment
2509 // decision for its own measured lane. A store whose `config.json`
2510 // predates the field is migrated by the open arm above
2511 // (`adopt_missing_config_keys`) — same value, written once, on
2512 // upgrade.
2513 node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
2514 retention: DURABLE_RETENTION,
2515 },
2516 haematite::RuntimeOptions {
2517 long_wait_interval: std::time::Duration::from_secs(5),
2518 // DUR-021 R1: no pass starts and no step runs until the
2519 // listeners are bound and runtime.rs releases it.
2520 reclaim_start: haematite::ReclaimStart::Held,
2521 },
2522 )
2523 };
2524 let database = result.map_err(|error| ServerError::ConfigValidation {
2525 message: format!(
2526 "failed to open durable store at {}: {error}",
2527 data_dir.display()
2528 ),
2529 })?;
2530 liminal::durability::record_store_opened();
2531 super::reclaim_placement::arm_watch(&database);
2532 Ok(database)
2533}
2534
2535/// The stored keys this crate's create arm always writes, with the value it
2536/// writes, for a `config.json` written before haematite had the key.
2537///
2538/// - `node_cache_budget`: haematite 0.8.3 refuses a store without it
2539/// (`MissingNodeCacheBudget`); `"unlimited"` is the pre-budget behaviour.
2540/// - `retention`: `"keep_all"` is the 0.14.1 behaviour, written in so the
2541/// store's stored retention is explicit before the open. The open itself is
2542/// [`Database::open_with_retention`] under [`DURABLE_RETENTION`], which then
2543/// records reclaim in place of it, so an adopted store reclaims from that
2544/// same open onward (spine reclaim, Hermes's ruling of 6 October 2026:
2545/// `past_roots` 0).
2546const ADOPTED_CONFIG_KEYS: [(&str, &str); 2] = [
2547 ("node_cache_budget", "unlimited"),
2548 ("retention", "keep_all"),
2549];
2550
2551/// Writes the values of [`ADOPTED_CONFIG_KEYS`] into a `config.json` that
2552/// lacks any of them, so a store created before haematite had the key opens on
2553/// upgrade with no operator hand. The open that follows then moves its
2554/// retention onto [`DURABLE_RETENTION`].
2555///
2556/// This crate IS the caller that owns these decisions, and it already makes
2557/// them on every create, so answering them here, once, on upgrade, changes
2558/// nothing about how the store runs and takes the file off the operator's
2559/// desk (the store migrates itself or it doesn't ship).
2560///
2561/// Touches the file in exactly one case: a readable JSON object missing one of
2562/// those keys. Anything else — unreadable bytes, a non-object, an explicit
2563/// value however wrong — is left byte-identical so [`Database::open`] surfaces
2564/// haematite's own sharpest refusal instead of this function guessing over
2565/// it. The write is atomic and durable (tmp file beside the target, fsync,
2566/// rename, directory fsync): a crash mid-migration leaves the original bytes,
2567/// and the next boot migrates again.
2568fn adopt_missing_config_keys(data_dir: &Path, config_file: &Path) -> Result<(), ServerError> {
2569 let Ok(bytes) = std::fs::read(config_file) else {
2570 return Ok(());
2571 };
2572 let Ok(mut config) = serde_json::from_slice::<serde_json::Value>(&bytes) else {
2573 return Ok(());
2574 };
2575 let Some(object) = config.as_object_mut() else {
2576 return Ok(());
2577 };
2578 let mut adopted = Vec::new();
2579 for (key, value) in ADOPTED_CONFIG_KEYS {
2580 if !object.contains_key(key) {
2581 object.insert(key.to_owned(), serde_json::Value::String(value.to_owned()));
2582 adopted.push(key);
2583 }
2584 }
2585 if adopted.is_empty() {
2586 return Ok(());
2587 }
2588 let keys = adopted.join(", ");
2589 let migrated =
2590 serde_json::to_vec_pretty(&config).map_err(|error| ServerError::ConfigValidation {
2591 message: format!(
2592 "durable store config at {} predates `{keys}` and its migration could not be \
2593 serialized: {error}",
2594 config_file.display()
2595 ),
2596 })?;
2597 let write_error = |step: &str, error: std::io::Error| ServerError::ConfigValidation {
2598 message: format!(
2599 "durable store config at {} predates `{keys}` and migrating it failed at {step}: \
2600 {error}",
2601 config_file.display()
2602 ),
2603 };
2604 let tmp_file = data_dir.join("config.json.migrate-tmp");
2605 let mut tmp = std::fs::File::create(&tmp_file)
2606 .map_err(|error| write_error("creating the temporary file", error))?;
2607 std::io::Write::write_all(&mut tmp, &migrated)
2608 .map_err(|error| write_error("writing the temporary file", error))?;
2609 tmp.sync_all()
2610 .map_err(|error| write_error("syncing the temporary file", error))?;
2611 drop(tmp);
2612 std::fs::rename(&tmp_file, config_file)
2613 .map_err(|error| write_error("renaming over config.json", error))?;
2614 std::fs::File::open(data_dir)
2615 .and_then(|dir| dir.sync_all())
2616 .map_err(|error| write_error("syncing the data directory", error))?;
2617 tracing::warn!(
2618 config = %config_file.display(),
2619 keys = %keys,
2620 "durable store config predates these keys -- wrote the create-arm values (the \
2621 behaviour-preserving ones) so the store opens with no operator action"
2622 );
2623 Ok(())
2624}
2625
2626pub(super) fn reserved_namespace_refusal(channel: &str) -> Option<ServerError> {
2627 if channel.starts_with("liminal.") {
2628 crate::metrics::self_stream_reserved_publication();
2629 Some(ServerError::ReservedNamespace {
2630 channel: channel.to_owned(),
2631 })
2632 } else {
2633 None
2634 }
2635}
2636
2637impl LiminalConnectionServices {
2638 pub(in crate::server) const fn signals(&self) -> &ServerSignals {
2639 &self.signals
2640 }
2641
2642 pub(in crate::server) fn self_stream_head(&self, name: &str) -> Result<u64, ServerError> {
2643 let channels = self.read_channels()?;
2644 let channel = channels
2645 .get(name)
2646 .ok_or_else(|| ServerError::ListenerAccept {
2647 message: format!("self stream '{name}' is absent after append"),
2648 })?;
2649 let heads =
2650 channel
2651 .handle
2652 .durable_positions()
2653 .map_err(|error| ServerError::ListenerAccept {
2654 message: format!("self stream '{name}' head is unavailable: {error}"),
2655 })?;
2656 drop(channels);
2657 match heads.as_slice() {
2658 [head] => Ok(*head),
2659 _ => Err(ServerError::ListenerAccept {
2660 message: format!("self stream '{name}' must have exactly one durable partition"),
2661 }),
2662 }
2663 }
2664
2665 pub(in crate::server) fn publish_self_stream_batch(
2666 &self,
2667 payloads: &[Vec<u8>],
2668 publisher: &str,
2669 ) -> Result<u64, ServerError> {
2670 if publisher.trim().is_empty() || payloads.is_empty() {
2671 return Err(ServerError::ConfigValidation {
2672 message: "self-stream batch requires a publisher and rows".to_owned(),
2673 });
2674 }
2675 let configured = self
2676 .admit_channel("liminal.events")
2677 .map_err(|error| access_to_server_error(&error))?;
2678 let deliveries = configured
2679 .handle
2680 .publish_batch_with_delivery(
2681 payloads,
2682 &liminal::envelope::PublisherId::new(publisher),
2683 None,
2684 )
2685 .map_err(|error| match error {
2686 liminal::error::LiminalError::PublishUnresolved { append } => {
2687 ServerError::PublishUnresolved {
2688 channel: "liminal.events".to_owned(),
2689 idempotency_key: None,
2690 sequence: Some(append.sequence()),
2691 message: append.to_string(),
2692 }
2693 }
2694 error => ServerError::ListenerAccept {
2695 message: format!("self-stream receipt batch failed: {error}"),
2696 },
2697 })?;
2698 if deliveries.len() != payloads.len()
2699 || deliveries
2700 .iter()
2701 .any(|delivery| delivery.durable_position().is_none())
2702 {
2703 return Err(ServerError::ListenerAccept {
2704 message: "self-stream receipt batch was refused before append".to_owned(),
2705 });
2706 }
2707 for pair in deliveries.windows(2) {
2708 if pair[0]
2709 .durable_position()
2710 .and_then(|position| position.checked_add(1))
2711 != pair[1].durable_position()
2712 {
2713 return Err(ServerError::ListenerAccept {
2714 message: "self-stream receipt batch positions are not contiguous".to_owned(),
2715 });
2716 }
2717 }
2718 for delivery in &deliveries {
2719 self.delivered(delivery);
2720 }
2721 deliveries
2722 .last()
2723 .and_then(ChannelDelivery::durable_position)
2724 .and_then(|position| position.checked_add(1))
2725 .ok_or_else(|| ServerError::ListenerAccept {
2726 message: "self-stream receipt batch head is unavailable".to_owned(),
2727 })
2728 }
2729
2730 pub(in crate::server) fn checkpoint_self_stream_heads(
2731 &self,
2732 heads: [u64; 2],
2733 ) -> Result<(), ServerError> {
2734 for (name, head) in ["liminal.events:0", "liminal.cost:0"]
2735 .into_iter()
2736 .zip(heads)
2737 {
2738 block_on(liminal::durability::head_checkpoint::advance_head(
2739 self.durable_store.as_ref(),
2740 name,
2741 head,
2742 ))
2743 .map_err(|error| ServerError::ShutdownFlush {
2744 message: format!("self-stream head checkpoint bridge for '{name}' failed: {error}"),
2745 })?
2746 .map_err(|error| ServerError::ShutdownFlush {
2747 message: format!("self-stream head checkpoint for '{name}' failed: {error}"),
2748 })?;
2749 }
2750 Ok(())
2751 }
2752
2753 pub(in crate::server) fn install_self_stream_channels(&self) -> Result<(), ServerError> {
2754 let names = ["liminal.cost", "liminal.events"];
2755 {
2756 let channels = self.read_channels()?;
2757 for name in names {
2758 if channels.contains_key(name) {
2759 return Err(ServerError::ConfigValidation {
2760 message: format!("self-stream channel '{name}' is already configured"),
2761 });
2762 }
2763 }
2764 }
2765 let mut built = Vec::with_capacity(names.len());
2766 for name in names {
2767 let schema =
2768 resolve_schema_bytes(None).map_err(|error| ServerError::ConfigValidation {
2769 message: format!("self-stream channel '{name}' schema failed: {error}"),
2770 })?;
2771 let channel = build_configured_channel(
2772 name,
2773 schema,
2774 true,
2775 ChannelOrigin::BootConfigured,
2776 &self.durable_store,
2777 self.cluster.supervisor(),
2778 )
2779 .map_err(|error| ServerError::ConfigValidation {
2780 message: error.boot_message(name),
2781 })?;
2782 built.push((name.to_owned(), Arc::new(channel)));
2783 }
2784 let mut channels =
2785 self.channels
2786 .write()
2787 .map_err(|error| ServerError::ConfigValidation {
2788 message: format!("self-stream channel roster lock poisoned: {error}"),
2789 })?;
2790 for (name, _) in &built {
2791 if channels.contains_key(name) {
2792 return Err(ServerError::ConfigValidation {
2793 message: format!("self-stream channel '{name}' was concurrently configured"),
2794 });
2795 }
2796 }
2797 channels.extend(built);
2798 drop(channels);
2799 // SRV-012 R7: every roster channel but `console.*` is a line, and
2800 // these join after the hub's model was built from the roster.
2801 for name in names {
2802 self.console_changed(crate::console::TopologyChange::Channel {
2803 name: name.to_owned(),
2804 life: line_life(true),
2805 state: crate::console::records::LineState::Live,
2806 });
2807 }
2808 Ok(())
2809 }
2810
2811 /// SRV-012 R4: registers `console.topology` as an ephemeral,
2812 /// server-owned channel (never a store write). A channel of that name
2813 /// already configured (ephemeral, since config refuses a durable one) is
2814 /// kept as it is.
2815 pub(in crate::server) fn install_console_channels(&self) -> Result<(), ServerError> {
2816 self.register_console_channel(crate::console::TOPOLOGY_SUBJECT)
2817 .map_err(|error| ServerError::ConfigValidation {
2818 message: format!("console channel registration failed: {error}"),
2819 })
2820 }
2821
2822 /// Puts the ephemeral console channel `name` on the roster unless it is
2823 /// there.
2824 fn register_console_channel(&self, name: &str) -> Result<(), ChannelRegistryError> {
2825 if self.read_roster()?.contains_key(name) {
2826 return Ok(());
2827 }
2828 let schema = resolve_schema_bytes(None).map_err(|error| {
2829 ChannelRegistryError::RosterUnavailable {
2830 message: format!("console channel '{name}' schema failed: {error}"),
2831 }
2832 })?;
2833 let channel = build_configured_channel(
2834 name,
2835 schema,
2836 false,
2837 ChannelOrigin::BootConfigured,
2838 &self.durable_store,
2839 self.cluster.supervisor(),
2840 )
2841 .map_err(|error| error.into_registry_error(name))?;
2842 self.write_roster()?
2843 .entry(name.to_owned())
2844 .or_insert_with(|| Arc::new(channel));
2845 Ok(())
2846 }
2847
2848 /// Publishes on an ordinary channel for an application embedding the server.
2849 ///
2850 /// The live and durable publisher identity is liminal.<host>. Reserved self
2851 /// streams and console channels retain the same refusal as connection publishes.
2852 /// An idempotency key has the same delivery semantics as a connection publish.
2853 ///
2854 /// # Errors
2855 ///
2856 /// Returns a configuration error for an empty or whitespace-only host name,
2857 /// a namespace refusal for a reserved channel, or the existing admission,
2858 /// durability, and publication error.
2859 pub fn publish_embedded(
2860 &self,
2861 host: &str,
2862 channel: &str,
2863 envelope: &MessageEnvelope,
2864 idempotency_key: Option<&str>,
2865 ) -> Result<PublishOutcome, ServerError> {
2866 self.publish_with_origin(
2867 channel,
2868 envelope,
2869 idempotency_key,
2870 PublishOrigin::Host { name: host },
2871 )
2872 .map(|published| published.outcome)
2873 }
2874
2875 pub(in crate::server) fn publish_with_origin(
2876 &self,
2877 channel: &str,
2878 envelope: &MessageEnvelope,
2879 idempotency_key: Option<&str>,
2880 origin: PublishOrigin<'_>,
2881 ) -> Result<OriginPublication, ServerError> {
2882 validate_publication_origin(channel, idempotency_key, origin)?;
2883 // SRV-012 to_stations: only while a console tap is live does the
2884 // fan-out record whom the message reached, in a buffer the tap lends
2885 // and takes back; otherwise the publish is the plain one.
2886 let mut recipients = self.signals.lend_recipients();
2887 // Lane brief ruling 6: a tapped message is first seen as the server
2888 // receives it, before admission; the clock is read only while a tap
2889 // lent a record.
2890 let received_at = recipients.is_some().then(Instant::now);
2891 let result = (|| {
2892 // The roster read holds its guard only long enough to clone the entry's
2893 // `Arc` out; everything below — the dedup bridge and the publish into the
2894 // channel actor — runs with no roster lock held.
2895 //
2896 // This inner admission STAYS even once the connection process consults
2897 // the roster ahead of delegating: this method is public and callable
2898 // without going through a frame at all, and a guard that only exists in
2899 // the caller is not a guard.
2900 let configured = self
2901 .admit_channel(channel)
2902 .map_err(|error| access_to_server_error(&error))?;
2903
2904 // Dedup-on-delivery: a publish carrying an idempotency key is delivered to
2905 // subscribers AT MOST ONCE across re-publishes of the same key. Only a
2906 // fresh `Claimed` decision proceeds to fan-out; a `Completed`/`InFlight`
2907 // decision is a duplicate and is suppressed (no second delivery), which is
2908 // the at-most-once guarantee the aion outbox relies on.
2909 // A1 §5: the namespace is chosen by the channel's DURABILITY, so an
2910 // ephemeral receipt cannot outlive the message it vouches for.
2911 // A keyed publish to a durable channel claims its key WITH the channel
2912 // as evidence and holds the claim across an unresolved append
2913 // (`unresolved.rs`); the ephemeral namespace keeps the claim below.
2914 if let (Some(key), ChannelMode::Durable) =
2915 (idempotency_key, configured.handle.config().mode)
2916 {
2917 return self
2918 .publish_durable_keyed(
2919 &configured,
2920 channel,
2921 (envelope, origin),
2922 key,
2923 recipients.as_mut(),
2924 )
2925 .map(OriginPublication::connection);
2926 }
2927 let dedup = self.dedup_for(&configured);
2928 if let Some(key) = idempotency_key {
2929 if !Self::claim_delivery(dedup, key)? {
2930 // A dedup-suppressed re-publish is still an accepted publish (it is
2931 // assigned a message id), but it reaches no subscriber, so it counts
2932 // toward publishes and not deliveries.
2933 crate::metrics::publish_accepted();
2934 return Ok(OriginPublication::connection(PublishOutcome {
2935 message_id: self.next_message_id(),
2936 delivered: false,
2937 durable_position: None,
2938 reach: Reach::Suppressed,
2939 }));
2940 }
2941 }
2942
2943 let delivery = fan_out(&configured, envelope, origin, recipients.as_mut());
2944 let delivery = match delivery {
2945 Ok(delivery) => delivery,
2946 // An unkeyed durable publish whose append outcome is not known is
2947 // answered as unresolved, never as a definite failure a producer
2948 // would retry into a second row.
2949 Err(liminal::error::LiminalError::PublishUnresolved { append }) => {
2950 return Err(ServerError::PublishUnresolved {
2951 channel: channel.to_owned(),
2952 idempotency_key: None,
2953 sequence: Some(append.sequence()),
2954 message: append.to_string(),
2955 });
2956 }
2957 Err(error) => {
2958 // The claim above appended an `InFlight` entry but the delivery
2959 // failed before `complete_receipt` could run. Release the claim so
2960 // the key is re-claimable; otherwise every re-publish would see
2961 // `InFlight` and be suppressed forever. Best-effort: surface the
2962 // ORIGINAL publish error regardless, but never swallow a release
2963 // failure silently (it leaves the leak intact).
2964 if let Some(key) = idempotency_key {
2965 Self::release_claim(dedup, key);
2966 }
2967 return Err(ServerError::ListenerAccept {
2968 message: format!("liminal publish failed for channel '{channel}': {error}"),
2969 });
2970 }
2971 };
2972
2973 // A1 §5, the two-way dedup mapping. Every publish outcome maps to
2974 // EXACTLY ONE of complete/release — the same shape as the C1 contract:
2975 //
2976 // * **Defer (or Accept) => `complete_receipt`.** The message is
2977 // admitted and bus-held, so the key is done: a producer retrying it
2978 // hits `Completed` and there is no duplicate delivery and no InFlight
2979 // leak. This is the path that makes "re-publishing with the same key
2980 // is safe and is suppressed while the bus owns the message" true.
2981 // * **Reject => `release_claim`.** Aggregation makes a producer-visible
2982 // Reject mean delivered-to-zero, so the correct dedup state is "as if
2983 // never published": tombstone it so a deliberate retry can re-claim.
2984 // `release_claim` never clobbers a receipt, so the at-most-once guard
2985 // survives a concurrent completion racing ahead.
2986 //
2987 // Mixed accept/reject reports Defer and so completes — consistent with
2988 // "at least one delivery happened".
2989 if let Some(key) = idempotency_key {
2990 if delivery.is_admitted() {
2991 block_on(dedup.complete_receipt(key, ProcessingReceipt::new(Vec::new())))
2992 .map_err(|error| ServerError::ListenerAccept {
2993 message: format!(
2994 "dedup receipt bridge failed for key '{key}': {error}"
2995 ),
2996 })?
2997 .map_err(|error| ServerError::ListenerAccept {
2998 message: format!("dedup receipt write failed for key '{key}': {error}"),
2999 })?;
3000 } else {
3001 Self::release_claim(dedup, key);
3002 }
3003 }
3004
3005 let self_row_at = delivery.durable_position().filter(|_| origin.is_server());
3006 Ok(OriginPublication {
3007 outcome: self.delivered(&delivery),
3008 self_row_appended: self_row_at.is_some(),
3009 self_row_at,
3010 })
3011 })();
3012 self.signals.published(
3013 channel,
3014 origin,
3015 envelope,
3016 &result,
3017 recipients.as_ref().zip(received_at),
3018 );
3019 if let Some(buffer) = recipients {
3020 self.signals.give_back(buffer);
3021 }
3022 result
3023 }
3024}
3025
3026/// The library publish of an unkeyed message, recording what it saw in
3027/// `recipients` when the tap lent a record.
3028fn fan_out(
3029 configured: &ConfiguredChannel,
3030 envelope: &MessageEnvelope,
3031 origin: PublishOrigin<'_>,
3032 recipients: Option<&mut liminal::channel::PublishRecord>,
3033) -> Result<liminal::channel::ChannelDelivery, liminal::error::LiminalError> {
3034 let publisher = || publisher_for_origin(origin);
3035 recipients.map_or_else(
3036 || {
3037 configured
3038 .handle
3039 .publish_with_delivery(&envelope.payload, publisher(), None)
3040 },
3041 |recipients| {
3042 configured.handle.publish_recording(
3043 &envelope.payload,
3044 publisher(),
3045 None,
3046 None,
3047 recipients,
3048 )
3049 },
3050 )
3051}
3052
3053fn validate_publication_origin(
3054 channel: &str,
3055 idempotency_key: Option<&str>,
3056 origin: PublishOrigin<'_>,
3057) -> Result<(), ServerError> {
3058 if !origin.is_server() {
3059 refuse_console_namespace(channel).map_err(|error| access_to_server_error(&error))?;
3060 if let Some(error) = reserved_namespace_refusal(channel) {
3061 return Err(error);
3062 }
3063 }
3064 if let PublishOrigin::Host { name } = origin
3065 && name.trim().is_empty()
3066 {
3067 return Err(ServerError::ConfigValidation {
3068 message: "embedded publication requires a nonempty host name".to_owned(),
3069 });
3070 }
3071 if let PublishOrigin::Server { device } = origin {
3072 if device.trim().is_empty() || idempotency_key.is_some() {
3073 return Err(ServerError::ConfigValidation {
3074 message: "server-origin publication requires a device and no idempotency key"
3075 .to_owned(),
3076 });
3077 }
3078 }
3079 Ok(())
3080}
3081
3082fn publisher_for_origin(origin: PublishOrigin<'_>) -> liminal::envelope::PublisherId {
3083 match origin {
3084 PublishOrigin::Connection { .. } => liminal::envelope::PublisherId::default(),
3085 PublishOrigin::Host { name } => {
3086 liminal::envelope::PublisherId::new(format!("liminal.{name}"))
3087 }
3088 PublishOrigin::Server { device } => {
3089 liminal::envelope::PublisherId::new(format!("liminal.{device}"))
3090 }
3091 }
3092}
3093
3094impl ConnectionServices for LiminalConnectionServices {
3095 fn participant_service(&self) -> Option<InstalledParticipantService> {
3096 self.participant_service.clone()
3097 }
3098
3099 /// The roster's admission decision, exposed across the trait boundary as the
3100 /// permission alone.
3101 ///
3102 /// Two methods share the name `admit_channel` on this type: this trait
3103 /// method, and the private inherent funnel it delegates to. They are the
3104 /// same decision at two different boundaries — the funnel returns the ENTRY
3105 /// the decision was made on, because its in-crate callers go on to publish
3106 /// or subscribe through it, and this one returns `()`, because a caller
3107 /// outside the crate is asking whether it MAY, not for the thing itself.
3108 /// Handing the entry out here would make [`ConfiguredChannel`] public
3109 /// surface and hand a frame-level caller a channel handle it has no business
3110 /// holding. Rust resolves an inherent method ahead of a trait method of the
3111 /// same name, so the funnel's existing callers — and the `Self::` call below
3112 /// — reach the funnel, not this method; the trait form is reached only
3113 /// through a `dyn ConnectionServices`, which is exactly the caller it exists
3114 /// for. `an_admitted_operation_the_service_then_refuses_stays_undifferentiated`
3115 /// is the instrument that would catch this delegation turning into a
3116 /// recursion.
3117 ///
3118 /// `operation` is where the answers part: absence and quiesce refuse
3119 /// publishes and subscribes alike, but the console namespace refuses only
3120 /// a connection's publish (a console reads `console.*` as an ordinary
3121 /// subscriber, and only the server may write there), and a tap target
3122 /// refuses only a subscribe. Both are checked before the roster, so a
3123 /// client is told the same thing whether or not the subject is registered
3124 /// yet.
3125 fn admit_channel(
3126 &self,
3127 operation: ChannelOperation,
3128 channel: &str,
3129 ) -> Result<(), ChannelAccessError> {
3130 match operation {
3131 ChannelOperation::Publish => refuse_console_namespace(channel)?,
3132 ChannelOperation::Subscribe => {
3133 refuse_console_tap_target(channel)?;
3134 self.refuse_console_tap_limit(channel)?;
3135 // SRV-012 R4/R6: a tap channel exists while it has a
3136 // subscriber, and only while the console streams run.
3137 let admitted = if self.signals.console_enabled()
3138 && super::channel_registry::console_tap_target(channel).is_some()
3139 {
3140 self.register_console_channel(channel).map_err(|error| {
3141 ChannelAccessError::RosterUnavailable {
3142 message: error.to_string(),
3143 }
3144 })
3145 } else {
3146 Ok(())
3147 }
3148 .and_then(|()| Self::admit_channel(self, channel).map(|_| ()));
3149 if admitted.is_err() {
3150 self.give_back_tap(channel);
3151 }
3152 return admitted;
3153 }
3154 }
3155 Self::admit_channel(self, channel)?;
3156 Ok(())
3157 }
3158
3159 fn publish(
3160 &self,
3161 connection: u64,
3162 channel: &str,
3163 envelope: &MessageEnvelope,
3164 idempotency_key: Option<&str>,
3165 ) -> Result<PublishOutcome, ServerError> {
3166 // The frame path refused this already at admission. This guard is for
3167 // a caller holding the trait directly: a guard that only exists in the
3168 // caller is not a guard.
3169 refuse_console_namespace(channel).map_err(|error| access_to_server_error(&error))?;
3170 self.publish_with_origin(
3171 channel,
3172 envelope,
3173 idempotency_key,
3174 PublishOrigin::Connection { id: connection },
3175 )
3176 .map(|published| published.outcome)
3177 }
3178
3179 fn subscribe(
3180 &self,
3181 channel: &str,
3182 accepted_schemas: &[ProtocolSchemaId],
3183 install: Option<liminal::channel::InboxInstall>,
3184 ) -> Result<ConnectionSubscription, ServerError> {
3185 let subscribed = self.open_subscription(channel, accepted_schemas, install, None);
3186 if subscribed.is_err() {
3187 // SRV-012 R6: a tap slot reserved at admission for a subscribe
3188 // that did not happen is given back, and a tap channel it
3189 // registered with no tap behind it leaves the roster.
3190 self.give_back_tap(channel);
3191 }
3192 subscribed
3193 }
3194
3195 fn unsubscribe(&self, subscription: ConnectionSubscription) -> Result<(), ServerError> {
3196 subscription.unsubscribe()
3197 }
3198
3199 fn open_conversation(
3200 &self,
3201 conversation_id: u64,
3202 subject: &str,
3203 ) -> Result<ConnectionConversation, ServerError> {
3204 self.open_routed_conversation(conversation_id, subject)
3205 }
3206
3207 fn conversation_message(
3208 &self,
3209 conversation: &ConnectionConversation,
3210 envelope: &MessageEnvelope,
3211 op_id: Option<u64>,
3212 ) -> Result<(), ServerError> {
3213 self.routed_conversation_message(conversation, envelope, op_id)
3214 }
3215
3216 fn close_conversation(&self, conversation: ConnectionConversation) -> Result<(), ServerError> {
3217 conversation.close()
3218 }
3219
3220 fn listening(self: Arc<Self>) -> Option<std::thread::JoinHandle<()>> {
3221 self.resolve_claims_behind_listener()
3222 }
3223
3224 fn stop_participant_warm(&self) {
3225 // DUR-020 R5: the warm stops at its next conversation boundary and
3226 // the stop does not wait for it; whatever it had not restored
3227 // restores on the next start. Then the clean stop records the
3228 // capacity snapshot (R3), and the final flush makes it durable.
3229 if let Some(warm) = &self.participant_warm {
3230 warm.stop();
3231 warm.record_clean_stop();
3232 }
3233 }
3234
3235 fn flush_durable_state(&self) -> Result<(), ServerError> {
3236 // A flush observer publishes to a durable channel, so no channel append lock can be held.
3237 // Separate ephemeral dedup state dies with this incarnation; shared dedup state is covered here.
3238 self.signals.with_flush_barrier(|| {
3239 block_on(self.durable_store.flush())
3240 .map_err(|error| ServerError::ShutdownFlush {
3241 message: format!("shared durable store flush bridge failed: {error}"),
3242 })?
3243 .map_err(|error| ServerError::ShutdownFlush {
3244 message: format!("shared durable store flush failed: {error}"),
3245 })?;
3246 Ok(())
3247 })?;
3248 Ok(())
3249 }
3250}
3251
3252/// One roster entry: a channel's library handle, the protocol schema id
3253/// advertised for it, where it came from, and its admission state.
3254#[derive(Debug)]
3255pub(super) struct ConfiguredChannel {
3256 handle: ChannelHandle,
3257 protocol_schema: ProtocolSchemaId,
3258 /// Durable (stored) or ephemeral, as built; the console's line life.
3259 durable: bool,
3260 /// Boot-configured or runtime-registered. Written once at construction and
3261 /// NEVER flipped: it is the only field that predicts what a restart does to
3262 /// this entry, and it defines the population the registration cap counts.
3263 origin: ChannelOrigin,
3264 /// [`STATE_ACTIVE`] or [`STATE_QUIESCED`]. Written at most once, by one
3265 /// `compare_exchange`, so the transition is one-way and exactly one caller
3266 /// can perform it.
3267 state: AtomicU8,
3268 /// The quiesce cause, set STRICTLY BEFORE `state` flips so any reader that
3269 /// observes [`STATE_QUIESCED`] can read it. Written once (the `OnceLock`
3270 /// enforces that structurally), which is why a re-quiesce under a different
3271 /// reason must refuse rather than silently keep or replace one of them.
3272 quiesce_reason: OnceLock<String>,
3273}
3274
3275impl ConfiguredChannel {
3276 /// The entry's admission state as a value.
3277 ///
3278 /// One `Acquire` load and, when quiesced, one read of the already-written
3279 /// reason. Touches no actor: the whole point of recording state on the entry
3280 /// is that observing it cannot spawn the thing being observed.
3281 fn state(&self) -> ChannelState {
3282 if self.state.load(Ordering::Acquire) == STATE_QUIESCED {
3283 ChannelState::Quiesced {
3284 reason: self.recorded_quiesce_reason(),
3285 }
3286 } else {
3287 ChannelState::Active
3288 }
3289 }
3290
3291 /// The recorded quiesce reason, for a caller that has already observed
3292 /// [`STATE_QUIESCED`] with an `Acquire` load.
3293 fn recorded_quiesce_reason(&self) -> String {
3294 self.quiesce_reason
3295 .get()
3296 .map_or_else(|| UNRECORDED_QUIESCE_REASON.to_owned(), Clone::clone)
3297 }
3298
3299 /// Moves this entry from active to quiesced, recording `reason` first.
3300 ///
3301 /// The reason is written to the `OnceLock` BEFORE the `Release`
3302 /// `compare_exchange` that flips the state, so no reader can observe
3303 /// `QUIESCED` without being able to read why. Re-quiescing under the
3304 /// IDENTICAL reason is `Ok(())` — the caller's intent already holds — and a
3305 /// DIFFERENT reason refuses, because the recorded one cannot be replaced
3306 /// without losing a cause and cannot be kept without lying to the caller.
3307 fn quiesce(&self, name: &str, reason: &str) -> Result<(), ChannelRegistryError> {
3308 // Whoever wins the `OnceLock` writes the cause on record; every other
3309 // caller — a re-quiesce or a concurrent racer — is judged against THAT
3310 // reason, never against whether it happened to perform the flip itself.
3311 // Reporting success for having won the CAS would tell a racer whose
3312 // reason lost that its reason took effect.
3313 let recorded = match self.quiesce_reason.set(reason.to_owned()) {
3314 Ok(()) => reason.to_owned(),
3315 Err(_rejected) => self.recorded_quiesce_reason(),
3316 };
3317 // The reason is now readable, so the flip may become visible. The CAS
3318 // result is deliberately unused: the state is `QUIESCED` afterwards
3319 // whether this call or a racer performed the transition, and the answer
3320 // to the caller is governed by the recorded reason above.
3321 let _flipped_here = self.state.compare_exchange(
3322 STATE_ACTIVE,
3323 STATE_QUIESCED,
3324 Ordering::Release,
3325 Ordering::Acquire,
3326 );
3327 if recorded == reason {
3328 return Ok(());
3329 }
3330 Err(ChannelRegistryError::AlreadyQuiesced {
3331 name: name.to_owned(),
3332 reason: recorded,
3333 })
3334 }
3335}
3336
3337#[derive(Debug)]
3338struct LiminalSubscriptionResource {
3339 subscription: liminal::channel::SubscriptionHandle,
3340}
3341
3342impl SubscriptionResource for LiminalSubscriptionResource {
3343 fn try_next_positioned(
3344 &mut self,
3345 ) -> Result<Option<(liminal::envelope::Envelope, Option<u64>)>, ServerError> {
3346 self.subscription
3347 .try_next_positioned()
3348 .map_err(|error| subscription_failure(&error))
3349 }
3350
3351 fn acknowledge_position(&self, position: u64) -> Result<(), ServerError> {
3352 self.subscription
3353 .acknowledge_position(position)
3354 .map_err(|error| subscription_failure(&error))
3355 }
3356
3357 fn consumer_position(&self) -> Result<Option<u64>, ServerError> {
3358 self.subscription
3359 .consumer_position()
3360 .map_err(|error| subscription_failure(&error))
3361 }
3362 fn unsubscribe(self: Box<Self>) -> Result<(), ServerError> {
3363 drop(self.subscription);
3364 Ok(())
3365 }
3366
3367 fn is_overflowed(&self) -> bool {
3368 self.subscription.is_overflowed()
3369 }
3370
3371 fn subscriber_pid(&self) -> Option<u64> {
3372 Some(self.subscription.subscriber_pid())
3373 }
3374
3375 fn has_pending(&self) -> bool {
3376 self.subscription.has_pending()
3377 }
3378
3379 fn try_next(&mut self) -> Option<liminal::envelope::Envelope> {
3380 match self.subscription.try_next() {
3381 Ok(envelope) => envelope,
3382 Err(error) => {
3383 tracing::error!(
3384 %error,
3385 "subscription could not receive its next envelope"
3386 );
3387 None
3388 }
3389 }
3390 }
3391}
3392
3393/// Preserves the library's subscription failure at the connection boundary.
3394fn subscription_failure(error: &liminal::error::LiminalError) -> ServerError {
3395 ServerError::ListenerAccept {
3396 message: format!("liminal subscription: {error}"),
3397 }
3398}
3399
3400/// Renders an admission refusal as the service error the trait's callers expect.
3401///
3402/// The two refusals that already existed keep their EXACT bytes: an absent
3403/// channel is still `channel '<name>' is not configured` and a poisoned roster
3404/// is still `channel roster lock poisoned`, so nothing that reads these messages
3405/// today sees a change. Discrimination lives in the reason code
3406/// ([`ChannelAccessError::reason_code`]), which the connection process carries to
3407/// the wire — this rendering is the degraded path for a caller that reaches the
3408/// service directly and has no code to carry.
3409///
3410/// `Quiesced` has no predecessor string to preserve; it carries the refusal's own
3411/// message, reason included.
3412///
3413/// The connection process renders its admission refusals through this SAME
3414/// function rather than through [`ChannelAccessError`]'s own `Display`. Two
3415/// consequences, both wanted. The bytes on the wire for an absent channel stay
3416/// exactly what they have always been, which is the whole of the semver promise
3417/// this lane makes. And an operation refused AT admission and one refused after
3418/// admission by the service produce the identical message, differing only in the
3419/// reason code — so the code really is the only discriminator, instead of the
3420/// message quietly becoming a second one.
3421pub(super) fn access_to_server_error(error: &ChannelAccessError) -> ServerError {
3422 let message = match error {
3423 ChannelAccessError::NotRegistered { name } => {
3424 format!("channel '{name}' is not configured")
3425 }
3426 ChannelAccessError::RosterUnavailable { message } => message.clone(),
3427 ChannelAccessError::Quiesced { .. }
3428 | ChannelAccessError::ConsoleNamespace { .. }
3429 | ChannelAccessError::ConsoleTapTarget { .. }
3430 | ChannelAccessError::ConsoleTapLimit { .. } => error.to_string(),
3431 };
3432 ServerError::ListenerAccept { message }
3433}
3434
3435/// Refuses a subscribe to `console.tap.<target>` whose target is empty or
3436/// under `console.`: the tap never taps the console's own records.
3437///
3438/// # Errors
3439/// Returns [`ChannelAccessError::ConsoleTapTarget`] naming the channel.
3440fn refuse_console_tap_target(channel: &str) -> Result<(), ChannelAccessError> {
3441 use super::channel_registry::{console_tap_target, is_console_subject};
3442 match console_tap_target(channel) {
3443 Some(target) if target.is_empty() || is_console_subject(target) => {
3444 Err(ChannelAccessError::ConsoleTapTarget {
3445 name: channel.to_owned(),
3446 })
3447 }
3448 _ => Ok(()),
3449 }
3450}
3451
3452/// Refuses a connection's publish under `console.`, the server's own namespace.
3453///
3454/// # Errors
3455/// Returns [`ChannelAccessError::ConsoleNamespace`] naming the channel.
3456fn refuse_console_namespace(channel: &str) -> Result<(), ChannelAccessError> {
3457 if super::channel_registry::is_console_subject(channel) {
3458 return Err(ChannelAccessError::ConsoleNamespace {
3459 name: channel.to_owned(),
3460 });
3461 }
3462 Ok(())
3463}
3464
3465pub(super) fn server_error_from_protocol(error: &ProtocolError) -> ServerError {
3466 ServerError::ListenerAccept {
3467 message: format!("protocol operation failed: {error}"),
3468 }
3469}
3470
3471/// The three pinned registration tests. A CHILD module of `services` because
3472/// test 2 must hold the exact entry the admission funnel handed out.
3473#[cfg(test)]
3474#[path = "services_registry_tests.rs"]
3475mod services_registry_tests;
3476
3477#[cfg(test)]
3478#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
3479mod durable_store_tests {
3480 use liminal::durability::open_ephemeral_rooted;
3481
3482 use super::subsystem_census::RecordingSubsystems;
3483 use super::{
3484 ConnectionServices, DEFAULT_SHARD_COUNT, LiminalConnectionServices, SchedulerSubsystem,
3485 build_connection_services, build_connection_services_via, build_durable_store,
3486 };
3487 use crate::ServerError;
3488 use crate::config::types::{LimitsConfig, ServerConfig, ServicesConfig};
3489
3490 /// Counts directory entries under `root`, for the empty/one-dir assertions
3491 /// on an injected ephemeral root.
3492 fn entry_count(root: &std::path::Path) -> usize {
3493 std::fs::read_dir(root)
3494 .expect("ephemeral root is readable")
3495 .count()
3496 }
3497
3498 /// A minimal channel-free config with the given service `profile`. No channels,
3499 /// routing, persistence, or cluster — the shape both profiles accept (the full
3500 /// profile simply builds an empty channel set; the worker-front-door profile
3501 /// requires exactly this shape).
3502 fn config_with_profile(profile: &str) -> ServerConfig {
3503 ServerConfig {
3504 listen_address: "127.0.0.1:0".parse().expect("valid socket addr"),
3505 health_listen_address: "127.0.0.1:1".parse().expect("valid socket addr"),
3506 drain_timeout_ms: 30_000,
3507 channels: Vec::new(),
3508 routing_rules: Vec::new(),
3509 persistence_path: None,
3510 cluster: None,
3511 auth: None,
3512 services: ServicesConfig {
3513 profile: profile.to_owned(),
3514 ..ServicesConfig::default()
3515 },
3516 limits: LimitsConfig::default(),
3517 participant: None,
3518 websocket: None,
3519 }
3520 }
3521
3522 /// §9 D2 front-door construction gate (fs half): building the worker-front-door
3523 /// services creates NO haematite store and NO temp dir, while the full profile
3524 /// over an equally-rooted factory DOES create exactly one store directory.
3525 ///
3526 /// The injected root is the only place an ephemeral store directory can appear
3527 /// (the recording factory roots its stores there), so "the root stays empty on
3528 /// the front-door branch" is a real negative assertion: a regression that gave
3529 /// the front door a store would land its directory here and fail this test. The
3530 /// full-profile arm is the positive control proving the seam genuinely
3531 /// constructs a store when the profile asks for one.
3532 #[test]
3533 fn worker_front_door_builds_no_store_and_no_temp_dir() {
3534 let front_door_root = tempfile::tempdir().expect("test can create an ephemeral root");
3535 let full_root = tempfile::tempdir().expect("test can create an ephemeral root");
3536
3537 let front_door_subsystems = RecordingSubsystems::rooted(front_door_root.path());
3538 let front_door: std::sync::Arc<dyn ConnectionServices> = build_connection_services_via(
3539 &config_with_profile("worker-front-door"),
3540 &front_door_subsystems,
3541 )
3542 .expect("worker-front-door services build");
3543 assert!(
3544 !front_door.supports_channel_operations(),
3545 "the worker front door serves no channel operations"
3546 );
3547 assert_eq!(
3548 entry_count(front_door_root.path()),
3549 0,
3550 "the worker front door creates no ephemeral store directory (no haematite, no temp dir)"
3551 );
3552
3553 let full_subsystems = RecordingSubsystems::rooted(full_root.path());
3554 let mut config = config_with_profile("full");
3555 config.persistence_path = Some(full_root.path().to_path_buf());
3556 let full =
3557 build_connection_services_via(&config, &full_subsystems).expect("full services build");
3558 assert!(
3559 full.supports_channel_operations(),
3560 "full mode serves channel operations"
3561 );
3562 assert_eq!(
3563 entry_count(full_root.path()),
3564 1,
3565 "full mode builds its persistent database in the explicit directory"
3566 );
3567
3568 drop(front_door);
3569 drop(full);
3570 }
3571
3572 /// §9 D2 front-door construction gate (thread half — record-by-construction
3573 /// census): the worker profile constructs NO channel-supervisor,
3574 /// conversation-supervisor, or haematite scheduler, while the SAME instrument
3575 /// over the full profile records all three — the positive control proving the
3576 /// census detects the extra schedulers, so an empty census on the worker branch
3577 /// is a real observation, not a decoration.
3578 ///
3579 /// The instrument's boundary: recording happens INSIDE the [`SubsystemFactory`]
3580 /// methods that are the profile-aware path's only route to these constructors,
3581 /// so a recording cannot be silently omitted — a future subsystem added to this
3582 /// path either goes through the factory (and is recorded by construction) or
3583 /// bypasses it, which is a code-review-visible structural violation of the
3584 /// factory seam, not a silently-missing side call. The connection supervisor's
3585 /// own scheduler is the shared baseline of both profiles and is asserted at the
3586 /// supervisor level (`supervisor::tests`); an OS-level thread census upgrades
3587 /// this when the beamr composition lane's scheduler-inventory API (currently on
3588 /// their branch, not yet consumable from liminal) lands.
3589 #[test]
3590 fn worker_profile_census_is_empty_and_full_profile_records_all_schedulers() {
3591 let worker_root = tempfile::tempdir().expect("test can create an ephemeral root");
3592 let full_root = tempfile::tempdir().expect("test can create an ephemeral root");
3593
3594 let worker_subsystems = RecordingSubsystems::rooted(worker_root.path());
3595 let front_door = build_connection_services_via(
3596 &config_with_profile("worker-front-door"),
3597 &worker_subsystems,
3598 )
3599 .expect("worker-front-door services build");
3600 assert_eq!(
3601 worker_subsystems.recorded(),
3602 Vec::<SchedulerSubsystem>::new(),
3603 "the worker front door constructs no scheduler-owning subsystem"
3604 );
3605
3606 let full_subsystems = RecordingSubsystems::rooted(full_root.path());
3607 let mut config = config_with_profile("full");
3608 config.persistence_path = Some(full_root.path().to_path_buf());
3609 let full =
3610 build_connection_services_via(&config, &full_subsystems).expect("full services build");
3611 assert_eq!(
3612 full_subsystems.recorded(),
3613 vec![
3614 SchedulerSubsystem::ChannelSupervisor,
3615 SchedulerSubsystem::ConversationSupervisor,
3616 SchedulerSubsystem::HaematiteStore,
3617 ],
3618 "the full profile constructs every scheduler-owning subsystem, once each — \
3619 the positive control proving the census instrument detects them"
3620 );
3621
3622 drop(front_door);
3623 drop(full);
3624 }
3625
3626 /// MAJOR-1 regression: the full-only constructors reject a worker-front-door
3627 /// config with a typed `ConfigValidation` error AT ENTRY — no full service can
3628 /// be created through any public config-based constructor under that profile.
3629 #[test]
3630 fn full_only_constructors_reject_worker_profile() {
3631 let config = config_with_profile("worker-front-door");
3632
3633 let from_config = LiminalConnectionServices::from_config(&config);
3634 assert!(
3635 matches!(from_config, Err(ServerError::ConfigValidation { .. })),
3636 "from_config must reject a worker-front-door profile with ConfigValidation, got {from_config:?}"
3637 );
3638
3639 let root = tempfile::tempdir().expect("test can create an ephemeral root");
3640 let store = open_ephemeral_rooted(root.path(), DEFAULT_SHARD_COUNT)
3641 .expect("test store for the rejection check builds");
3642 let from_config_with_store =
3643 LiminalConnectionServices::from_config_with_store(&config, std::sync::Arc::new(store));
3644 assert!(
3645 matches!(
3646 from_config_with_store,
3647 Err(ServerError::ConfigValidation { .. })
3648 ),
3649 "from_config_with_store must reject a worker-front-door profile with ConfigValidation"
3650 );
3651 }
3652
3653 /// MAJOR-1 regression: the profile-aware factory itself re-runs the
3654 /// worker-front-door cross-field checks, so a directly-constructed config (one
3655 /// that never passed file-loading validation) combining the worker profile with
3656 /// full-only machinery is refused with the same typed `ConfigValidation` errors.
3657 #[test]
3658 fn build_connection_services_rejects_worker_profile_with_full_only_fields() {
3659 let mut config = config_with_profile("worker-front-door");
3660 config.channels = vec![crate::config::types::ChannelDef {
3661 name: "orders".to_owned(),
3662 schema_ref: None,
3663 durable: false,
3664 loaded_schema: None,
3665 }];
3666 config.persistence_path = Some(std::path::PathBuf::from("/tmp"));
3667
3668 let result = build_connection_services(&config);
3669 let Err(ServerError::ConfigValidation { message }) = result else {
3670 panic!("expected ConfigValidation for worker profile with full-only fields");
3671 };
3672 assert!(message.contains("builds no channels"), "got: {message}");
3673 assert!(
3674 message.contains("builds no durable store"),
3675 "got: {message}"
3676 );
3677 }
3678
3679 /// The required-path constructor creates the persistent database in its
3680 /// configured directory without populating the separate throwaway root.
3681 #[test]
3682 fn persistent_store_uses_configured_path_and_creates_no_temp_dir() {
3683 let home = tempfile::tempdir().expect("test can create a temp dir");
3684 let ephemeral_root = tempfile::tempdir().expect("test can create an ephemeral root");
3685
3686 let store = build_durable_store(home.path()).expect("persistent store builds");
3687
3688 assert!(
3689 home.path().join("durability").join("config.json").exists(),
3690 "the persistent database is created under the configured path"
3691 );
3692 assert_eq!(
3693 entry_count(ephemeral_root.path()),
3694 0,
3695 "the persistent branch creates no ephemeral guard directory"
3696 );
3697
3698 drop(store);
3699 }
3700
3701 #[test]
3702 fn missing_path_refuses_before_any_store_is_opened() -> Result<(), Box<dyn std::error::Error>> {
3703 let root = tempfile::tempdir()?;
3704 let factory = RecordingSubsystems::rooted(root.path());
3705 let result = build_connection_services_via(&config_with_profile("full"), &factory);
3706 assert!(
3707 matches!(result, Err(ServerError::ConfigValidation { message }) if message.contains("no store is opened"))
3708 );
3709 assert_eq!(entry_count(root.path()), 0);
3710 assert!(factory.recorded().is_empty());
3711 Ok(())
3712 }
3713
3714 /// The store migrates itself: a `config.json` written before haematite
3715 /// required `node_cache_budget` opens on the new server with no operator
3716 /// hand, and the file afterwards carries the key — spelled exactly as the
3717 /// create arm spells it. The pre-field store is fabricated by creating a
3718 /// current one and deleting the key, which is byte-faithful to what a
3719 /// 0.8.1-era store looks like to the parser: the key absent, everything
3720 /// else in place.
3721 #[test]
3722 fn a_store_whose_config_predates_the_budget_field_opens_with_no_operator_hand() {
3723 let home = tempfile::tempdir().expect("test can create a temp dir");
3724 let store = build_durable_store(home.path()).expect("persistent store builds");
3725 drop(store);
3726
3727 let config_file = home.path().join("durability").join("config.json");
3728 let bytes = std::fs::read(&config_file).expect("config.json is readable");
3729 let mut config: serde_json::Value =
3730 serde_json::from_slice(&bytes).expect("config.json parses");
3731 config
3732 .as_object_mut()
3733 .expect("config.json is an object")
3734 .remove("node_cache_budget")
3735 .expect("the created config carries the budget to remove");
3736 std::fs::write(
3737 &config_file,
3738 serde_json::to_vec_pretty(&config).expect("serializes"),
3739 )
3740 .expect("test can age the config");
3741
3742 let reopened = build_durable_store(home.path())
3743 .expect("a pre-budget store opens with no operator action");
3744 drop(reopened);
3745
3746 let after = std::fs::read(&config_file).expect("config.json is readable after");
3747 let migrated: serde_json::Value =
3748 serde_json::from_slice(&after).expect("migrated config parses");
3749 assert_eq!(
3750 migrated.get("node_cache_budget"),
3751 Some(&serde_json::Value::String("unlimited".to_owned())),
3752 "the migrated file carries the budget the create arm chooses"
3753 );
3754 }
3755
3756 /// The migration touches only the one migratable case. A corrupt
3757 /// `config.json` still refuses — through haematite's own parse refusal,
3758 /// which is sharper than anything this crate could say — and the bytes are
3759 /// left exactly as found, so the operator diagnoses the file that failed,
3760 /// not one the migration half-rewrote.
3761 #[test]
3762 fn a_corrupt_config_still_refuses_and_is_left_untouched() {
3763 let home = tempfile::tempdir().expect("test can create a temp dir");
3764 let data_dir = home.path().join("durability");
3765 std::fs::create_dir_all(&data_dir).expect("test can create the data dir");
3766 let config_file = data_dir.join("config.json");
3767 let garbage = b"{ not json at all";
3768 std::fs::write(&config_file, garbage).expect("test can write garbage");
3769
3770 let refused = build_durable_store(home.path());
3771 assert!(refused.is_err(), "a corrupt config refuses to open");
3772 assert_eq!(
3773 std::fs::read(&config_file)
3774 .expect("config.json is still readable")
3775 .as_slice(),
3776 garbage,
3777 "the refusing file is left byte-identical for the operator to read"
3778 );
3779 }
3780
3781 /// A config already carrying the budget is not rewritten: reopening leaves
3782 /// the file byte-identical, so the migration cannot churn mtimes, reorder
3783 /// keys, or otherwise touch a store that needs nothing.
3784 #[test]
3785 fn a_config_already_carrying_the_budget_is_left_byte_identical() {
3786 let home = tempfile::tempdir().expect("test can create a temp dir");
3787 let store = build_durable_store(home.path()).expect("persistent store builds");
3788 drop(store);
3789
3790 let config_file = home.path().join("durability").join("config.json");
3791 let before = std::fs::read(&config_file).expect("config.json is readable");
3792 let reopened = build_durable_store(home.path()).expect("a current store reopens");
3793 drop(reopened);
3794 assert_eq!(
3795 std::fs::read(&config_file).expect("config.json is readable after"),
3796 before,
3797 "a config that already carries the budget is left byte-identical"
3798 );
3799 }
3800
3801 /// A durable store created under `KeepAll` is moved onto reclaim by the
3802 /// server's next open: its `config.json` names `reclaim` afterwards, and the
3803 /// copies its old commits left are gone once reclaim has run.
3804 #[test]
3805 fn a_keep_all_durable_store_reclaims_after_the_next_open() {
3806 let dir = tempfile::tempdir().expect("test can create a store dir");
3807 let data_dir = dir.path().join("durability");
3808 let database = haematite::Database::create(
3809 haematite::DatabaseConfig {
3810 data_dir: data_dir.clone(),
3811 shard_count: 1,
3812 distributed: None,
3813 executor_threads: None,
3814 node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
3815 retention: haematite::Retention::KeepAll,
3816 },
3817 haematite::RuntimeOptions {
3818 long_wait_interval: std::time::Duration::from_secs(5),
3819 reclaim_start: haematite::ReclaimStart::AtOpen,
3820 },
3821 )
3822 .expect("a keep-all store is created");
3823 for round in 0_u32..200 {
3824 let mut value = round.to_be_bytes().to_vec();
3825 value.resize(1_024, 0x5a);
3826 database.put(b"key".to_vec(), value).expect("put");
3827 database.commit().expect("commit");
3828 }
3829 drop(database);
3830 let grown =
3831 haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir.clone()))
3832 .expect("stats of the grown store")
3833 .totals
3834 .enumerated_bytes;
3835
3836 let reopened = super::open_or_create_database(&data_dir).expect("the server opens it");
3837 // The server holds reclaim at open and releases it once its listeners
3838 // bind (DUR-021 R1); the test releases it the same way.
3839 reopened
3840 .release_reclaim()
3841 .expect("the held reclaim is released");
3842 reopened.quiesce_reclaim().expect("reclaim runs to rest");
3843 drop(reopened);
3844
3845 let config: serde_json::Value = serde_json::from_slice(
3846 &std::fs::read(data_dir.join("config.json")).expect("config is readable"),
3847 )
3848 .expect("config is JSON");
3849 assert_eq!(
3850 config["retention"],
3851 serde_json::to_value(super::DURABLE_RETENTION).expect("the retention serializes")
3852 );
3853 let report = haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir))
3854 .expect("stats of the reclaimed store");
3855 assert!(
3856 report.totals.enumerated_bytes <= report.totals.marked_bytes.saturating_mul(3),
3857 "the reopened store holds about its live set: enumerated {} B, marked {} B",
3858 report.totals.enumerated_bytes,
3859 report.totals.marked_bytes
3860 );
3861 assert!(
3862 report.totals.enumerated_bytes.saturating_mul(10) < grown,
3863 "the old copies are gone: {} B now against {grown} B grown",
3864 report.totals.enumerated_bytes
3865 );
3866 }
3867
3868 /// `config.json` exactly as a store grown by an engine that never reclaimed
3869 /// carries it: `reclaim` with `past_roots` 0 is already written, so the
3870 /// open has no retention to adopt (it does move the format to version 3),
3871 /// and every superseded copy is still on disk.
3872 /// `DATA_DIR` stands for the one field the fixture rewrites.
3873 const STORED_RECLAIM_CONFIG: &str = r#"{
3874 "chunking_policy": {
3875 "chunking": "bytes-v2",
3876 "internal_target_bytes": 49152,
3877 "leaf_target_bytes": 65536
3878 },
3879 "data_dir": "DATA_DIR",
3880 "distributed": null,
3881 "executor_threads": null,
3882 "format_version": 2,
3883 "node_cache_budget": "unlimited",
3884 "retention": {
3885 "reclaim": {
3886 "past_roots": 0
3887 }
3888 },
3889 "shard_count": 8
3890}"#;
3891
3892 /// A store whose `config.json` already names reclaim, but whose history was
3893 /// grown with nothing reclaimed, is opened by the server and gives its old
3894 /// copies back with no write and no quiesce. A store with no reclaim
3895 /// record owes a startup pass, and a shard runs it on its first turn. The
3896 /// seed holds both shapes the pass must free: one row overwritten many
3897 /// times, and rows written and then deleted, as a prune leaves them.
3898 #[test]
3899 fn a_stored_reclaim_store_grown_without_reclaim_gives_its_old_copies_back_on_open() {
3900 let dir = tempfile::tempdir().expect("test can create a store dir");
3901 let data_dir = dir.path().join("durability");
3902 let database = haematite::Database::create(
3903 haematite::DatabaseConfig {
3904 data_dir: data_dir.clone(),
3905 shard_count: 8,
3906 distributed: None,
3907 executor_threads: None,
3908 node_cache_budget: Some(haematite::NodeCacheBudget::Unlimited),
3909 retention: haematite::Retention::KeepAll,
3910 },
3911 haematite::RuntimeOptions {
3912 long_wait_interval: std::time::Duration::from_secs(5),
3913 reclaim_start: haematite::ReclaimStart::AtOpen,
3914 },
3915 )
3916 .expect("a keep-all store is created");
3917 for round in 0_u32..200 {
3918 let mut value = round.to_be_bytes().to_vec();
3919 value.resize(1_024, 0x5a);
3920 database.put(b"key".to_vec(), value).expect("put");
3921 database.commit().expect("commit");
3922 }
3923 for row in 0_u32..200 {
3924 let key = [b"pruned-".as_slice(), &row.to_be_bytes()].concat();
3925 database.put(key, vec![0xa5; 1_024]).expect("put");
3926 database.commit().expect("commit");
3927 }
3928 for row in 0_u32..200 {
3929 let key = [b"pruned-".as_slice(), &row.to_be_bytes()].concat();
3930 database.delete(key).expect("delete");
3931 }
3932 database.commit().expect("commit");
3933 drop(database);
3934 let grown_report =
3935 haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir.clone()))
3936 .expect("stats of the grown store");
3937 let grown = grown_report.totals;
3938 assert!(
3939 grown.unmarked_nodes > 0,
3940 "the seed leaves copies no root reaches"
3941 );
3942 let mut owed: std::collections::BTreeSet<usize> = grown_report
3943 .shards
3944 .iter()
3945 .filter(|shard| shard.nodes.unmarked_nodes > 0)
3946 .map(|shard| shard.shard_id)
3947 .collect();
3948
3949 let config_file = data_dir.join("config.json");
3950 let seeded = STORED_RECLAIM_CONFIG.replace(
3951 "DATA_DIR",
3952 data_dir.to_str().expect("the test dir is UTF-8"),
3953 );
3954 std::fs::write(&config_file, &seeded).expect("config.json is written");
3955
3956 let reopened = super::open_or_create_database(&data_dir).expect("the server opens it");
3957 let watch = reopened
3958 .watch_reclaim_passes(64)
3959 .expect("every shard accepts a pass watch");
3960 // Released as the server releases it once its listeners bind (DUR-021 R1).
3961 reopened
3962 .release_reclaim()
3963 .expect("the held reclaim is released");
3964 let mut unmarked_by_shard = std::collections::BTreeMap::new();
3965 while !owed.is_empty() {
3966 let report = watch
3967 .recv_timeout(std::time::Duration::from_secs(30))
3968 .expect("each pass the open owes ends without a write or a quiesce");
3969 assert_eq!(
3970 report.outcome,
3971 haematite::ReclaimPassOutcome::Passed,
3972 "the pass the open owed on shard {} finished",
3973 report.shard_id
3974 );
3975 if owed.remove(&report.shard_id) {
3976 unmarked_by_shard.insert(report.shard_id, report.unmarked_bytes);
3977 }
3978 }
3979 drop(reopened);
3980
3981 let after = haematite::db::vacuum::stats(&haematite::db::vacuum::Options::new(data_dir))
3982 .expect("stats of the reclaimed store")
3983 .totals;
3984 eprintln!(
3985 "grown {} B, after {} B, {} unreachable nodes seeded, unmarked bytes by shard {unmarked_by_shard:?}",
3986 grown.enumerated_bytes, after.enumerated_bytes, grown.unmarked_nodes
3987 );
3988 assert!(
3989 after.enumerated_bytes.saturating_mul(10) < grown.enumerated_bytes,
3990 "the old copies are gone: {} B now against {} B grown",
3991 after.enumerated_bytes,
3992 grown.enumerated_bytes
3993 );
3994 let config: serde_json::Value = serde_json::from_slice(
3995 &std::fs::read(&config_file).expect("config.json is readable after"),
3996 )
3997 .expect("config is JSON");
3998 assert_eq!(
3999 config["retention"],
4000 serde_json::to_value(super::DURABLE_RETENTION).expect("the retention serializes"),
4001 "config.json names reclaim after the open"
4002 );
4003 }
4004}