1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
use super::{
Arc, Atom, ConnectionIncarnation, ConnectionRuntime, Duration, Frame, Instant,
PushSlotDisposition, PushSlotResolution, Receiver, RecvTimeoutError, ServerError, SocketAddr,
SupervisorInner, TryRecvError, Weak, WorkerRefusal,
};
/// Handle for one supervised connection process.
#[derive(Clone, Debug)]
pub struct ConnectionHandle {
pub(super) pid: u64,
pub(super) peer_addr: Option<SocketAddr>,
pub(super) connection_incarnation: Option<ConnectionIncarnation>,
pub(super) supervisor: Arc<SupervisorInner>,
}
impl ConnectionHandle {
/// Every later report of the frames this connection writes to its
/// transport. Test-only (R3b F8, ruling 2a).
#[cfg(test)]
pub(crate) fn observe_frames_written(&self) -> Receiver<Vec<liminal::protocol::FrameType>> {
self.supervisor.runtime.observe_frames_written(self.pid)
}
/// Returns the beamr process id for this connection.
#[must_use]
pub const fn pid(&self) -> u64 {
self.pid
}
/// Returns the peer address if it was available from the accepted stream.
#[must_use]
pub const fn peer_addr(&self) -> Option<SocketAddr> {
self.peer_addr
}
/// Returns the durable participant connection incarnation, when this
/// supervisor has a complete participant service installed.
///
/// `None` identifies a services adapter that does not advertise participant
/// lifecycle semantics.
#[must_use]
pub const fn connection_incarnation(&self) -> Option<ConnectionIncarnation> {
self.connection_incarnation
}
/// Returns whether the beamr process is still live.
#[must_use]
pub fn is_live(&self) -> bool {
self.supervisor
.scheduler
.process_table()
.get(self.pid)
.is_some()
}
/// Requests an error exit for tests and supervisor control paths.
///
/// # Errors
/// Returns [`ServerError`] when the process is no longer live.
pub fn request_crash(&self) -> Result<(), ServerError> {
if self
.supervisor
.scheduler
.enqueue_atom_message(self.pid, Atom::ERROR)
{
Ok(())
} else {
Err(ServerError::ListenerAccept {
message: format!("connection process {} is not live", self.pid),
})
}
}
}
/// Awaits the correlated reply to a single server-initiated push.
///
/// Returned by [`ConnectionSupervisor::push_to_connection`]. The reply slot is
/// resolved when the originating connection process receives a `PushReply` frame
/// carrying the same correlation id, so [`PushReplyAwaiter::receive`] blocks
/// (bounded) for that one correlated answer.
#[derive(Debug)]
pub struct PushReplyAwaiter {
pub(super) correlation_id: u64,
pub(super) receiver: Receiver<PushSlotResolution>,
/// This push's absolute reply deadline, mirrored from its slot. `None` (the
/// default push) selects the no-deadline receive path, which NEVER touches
/// the runtime — byte-compatible with 0.2.3, no shared-lock exposure.
/// `Some` lets `receive` wait `min(caller quantum, time until deadline)` and
/// resolve expiry promptly, so the caller's quantum can never select a
/// deadlined push's terminal outcome.
pub(super) deadline: Option<Instant>,
/// Weak handle to the owning runtime, used ONLY by the explicit-deadline
/// path to resolve expiry host-side at [`receive`](Self::receive). A
/// no-deadline push never upgrades it. `Weak` so the awaiter never keeps the
/// runtime alive; if it is already gone, the slot (and its sender) is gone
/// with it — the connection side is torn down.
pub(super) runtime: Weak<ConnectionRuntime>,
}
impl PushReplyAwaiter {
/// Returns the correlation id this awaiter is matched on.
#[must_use]
pub const fn correlation_id(&self) -> u64 {
self.correlation_id
}
/// Blocks up to `timeout` for the client's correlated reply payload.
///
/// `timeout` is a WAIT QUANTUM ONLY — a MAXIMUM wait, not a promise to
/// block: an elapsed poll is a benign re-arm, never a failure; the reply's
/// lifetime belongs to the push. A caller may re-invoke `receive`
/// indefinitely after a [`ServerError::PushReplyTimeout`]: the reserved slot
/// is untouched and a later reply is still delivered byte-exact. The poll
/// quantum never changes the protocol outcome — for a deadlined push the
/// call waits no longer than the EARLIER of the caller's quantum and the
/// push's deadline, so the terminal expiry is returned promptly once due,
/// never held until the quantum ends and never deferred past it.
///
/// A push with no explicit deadline never touches shared supervisor state
/// here: the elapsed quantum returns straight from the channel wait
/// (behaviour-compatible with 0.2.3 — no registry lock, no contention, no
/// poison exposure on the unchanged API).
///
/// # Errors
/// Returns [`ServerError::PushReplyTimeout`] when no reply arrived within this
/// `timeout` quantum and the push's deadline (if any) is not yet due (a
/// benign re-arm — call again to keep waiting);
/// [`ServerError::PushReplyExpired`] when the push carried an explicit reply
/// deadline (via
/// [`push_to_connection_with_deadline`](ConnectionSupervisor::push_to_connection_with_deadline))
/// and that deadline is due (terminal: the slot is removed and its §5 cap
/// admission released; returned as soon as the deadline passes, even
/// mid-quantum — but evaluated at observation points, not against the wall
/// clock: a reply already delivered when this call observes the slot wins
/// over expiry, even if it arrived after the deadline instant); or
/// [`ServerError::PushReplyDisconnected`] when the connection process
/// dropped the reply slot (the connection closed — the prompt worker-death
/// signal). The variants are distinct so callers classify by type, not
/// message.
pub fn receive(&self, timeout: Duration) -> Result<Vec<u8>, ServerError> {
self.deadline.map_or_else(
|| self.receive_no_deadline(timeout),
|deadline| self.receive_deadlined(timeout, deadline),
)
}
/// The default-push receive: exactly the 0.2.3 shape. One bounded channel
/// wait; an elapsed quantum is a benign timeout straight from the channel —
/// no runtime upgrade, no registry lock, EVER (unrelated registry work can
/// never stretch this call past its quantum, and registry poison cannot
/// reach it).
fn receive_no_deadline(&self, timeout: Duration) -> Result<Vec<u8>, ServerError> {
match self.receiver.recv_timeout(timeout) {
Ok(resolution) => self.resolution_into_result(resolution),
Err(RecvTimeoutError::Timeout) => Err(ServerError::PushReplyTimeout {
correlation_id: self.correlation_id,
}),
Err(RecvTimeoutError::Disconnected) => Err(ServerError::PushReplyDisconnected {
correlation_id: self.correlation_id,
}),
}
}
/// The deadlined receive: waits `min(caller quantum, time until deadline)`
/// and re-evaluates reply-first-then-expiry on every wake, so the caller's
/// quantum can never select the terminal outcome (S1). Order per iteration:
///
/// 1. Deliver a reply already in hand — an answer that is here must never be
/// reported as a timeout OR an expiry (the observation-point rule).
/// 2. If the deadline is due, resolve expiry atomically against the registry
/// (`expire_slot`) and return the terminal outcome promptly — even when
/// the caller's quantum has time left (the quantum is a max wait).
/// 3. Otherwise wait for the earlier of quantum-remaining and deadline; a
/// wake re-runs 1-2, and an exhausted quantum before the deadline is the
/// benign `PushReplyTimeout` re-arm with the slot untouched.
fn receive_deadlined(
&self,
timeout: Duration,
deadline: Instant,
) -> Result<Vec<u8>, ServerError> {
let started = Instant::now();
loop {
if let Some(result) = self.try_take_reply() {
return result;
}
let now = Instant::now();
if now >= deadline {
return self.expire_slot();
}
let quantum_left = timeout.saturating_sub(now.duration_since(started));
if quantum_left.is_zero() {
return Err(ServerError::PushReplyTimeout {
correlation_id: self.correlation_id,
});
}
match self
.receiver
.recv_timeout(quantum_left.min(deadline.duration_since(now)))
{
Ok(resolution) => return self.resolution_into_result(resolution),
Err(RecvTimeoutError::Disconnected) => {
return Err(ServerError::PushReplyDisconnected {
correlation_id: self.correlation_id,
});
}
// Re-loop: deliver a reply that raced the wake, expire a
// now-due deadline, or report the exhausted quantum benignly.
Err(RecvTimeoutError::Timeout) => {}
}
}
}
/// Resolves a due deadline against the registry's atomic removal transition.
fn expire_slot(&self) -> Result<Vec<u8>, ServerError> {
let timeout_error = || ServerError::PushReplyTimeout {
correlation_id: self.correlation_id,
};
let Some(runtime) = self.runtime.upgrade() else {
// The runtime is gone, and the slot map (with every sender) with it:
// the connection side is torn down. Re-check the channel so the
// dropped sender reads as the established DISCONNECTED outcome — a
// dead runtime must not be misreported as a benign healthy-but-slow
// timeout (S4).
return self
.try_take_reply()
.unwrap_or(Err(ServerError::PushReplyDisconnected {
correlation_id: self.correlation_id,
}));
};
match runtime.expire_push_if_due(self.correlation_id) {
PushSlotDisposition::Expired => Err(ServerError::PushReplyExpired {
correlation_id: self.correlation_id,
}),
// Unreachable by construction (this is only called with the deadline
// due, and the registry re-reads a monotonic clock); honest benign
// fallback rather than a panic.
PushSlotDisposition::Live => Err(timeout_error()),
// Another path (a concurrent `resolve_push`, or connection close)
// removed the slot under the registry lock while we waited on it. Its
// send, if any, happens under that same lock, so re-check the channel:
// a delivered reply is present now; a dropped sender is disconnected.
PushSlotDisposition::Absent => self
.try_take_reply()
.unwrap_or_else(|| Err(timeout_error())),
}
}
/// SRV-027 R2: the outcome if the slot has resolved, read without waiting
/// by the requester's slice after its wake: `Some(Ok(payload))` for the
/// worker's reply, `Some(Err(..))` for a refusal, a frame the worker's
/// connection could not carry, or the worker gone (the slot swept at its
/// close), `None` while nothing has arrived.
pub(in crate::server::connection) fn try_receive(
&self,
) -> Option<Result<Vec<u8>, PushReplyFailure>> {
match self.receiver.try_recv() {
Ok(PushSlotResolution::WorkerRefused(refusal)) => {
Some(Err(PushReplyFailure::Worker(refusal)))
}
Ok(resolution) => Some(
self.resolution_into_result(resolution)
.map_err(PushReplyFailure::Server),
),
Err(TryRecvError::Empty) => None,
Err(TryRecvError::Disconnected) => Some(Err(PushReplyFailure::Server(
ServerError::PushReplyDisconnected {
correlation_id: self.correlation_id,
},
))),
}
}
/// Non-blocking check for a reply already sitting in the channel. `Some` with
/// the payload or a disconnected error; `None` when the channel is still empty
/// (no reply yet — the caller re-arms).
pub(super) fn try_take_reply(&self) -> Option<Result<Vec<u8>, ServerError>> {
match self.receiver.try_recv() {
Ok(resolution) => Some(self.resolution_into_result(resolution)),
Err(TryRecvError::Disconnected) => Some(Err(ServerError::PushReplyDisconnected {
correlation_id: self.correlation_id,
})),
Err(TryRecvError::Empty) => None,
}
}
/// Turns one delivered [`PushSlotResolution`] into this awaiter's result, so
/// all three receive paths read a resolution the same way and none of them
/// can grow its own interpretation of one.
///
/// Note what is NOT here: a dropped `Sender` never reaches this function. It
/// is a channel disconnect, and so stays the retryable
/// [`ServerError::PushReplyDisconnected`] on every path that kills a slot
/// without naming a reason — connection close included.
fn resolution_into_result(
&self,
resolution: PushSlotResolution,
) -> Result<Vec<u8>, ServerError> {
match resolution {
PushSlotResolution::Reply(payload) => Ok(payload),
PushSlotResolution::WorkerRefused(refusal) => Err(ServerError::PushReplyRefused {
correlation_id: self.correlation_id,
reason: refusal.code().caller_name().to_owned(),
}),
PushSlotResolution::ReplyRefused { reason } => Err(ServerError::PushReplyRefused {
correlation_id: self.correlation_id,
reason,
}),
PushSlotResolution::OutboundFrameTooLarge {
needed,
capacity,
queued,
} => Err(ServerError::PushFrameExceedsOutboundCapacity {
correlation_id: self.correlation_id,
needed,
capacity,
queued,
}),
}
}
}
/// Routed calls retain words only inside the validated, redacted record. Host
/// push APIs predate worker refusals and return the stable code alone, keeping
/// worker data out of their generic error's derived debug representation.
#[derive(Debug)]
pub(in crate::server::connection) enum PushReplyFailure {
Worker(WorkerRefusal),
Server(ServerError),
}
impl PushReplyFailure {
/// Settles the pending operation and its tap exactly once. Worker words
/// are released only after the caller's current membership permits the
/// read, and only the stable refusal code reaches the tap.
pub(in crate::server::connection) fn into_frame(
self,
state: &mut super::state::ConnectionProcessState,
op_id: u64,
stream_id: u32,
conversation_id: u64,
subject: &str,
route_words: impl FnOnce(&str, &ServerError) -> String,
) -> Frame {
let (reason_code, name, words) = match self {
Self::Worker(refusal) => match state
.membership
.release(subject, crate::auth_membership::MembershipAction::Read)
{
Ok(_) => (
refusal.code().reason_code(),
refusal.code().caller_name().to_owned(),
refusal.into_words(),
),
Err(error) => (0xffff, error.name().to_owned(), error.to_string()),
},
Self::Server(error) => {
let words = route_words(subject, &error);
(0xffff, words.clone(), words)
}
};
state.pending_replies.refuse(op_id, &name);
Frame::ConversationError {
flags: 0,
stream_id,
conversation_id,
reason_code,
message: Some(words),
}
}
}