sozu_lib/protocol/mux/mod.rs
1//! HTTP/1.1 and HTTP/2 multiplexing layer.
2//!
3//! This module unifies HTTP/1.1 and HTTP/2 behind a single [`Mux`] session
4//! state machine that integrates with sozu's mio event loop. The key types:
5//!
6//! - [`Mux`]: The top-level session state, generic over socket (`TcpStream` or
7//! `FrontRustls`) and listener. Implements `SessionState`.
8//! - [`Connection`]: Enum dispatching to [`ConnectionH1`] or [`ConnectionH2`]
9//! for protocol-specific readable/writable logic.
10//! - [`Stream`]: Per-request state with front/back kawa buffers, metrics, and
11//! lifecycle tracking. Shared between H1 and H2 paths.
12//! - [`Context`]: Per-session context (cluster, backends, routing, timeouts).
13//!
14//! The H2 implementation handles RFC 9113 framing, HPACK (RFC 7541), flow
15//! control, flood detection (CVE-2023-44487, CVE-2019-9512/9514/9515/9518,
16//! CVE-2024-27316), and graceful shutdown (double-GOAWAY per RFC 9113 §6.8).
17
18use std::{
19 cell::RefCell,
20 collections::{BTreeMap, VecDeque},
21 fmt::Debug,
22 io::ErrorKind,
23 net::{Shutdown, SocketAddr},
24 rc::{Rc, Weak},
25 sync::Arc,
26 time::{Duration, Instant, SystemTime, UNIX_EPOCH},
27};
28
29use mio::{Interest, Token, net::TcpStream};
30use rand::{SeedableRng, rngs::StdRng};
31use rusty_ulid::Ulid;
32use sozu_command::{
33 logging::ansi_palette,
34 proto::command::{Event, EventKind},
35 ready::Ready,
36};
37
38/// Protocol label + session descriptor used as a prefix on every [`Mux`] log
39/// line. Matches the RUSTLS log-context convention:
40/// `[<ulid> - - -]\tMUX\tSession(...)\t >>>`. When colored output is enabled
41/// (via [`ansi_palette`]) the label is wrapped in bold bright-white ANSI
42/// (uniform across every protocol) and the session detail block is rendered
43/// in light grey.
44///
45/// Fields included in the session block:
46/// - `frontend` — mio token of the frontend socket
47/// - `peer` — [`Connection::peer_address`]: a snapshot, not a live lookup
48/// - `streams` — number of streams currently held by the [`Context`]
49/// - `backends` — number of backend connections in the [`Router`]
50/// - `pending_links` — streams waiting to be linked to a backend
51/// - `readiness` — frontend mio readiness snapshot
52macro_rules! log_context {
53 ($self:expr) => {{
54 let (open, reset, grey, gray, white) = ansi_palette();
55 format!(
56 "[{ulid} - - -]\t{open}MUX{reset}\t{grey}Session{reset}({gray}frontend{reset}={white}{frontend}{reset}, {gray}peer{reset}={white}{peer:?}{reset}, {gray}streams{reset}={white}{streams}{reset}, {gray}backends{reset}={white}{backends}{reset}, {gray}pending_links{reset}={white}{pending_links}{reset}, {gray}readiness{reset}={white}{readiness}{reset})\t >>>",
57 open = open,
58 reset = reset,
59 grey = grey,
60 gray = gray,
61 white = white,
62 ulid = $self.session_ulid,
63 frontend = $self.frontend_token.0,
64 peer = $self.frontend.peer_address(),
65 streams = $self.context.streams.len(),
66 backends = $self.router.backends.len(),
67 pending_links = $self.context.pending_links.len(),
68 readiness = $self.frontend.readiness(),
69 )
70 }};
71}
72
73/// Lighter variant of [`log_context!`] that omits the
74/// `streams`/`backends`/`pending_links` counts. Used at sites where the
75/// borrow checker forbids reading `self.router.backends` or
76/// `self.context.streams` (e.g. inside a method that already holds a mutable
77/// borrow on one of them). The ULID and frontend snapshot still carry enough
78/// context to correlate the line back to the rest of the session.
79macro_rules! log_context_lite {
80 ($self:expr) => {{
81 let (open, reset, grey, gray, white) = ansi_palette();
82 format!(
83 "[{ulid} - - -]\t{open}MUX{reset}\t{grey}Session{reset}({gray}frontend{reset}={white}{frontend}{reset}, {gray}peer{reset}={white}{peer:?}{reset}, {gray}readiness{reset}={white}{readiness}{reset})\t >>>",
84 open = open,
85 reset = reset,
86 grey = grey,
87 gray = gray,
88 white = white,
89 ulid = $self.session_ulid,
90 frontend = $self.frontend_token.0,
91 peer = $self.frontend.peer_address(),
92 readiness = $self.frontend.readiness(),
93 )
94 }};
95}
96
97/// Module-level prefix for logs emitted from free functions or routing
98/// blocks where no [`Mux`] is in scope. Honours the colored flag.
99///
100/// Two arms:
101/// * `log_module_context!()` — zero-arg, legacy `MUX\t >>>` output. Kept
102/// for sites without an `HttpContext` in scope (e.g. the generic
103/// `trace!` that fires before the variant-specific match).
104/// * `log_module_context!($http_context)` — rich form. `$http_context`
105/// must be `&HttpContext`. Produces the same
106/// `[session req cluster backend]` bracket as RUSTLS/PIPE/TCP followed
107/// by a `Session(...)` block, so MUX lines emitted from variant match
108/// arms stay filterable by session ULID or request ULID. Mirrors
109/// `router.rs:log_module_context!($http_context)` (see there). Custom
110/// methods render only their byte length, and authority renders only its
111/// byte length.
112macro_rules! log_module_context {
113 () => {{
114 let (open, reset, _, _, _) = ansi_palette();
115 format!("{open}MUX{reset}\t >>>", open = open, reset = reset)
116 }};
117 ($http_context:expr) => {{
118 let (open, reset, grey, gray, white) = ansi_palette();
119 let http_ctx: &HttpContext = &$http_context;
120 let ctx = http_ctx.log_context();
121 format!(
122 "{gray}{ctx}{reset}\t{open}MUX{reset}\t{grey}Session{reset}({gray}frontend{reset}={white}{frontend:?}{reset}, {gray}method{reset}={white}{method:?}{reset}, {gray}authority_bytes{reset}={white}{authority_bytes:?}{reset})\t >>>",
123 open = open,
124 reset = reset,
125 grey = grey,
126 gray = gray,
127 white = white,
128 ctx = ctx,
129 frontend = http_ctx.session_address,
130 method = http_ctx.method,
131 authority_bytes = http_ctx.authority.as_ref().map(String::len),
132 )
133 }};
134}
135
136use self::h2::MetricEvent;
137
138pub mod answers;
139pub mod auth;
140pub mod buffer_source;
141pub mod connection;
142mod converter;
143pub mod debug;
144mod h1;
145mod h2;
146mod h2_close;
147mod h2_control_tx;
148mod h2_drain;
149mod h2_flood_detector;
150mod h2_flow_control;
151mod h2_header_reassembly;
152mod h2_output;
153mod h2_scheduler;
154mod h2_stream_table;
155pub mod h2_transmit;
156mod h2_write_pass;
157pub mod hpack;
158mod hpack_state;
159pub mod parser;
160mod pkawa;
161pub mod router;
162pub(crate) mod serializer;
163mod shared;
164pub mod stream;
165
166use crate::metrics::names;
167use crate::{
168 BackendConnectionError, FrontendFromRequestError, L7ListenerHandler, L7Proxy, ListenerHandler,
169 Protocol, ProxySession, Readiness, RetrieveClusterError, SessionIsToBeClosed, SessionMetrics,
170 SessionResult, StateResult,
171 backends::{Backend, BackendError, BackendMap},
172 http::HttpListener,
173 https::HttpsListener,
174 pool::{Checkout, Pool},
175 protocol::{SessionState, http::editor::HttpContext},
176 retry::RetryPolicy,
177 server::push_event,
178 socket::{FrontRustls, SessionTcpStream, SocketHandler, SocketResult, stats::socket_rtt},
179 timer::TimeoutContainer,
180};
181
182use crate::protocol::mux::answers::silent_close_answer;
183pub(crate) use crate::protocol::mux::answers::{
184 forcefully_terminate_answer, set_default_answer, set_default_answer_with_retry_after,
185};
186use crate::protocol::mux::connection::{EndpointClient, EndpointServer};
187pub use crate::protocol::mux::{
188 answers::terminate_default_answer,
189 buffer_source::{BufferSource, PoolBufferSource},
190 connection::Connection,
191 debug::{DebugEvent, DebugHistory},
192 h1::ConnectionH1,
193 h2::CLIENT_PREFACE_SIZE,
194 h2::ConnectionH2,
195 h2::H2ByteAccounting,
196 h2::H2ConnectionConfig,
197 h2::{
198 H2ControlFlushTarget, H2FinalizeTarget, H2ForceDisconnectTarget, H2ReadOutcome,
199 H2ReadTarget, H2Shell, H2StreamId, H2WritableStateTarget, H2WriteTarget, H2Written,
200 },
201 h2_flood_detector::H2FloodConfig,
202 h2_write_pass::H2WritePass,
203 parser::H2Error,
204 router::Router,
205 stream::{Stream, StreamParts, StreamState},
206};
207
208// ── Tuning Constants ─────────────────────────────────────────────────────────
209
210/// Maximum inner-loop iterations of one `Mux::ready_inner` call, a bound
211/// that keeps one session from holding the single-threaded worker. A call
212/// that forwarded stream bytes yields there, counted in
213/// `http.mux.loop_budget_yield`, and the event loop resumes it on its next
214/// turn (sozu-proxy/sozu#1906). Reaching it without forwarding a byte is a
215/// bug: it increments `http.infinite_loop.error` and logs a warning. A
216/// session with output still queued for the client then waits for its next
217/// writable event; any other session is closed.
218const MAX_LOOP_ITERATIONS: i32 = 10_000;
219
220/// Whether a response in progress waits on the CLIENT rather than on its
221/// backend: Sōzu still holds response bytes for this stream that the client
222/// has not taken. `front_pending_write` is the frontend connection's own
223/// pending output (`Connection::has_pending_write`, a TLS record for
224/// instance); it is stream-specific only on an H1 frontend, so an H2
225/// frontend passes `false` and only the stream's own queued bytes count.
226///
227/// `back_timeout` and `front_timeout` bound the inactivity of one socket
228/// each (`doc/configure.md`). The two timeout arms of `Mux::timeout_inner`
229/// read this predicate with opposite polarity, so exactly one of the two
230/// timers can cut a response at any moment: a backend whose bytes Sōzu has
231/// stopped reading because a slow client left them queued is not inactive,
232/// and neither is a client waiting on a slow backend (sozu-proxy/sozu#1906).
233fn response_waits_on_client(stream: &Stream, front_pending_write: bool) -> bool {
234 let queued = !stream.back.out.is_empty() || !stream.back.blocks.is_empty();
235 debug_assert!(
236 stream.back.consumed,
237 "only a response in progress can wait on either peer"
238 );
239 debug_assert_eq!(
240 queued,
241 !stream.back.is_completed(),
242 "kawa's `is_completed` is exactly \"nothing queued\""
243 );
244 queued || front_pending_write
245}
246
247/// Whether a `Mux::ready_inner` call that reached `MAX_LOOP_ITERATIONS`
248/// moved enough stream bytes to be a transfer whose peers keep up, which
249/// yields to the event loop, rather than a spin, which the loop budget
250/// exists to catch (sozu-proxy/sozu#1906). A real transfer forwards up to a
251/// buffer per iteration; at least one byte per budgeted iteration on average
252/// is required, so a spin that trickles a few bytes still reaches
253/// `Mux::wait_for_client_at_loop_limit` instead of yielding forever.
254/// `Context::stream_bytes_out` wraps, hence the wrapping difference.
255fn progressed_through_loop_budget(bytes_out_at_entry: usize, bytes_out_now: usize) -> bool {
256 let moved = bytes_out_now.wrapping_sub(bytes_out_at_entry);
257 let progressed = moved >= MAX_LOOP_ITERATIONS as usize;
258 debug_assert!(
259 !progressed || moved > 0,
260 "a call judged as progressing must have moved bytes"
261 );
262 debug_assert!(
263 progressed || moved < MAX_LOOP_ITERATIONS as usize,
264 "a call that moved a byte per iteration must be judged as progressing"
265 );
266 progressed
267}
268// ─────────────────────────────────────────────────────────────────────────────
269
270/// Debug tripwire for the one-active-stream invariant that
271/// `ConnectionH1::end_stream`'s early-return guard rests on.
272///
273/// `Mux::timeout` retires streams through `Connection::end_stream`, and its
274/// proof that no linked stream survives (LIFECYCLE invariant 22) depends on
275/// that call actually unlinking. `ConnectionH2::end_stream` always does.
276/// `ConnectionH1::end_stream` does NOT: it guards `self.stream != Some(stream)`
277/// and returns early, skipping the `Context::unlink_stream`. The guard is safe
278/// only because an H1 connection carries at most one linked stream and it
279/// equals `self.stream` — `self.stream = Some(..)` occurs only in
280/// `ConnectionH1::start_stream`, paired with `Context::link_stream`, and
281/// `self.stream = None` only inside `end_stream` itself, after the unlink.
282///
283/// This is the tripwire for that invariant: a drift between
284/// `context.backend_streams` (or a stream's `StreamState::Linked`) and
285/// `ConnectionH1::stream` fires here, in every debug / test / e2e build,
286/// instead of silently leaving a stream in the reverse index with no backend
287/// timer.
288///
289/// To SEE THIS FIRE: in
290/// `a_backend_timeout_leaves_no_linked_stream_behind_on_the_close_path`, set
291/// `h1.stream = Some(1)` instead of `Some(0)` while still linking stream 0 to
292/// the backend.
293///
294/// Deliberately at the CALL SITE rather than inside `ConnectionH1::end_stream`,
295/// and the reason is scope, not risk. That function has around a dozen callers
296/// reaching it through `Connection::end_stream` and the two `Endpoint`
297/// adaptors, and the H2-driven ones are precisely the ones this proof never
298/// traced: `H2Shell::close` and `ConnectionH2::end_stream` iterate their
299/// OWN wire map (`for global_stream_id in self.streams.values()`, `h2.rs`) and
300/// hand each `StreamState::Linked` gid to the peer, which on an H2-frontend /
301/// H1-backend session is a `ConnectionH1`. Add the reset and HUP paths and the
302/// set is wider than the claim.
303///
304/// Asserting inside the function would therefore assert something broader than
305/// what was traced. Note what does NOT justify the placement: running the full
306/// e2e suite with a `debug_assert_eq!` in that mismatch arm passes with zero
307/// hits, and green e2e shows absence of COVERAGE, not absence of the path. The
308/// claim is about the two `Mux::timeout` loops, so the assertion is too.
309#[cfg(debug_assertions)]
310fn debug_assert_h1_owns_stream<Front: SocketHandler>(
311 connection: &Connection<Front>,
312 stream_id: GlobalStreamId,
313) {
314 if let Connection::H1(h1) = connection {
315 debug_assert_eq!(
316 h1.stream,
317 Some(stream_id),
318 "ConnectionH1::end_stream would skip its unlink: the connection's \
319 active stream is {:?}, not the {stream_id} Mux::timeout is ending \
320 — the backend index and ConnectionH1::stream have drifted",
321 h1.stream,
322 );
323 }
324}
325
326/// Generic Http representation using the Kawa crate using the Checkout of Sozu as buffer
327type GenericHttpStream = kawa::Kawa<Checkout>;
328type StreamId = u32;
329type GlobalStreamId = usize;
330pub type MuxClear = Mux<SessionTcpStream, HttpListener>;
331pub type MuxTls = Mux<FrontRustls, HttpsListener>;
332
333/// An opaque, session-scoped name for one entry of the worker's backend
334/// registry.
335///
336/// It is an index into `Mux::backend_registry` and nothing else: the core
337/// can carry it, compare it and hand it back, and cannot turn it into a
338/// `Backend`. Only the embedder resolves it, which is what keeps
339/// `Rc<RefCell<Backend>>` out of the core (#1340, Question 12).
340///
341/// Session-scoped rather than derived from `(cluster_id, backend_id)` on
342/// purpose. A reload that replaces a registry entry mid-session builds a NEW
343/// `Rc`, which takes a NEW slot, so connections opened before it keep charging
344/// the handle they were dialled against — exactly what holding the `Rc`
345/// inside `Position::Client` used to do. Keying on the backend's name instead
346/// would split a balanced start/end pair across two `Backend` values: the
347/// superseded one would keep a charge forever and the fresh one would
348/// saturate at zero.
349#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
350pub struct BackendSlot(usize);
351
352/// A backend as the core sees it: the opaque slot, plus the two identity
353/// fields the datapath renders.
354///
355/// `backend_id` and `address` are copied once, when the session first interns
356/// the backend in its `BackendRegistry`, and every later dial of it clones
357/// them from there. Both are immutable for the life of a registry entry, so
358/// carrying them is not a cached view of mutable state — no load state
359/// crosses this boundary.
360/// The mutable half (`active_connections`, `active_requests`, `failures`,
361/// `connection_time`, `health`, `status`) stays behind the slot, and the core
362/// reaches it only by emitting a [`BackendDelta`].
363///
364/// `backend_id` is an `Rc<str>` so that no step past interning copies it:
365/// rebuilding `Position::Client` on `Connecting` -> `Connected` once per dial,
366/// minting the id for a redial (the registry keeps the `Rc<str>` it built at
367/// interning, #1564), and stamping a request with the backend it runs on,
368/// dialled or reused, into `HttpContext::backend_id` and
369/// `SessionMetrics::backend_id` (#1579) are each a reference-count increment.
370/// A `String` would make every one of them a heap copy.
371#[derive(Clone, Debug, PartialEq, Eq)]
372pub struct BackendId {
373 /// Private to this module: [`Mux::backend`] is the only way out.
374 slot: BackendSlot,
375 /// `Backend::backend_id`, for log lines and per-backend metric labels.
376 pub backend_id: Rc<str>,
377 /// `Backend::address`, for log lines and `push_event` payloads.
378 pub address: SocketAddr,
379 /// Metrics lifetime captured by the request that opened this connection.
380 /// It stays with the connection through draining so its close cannot
381 /// mutate a replacement cluster with the same labels.
382 metric_incarnation: crate::metrics::ClusterMetricsCapture,
383}
384
385impl BackendId {
386 /// An id for the backend at `address`, named `backend_id`, in `slot` of
387 /// whichever registry mints it (#1684).
388 ///
389 /// The worker's mux never calls this: its ids come from the session's
390 /// `BackendRegistry`, which interns the backend it names and is the only
391 /// thing that can resolve a slot back to it. This is for an embedder that
392 /// owns its own selector — a simulator implementing
393 /// [`router::BackendSelector`] — and numbers its backends itself. A slot
394 /// means something only to the registry that issued it.
395 pub fn new(slot: usize, backend_id: Rc<str>, address: SocketAddr) -> Self {
396 BackendId {
397 slot: BackendSlot(slot),
398 backend_id,
399 address,
400 metric_incarnation: Default::default(),
401 }
402 }
403
404 pub(crate) fn metric_incarnation(&self) -> crate::metrics::ClusterMetricsCapture {
405 self.metric_incarnation
406 }
407
408 fn set_metric_incarnation(&mut self, incarnation: crate::metrics::ClusterMetricsCapture) {
409 self.metric_incarnation = incarnation;
410 }
411
412 /// The slot this id names. Module-private on purpose — see [`Mux::backend`]
413 /// for the crate-visible resolution path.
414 fn slot(&self) -> BackendSlot {
415 self.slot
416 }
417}
418
419/// One accounting change the core decided and the embedder performs.
420///
421/// LIFECYCLE §9 invariant 14 is a balance, and this is the ledger it balances:
422/// every [`BackendChange::StreamsStarted`] unit the mux emits is covered by
423/// exactly one [`BackendChange::StreamsEnded`] unit against the same slot.
424/// Because every write to `Backend::active_requests` and
425/// `Backend::active_connections` on the mux path now goes through
426/// `Mux::apply_backend_deltas`, reconciling the emitted stream to zero is a
427/// direct test of the invariant rather than a proxy for it.
428#[derive(Clone, Copy, Debug, PartialEq, Eq)]
429pub struct BackendDelta {
430 /// The registry entry to charge.
431 pub slot: BackendSlot,
432 /// What to charge it.
433 pub change: BackendChange,
434}
435
436/// The accounting operations [`BackendDelta`] can carry.
437///
438/// The two counters invariant 14 is about, plus the release of a
439/// reservation whose dial never started. The other health and latency writes
440/// the mux makes (`failures` and `retry_policy` on an asynchronous connect
441/// failure, `connection_time`) are NOT here: they stay immediate at their
442/// existing embedder-side call sites rather than being deferred to a drain.
443#[derive(Clone, Copy, Debug, PartialEq, Eq)]
444pub enum BackendChange {
445 /// `Backend::active_requests += n` — `n` streams started on this backend.
446 StreamsStarted(usize),
447 /// `Backend::active_requests -= n`, saturating — `n` streams left it.
448 StreamsEnded(usize),
449 /// `Backend::dec_connections()` — the connection to this backend closed,
450 /// or a dial that connected was abandoned before its connection was
451 /// registered (`Mux::dial_backend`, #1713): either way the reservation
452 /// selection took is released, and no failure is recorded.
453 ConnectionClosed,
454 /// The `connect(2)` of a dial failed at this instant (#1684): release the
455 /// reservation selection took (`Backend::dec_connections()`), and record
456 /// the failure (`failures += 1`, `retry_policy.fail`), as
457 /// `Backend::release_failed_dial` does. The drain before the next
458 /// selection applies it, so that selection sees the backend exactly as a
459 /// failed `Backend::try_connect` left it.
460 DialFailed(Instant),
461}
462
463/// The embedder's table of registry handles, indexed by [`BackendSlot`].
464///
465/// A newtype rather than a bare `Vec` so the two operations that may cross
466/// Question 12's perimeter — minting a slot for a freshly dialled backend,
467/// and performing a [`BackendDelta`] against one — are the only ones there
468/// are. Nothing hands a `&Rc<RefCell<Backend>>` out of here except
469/// [`Mux::backend`], whose one caller is the WebSocket upgrade.
470#[derive(Debug, Default)]
471pub(crate) struct BackendRegistry(Vec<RegistryEntry>);
472
473/// One slot of the [`BackendRegistry`]: the handle, plus the [`BackendId`]
474/// identity fields read from it once, when it was interned.
475///
476/// The cached `backend_id` is what keeps `BackendRegistry::id_for`
477/// allocation-free on every dial after a backend's first (#1564). It cannot go
478/// stale: `Backend::backend_id` and `Backend::address` are never written after
479/// construction, and a reload that replaces the backend builds a new `Rc`,
480/// which takes a new slot with its own entry.
481#[derive(Debug)]
482struct RegistryEntry {
483 handle: Rc<RefCell<Backend>>,
484 backend_id: Rc<str>,
485 address: SocketAddr,
486}
487
488impl BackendRegistry {
489 /// Name `backend` with a slot, reusing the one it already has.
490 ///
491 /// Identity is `Rc::ptr_eq`, not the backend's id or address: two dials
492 /// separated by a reload that replaced the registry entry are two
493 /// different `Backend` values, and each must keep its own accounting.
494 /// The scan is linear over the backends ONE session has dialled, which
495 /// is the cluster's backend count at worst — the same order as the
496 /// `Router::backends` walk the caller has just done.
497 ///
498 /// A new slot copies the backend's id into an `Rc<str>` here, once per
499 /// session and backend; that copy and the table's own growth are the only
500 /// allocations this path makes.
501 fn intern(&mut self, backend: &Rc<RefCell<Backend>>) -> BackendSlot {
502 match self
503 .0
504 .iter()
505 .position(|known| Rc::ptr_eq(&known.handle, backend))
506 {
507 Some(slot) => BackendSlot(slot),
508 None => {
509 let (backend_id, address) = {
510 let borrow = backend.borrow();
511 (Rc::from(borrow.backend_id.as_str()), borrow.address)
512 };
513 // A session almost always dials one backend: its first entry
514 // sizes the table for exactly one instead of four (#1610),
515 // and a second distinct backend grows it as usual.
516 if self.0.capacity() == 0 {
517 self.0.reserve_exact(1);
518 }
519 self.0.push(RegistryEntry {
520 handle: backend.clone(),
521 backend_id,
522 address,
523 });
524 BackendSlot(self.0.len() - 1)
525 }
526 }
527 }
528
529 /// Build the core's view of a backend it is about to be handed.
530 ///
531 /// Runs once per backend dial. Past a backend's first dial in this
532 /// session it clones the id cached in its slot and allocates nothing.
533 fn id_for(&mut self, backend: &Rc<RefCell<Backend>>) -> BackendId {
534 let slot = self.intern(backend);
535 let entry = &self.0[slot.0];
536 BackendId {
537 slot,
538 backend_id: entry.backend_id.clone(),
539 address: entry.address,
540 metric_incarnation: Default::default(),
541 }
542 }
543
544 fn get(&self, slot: BackendSlot) -> Option<&Rc<RefCell<Backend>>> {
545 self.0.get(slot.0).map(|entry| &entry.handle)
546 }
547
548 /// Resolve one of the core's opaque [`BackendId`]s to the registry handle
549 /// it names.
550 ///
551 /// The single crate-visible way back from an id to an
552 /// `Rc<RefCell<Backend>>`, so the perimeter Question 12 draws has one
553 /// door rather than a `pub` slot every caller can walk through. Its only
554 /// callers are `upgrade_mux` in `lib/src/http.rs` and `lib/src/https.rs`,
555 /// which hand the live handle to the WebSocket `Pipe` that takes the
556 /// backend over.
557 ///
558 /// It sits here rather than on [`Mux`] because those callers have already
559 /// moved `Mux::frontend` out by the time they need it; a field access
560 /// borrows disjointly where a `&self` method on `Mux` could not.
561 ///
562 /// `None` cannot happen for an id this session minted — the table never
563 /// shrinks — so a caller treats it as the desync it would be.
564 pub(crate) fn handle(&self, backend: &BackendId) -> Option<&Rc<RefCell<Backend>>> {
565 self.get(backend.slot())
566 }
567
568 /// Whether the backend `backend` names has left the configuration.
569 ///
570 /// `BackendList::remove_backend` and `BackendMap::remove_cluster`
571 /// (`lib/src/backends.rs`) mark a backend `Closing` as they drop it, and
572 /// the registry still holds that very `Rc`. A backend added again later,
573 /// even at the same address, is a new `Rc` in a new slot, so a
574 /// connection dialled to the old one stays retired. Reads one `RefCell`
575 /// borrow and allocates nothing.
576 pub(crate) fn is_retired(&self, backend: &BackendId) -> bool {
577 self.get(backend.slot())
578 .is_some_and(|handle| handle.borrow().status != crate::backends::BackendStatus::Normal)
579 }
580
581 /// Perform every accounting change in `deltas`, in the order they were
582 /// pushed, and leave the ledger empty.
583 ///
584 /// The one drain both `Mux::apply_backend_deltas` and the pre-plan settle
585 /// in `Mux::ready_inner` go through.
586 ///
587 /// Drained in place rather than taken, so the ledger keeps its capacity
588 /// for the next change: the charge `Connection::start_stream` records on
589 /// every request would otherwise reallocate it each time (#1583). The
590 /// capacity is bounded by the most deltas a single pass has pushed.
591 ///
592 /// `backends` is the worker's backend map, borrowed only to draw the
593 /// backoff jitter of a [`BackendChange::DialFailed`] from its generator;
594 /// every other change leaves it untouched.
595 pub(crate) fn apply_all(&self, deltas: &mut Vec<BackendDelta>, backends: &RefCell<BackendMap>) {
596 for delta in deltas.drain(..) {
597 self.apply(delta, backends);
598 }
599 }
600
601 /// Perform one accounting change the core decided.
602 ///
603 /// The before/after pair-assertions that used to sit at each emitting
604 /// site live here now: this is where both halves are observable, and one
605 /// copy covers every emitter instead of each carrying its own.
606 fn apply(&self, delta: BackendDelta, backends: &RefCell<BackendMap>) {
607 let Some(backend) = self.get(delta.slot) else {
608 // Unreachable while the vector only grows, which it does. Report
609 // rather than panic: losing one charge is a drifting gauge, and
610 // dropping a session over it would be worse.
611 error!(
612 "{} backend accounting delta {:?} names a slot this session never minted",
613 log_module_context!(),
614 delta
615 );
616 return;
617 };
618 let mut backend = backend.borrow_mut();
619 match delta.change {
620 BackendChange::StreamsStarted(count) => {
621 let before = backend.active_requests;
622 backend.active_requests += count;
623 debug_assert_eq!(
624 backend.active_requests,
625 before + count,
626 "a StreamsStarted delta must raise active_requests by exactly its count"
627 );
628 }
629 BackendChange::StreamsEnded(count) => {
630 // `saturating_sub` is the network-safe floor a desynced peer
631 // needs, so the post-relation is all that can be asserted —
632 // not that `before >= count`.
633 let before = backend.active_requests;
634 backend.active_requests = backend.active_requests.saturating_sub(count);
635 debug_assert_eq!(
636 backend.active_requests,
637 before.saturating_sub(count),
638 "a StreamsEnded delta must lower active_requests by its count (saturating)"
639 );
640 debug_assert!(
641 backend.active_requests <= before,
642 "active_requests must not grow on a StreamsEnded delta"
643 );
644 }
645 BackendChange::ConnectionClosed => {
646 // `dec_connections` floors at 0 (a double close from a
647 // desynced peer must not panic), so the post-relation is
648 // "decreased by one, unless already at zero".
649 let before = backend.active_connections;
650 backend.dec_connections();
651 debug_assert_eq!(
652 backend.active_connections,
653 before.saturating_sub(1),
654 "a ConnectionClosed delta must release exactly one backend connection \
655 (saturating at 0)"
656 );
657 }
658 BackendChange::DialFailed(at) => {
659 // The one change that draws randomness, so the only one that
660 // borrows the map. `release_failed_dial` carries the
661 // before/after assertions for both halves.
662 backend.release_failed_dial(at, backends.borrow_mut().rng());
663 }
664 }
665 trace!(
666 "{} backend accounting {:?} applied: {:#?}",
667 log_module_context!(),
668 delta.change,
669 backend
670 );
671 }
672}
673
674pub enum Position {
675 Client(sozu_command::state::ClusterId, BackendId, BackendStatus),
676 Server,
677}
678
679impl Debug for Position {
680 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
681 match self {
682 Self::Client(cluster_id, _, status) => f
683 .debug_tuple("Client")
684 .field(cluster_id)
685 .field(status)
686 .finish(),
687 Self::Server => write!(f, "Server"),
688 }
689 }
690}
691
692impl Position {
693 fn is_server(&self) -> bool {
694 match self {
695 Position::Client(..) => false,
696 Position::Server => true,
697 }
698 }
699 fn is_client(&self) -> bool {
700 !self.is_server()
701 }
702
703 /// The bytes-read event for this side.
704 ///
705 /// Returns exactly one event — never zero, never two — because both arms
706 /// of the match below emit one, so the caller needs no container to carry
707 /// it. A caller that is its own shell records it at once; a caller inside
708 /// the H2 core queues it. `#[must_use]` is what makes that a compiler
709 /// obligation rather than a convention: this leaf is reached from a core
710 /// AND from shells, and an event silently dropped here is an upward
711 /// drift that never underflows, so nothing would log and nothing would
712 /// saturate.
713 #[must_use]
714 fn bytes_in_event(&self, size: usize) -> MetricEvent {
715 match self {
716 Position::Client(..) => MetricEvent::BackendBytesIn(size as i64),
717 Position::Server => MetricEvent::FrontendBytesIn(size as i64),
718 }
719 }
720
721 /// The bytes-written event for this side. Same one-event guarantee and
722 /// same reason for `#[must_use]` as [`Self::bytes_in_event`].
723 #[must_use]
724 fn bytes_out_event(&self, size: usize) -> MetricEvent {
725 match self {
726 Position::Client(..) => MetricEvent::BackendBytesOut(size as i64),
727 Position::Server => MetricEvent::FrontendBytesOut(size as i64),
728 }
729 }
730
731 /// Attribute `size` bytes read to the appropriate `SessionMetrics` field.
732 pub fn count_bytes_in(&self, metrics: &mut SessionMetrics, size: usize) {
733 match self {
734 Position::Client(..) => metrics.backend_bin += size,
735 Position::Server => metrics.bin += size,
736 }
737 }
738
739 /// Attribute `size` bytes written to the appropriate `SessionMetrics` field.
740 pub fn count_bytes_out(&self, metrics: &mut SessionMetrics, size: usize) {
741 match self {
742 Position::Client(..) => metrics.backend_bout += size,
743 Position::Server => metrics.bout += size,
744 }
745 }
746}
747
748#[derive(Debug)]
749pub enum BackendStatus {
750 Connecting(Instant),
751 Connected,
752 KeepAlive,
753 Disconnecting,
754}
755
756#[derive(Debug, Clone, Copy)]
757pub enum MuxResult {
758 Continue,
759 Upgrade,
760 CloseSession,
761}
762
763pub trait Endpoint: Debug {
764 fn readiness(&self, token: Token) -> &Readiness;
765 fn readiness_mut(&mut self, token: Token) -> &mut Readiness;
766 /// Smoothed round-trip time of the peer side of a stream, already
767 /// sampled.
768 ///
769 /// Used by access-log emission to report TCP_INFO RTT for the side the
770 /// caller does NOT own directly: a frontend connection (Position::Server)
771 /// reports the backend's RTT through this method, and a backend connection
772 /// (Position::Client) reports the frontend's the same way. `token` is
773 /// ignored by `connection::EndpointServer` (which has a single
774 /// frontend connection) and used as a key by
775 /// `connection::EndpointClient` (which keys backends by token).
776 /// Returns `None` when the token doesn't resolve, mirroring the existing
777 /// fallback paths in `readiness`/`readiness_mut`, and also when the
778 /// platform declines to answer. The value is the peer connection's one
779 /// sample, taken by `memoized_rtt` the first time it is asked for.
780 ///
781 /// This returns the VALUE, not the socket it came from. The predecessor,
782 /// `fn socket(&self, token) -> Option<&TcpStream>`, handed out a
783 /// `mio::net::TcpStream` — a concrete OS type — so any connection could
784 /// reach any other connection's socket for any purpose, and no in-memory
785 /// transport could ever satisfy the trait. RTT is intrinsically a live
786 /// socket property and stays on the embedder's side of the boundary; the
787 /// cores receive an `Option<Duration>` that was captured for them.
788 fn peer_rtt(&self, token: Token) -> Option<Duration>;
789 /// Smoothed round-trip time of the connection this endpoint was handed
790 /// TO — the local side, where [`Self::peer_rtt`] reports the other one.
791 ///
792 /// Asked by `ConnectionH2::snapshot_rtts` for the access log's
793 /// `client_rtt` cell, at the moment a stream is logged and at no other
794 /// time, so a pass that logs nothing costs no `getsockopt(TCP_INFO)`.
795 /// Neither `EndpointClient` nor `EndpointServer` owns the local socket,
796 /// so both answer `None`: the value comes from `h2::ShellEndpoint`, the
797 /// wrapper `H2Shell` puts around them, which samples its own socket once
798 /// per connection through `memoized_rtt`. The core still never reaches
799 /// a socket.
800 fn local_rtt(&self) -> Option<Duration>;
801 /// If end_stream is called on a client it means the stream has PROPERLY finished,
802 /// the server has completed serving the response and informs the endpoint that this stream won't be used anymore.
803 /// If end_stream is called on a server it means the stream was BROKEN, the client was most likely disconnected or encountered an error
804 /// it is for the server to decide if the stream can be retried or an error should be sent. It should be GUARANTEED that all bytes from
805 /// the backend were read. However it is almost certain that all bytes were not already sent to the client.
806 fn end_stream<L: ListenerHandler + L7ListenerHandler>(
807 &mut self,
808 token: Token,
809 stream: GlobalStreamId,
810 context: &mut Context<L>,
811 );
812 /// If start_stream is called on a client it means the stream should be attached to this endpoint,
813 /// the stream might be recovering from a disconnection, in any case at this point its response MUST be empty.
814 /// If the start_stream is called on a H2 server it means the stream is a server push and its request MUST be empty.
815 /// Returns false if the stream could not be started (e.g. max concurrent streams reached).
816 fn start_stream<L: ListenerHandler + L7ListenerHandler>(
817 &mut self,
818 token: Token,
819 stream: GlobalStreamId,
820 context: &mut Context<L>,
821 ) -> bool;
822}
823
824/// Every `getsockopt(TCP_INFO)` the mux issues for an access log goes through
825/// here, and only through [`memoized_rtt`]: once per connection, frontend and
826/// backend alike.
827///
828/// One function so the tests can count the reads: under `cfg(test)` each call
829/// bumps a thread-local counter, read back by [`tcp_info_reads`]. The
830/// production body is the bare `socket_rtt` call.
831pub(super) fn sample_rtt<A: std::os::fd::AsRawFd>(socket: &A) -> Option<Duration> {
832 #[cfg(test)]
833 TCP_INFO_READS.with(|reads| reads.set(reads.get() + 1));
834 socket_rtt(socket)
835}
836
837/// The smoothed RTT of `socket`, read at most once for the connection that
838/// owns `memo` and reused by every later access log of that connection.
839///
840/// `memo` is `None` until the connection's first access log asks for its RTT
841/// — the first request completed on a frontend, the first response logged
842/// through a backend — and holds that one sample, `Some(None)` included when
843/// the platform declined, for the rest of the connection's life. Nothing
844/// resets it: a keep-alive frontend serving twenty requests, or a backend
845/// connection taken back out of the keep-alive pool, costs one
846/// `getsockopt(TCP_INFO)`, not one per request.
847///
848/// The first access log and not `accept`/`connect`: Linux already has a
849/// handshake RTT at `accept` (on loopback, 27–34 µs straight out of
850/// `accept`), so an earlier sample would be valid, but a lazy one is free on
851/// a connection that never logs, and a backend dial is still in flight when
852/// its `connect(2)` returns. The price is a frozen value: the kernel's SRTT
853/// keeps moving over the connection's life and the access log reports the
854/// sample taken at its first request. A `Cell` because `Endpoint::peer_rtt`
855/// and `Endpoint::local_rtt` take `&self`.
856pub(super) fn memoized_rtt<A: std::os::fd::AsRawFd>(
857 memo: &std::cell::Cell<Option<Option<Duration>>>,
858 socket: &A,
859) -> Option<Duration> {
860 if let Some(sample) = memo.get() {
861 return sample;
862 }
863 let sample = sample_rtt(socket);
864 memo.set(Some(sample));
865 debug_assert_eq!(memo.get(), Some(sample), "the first sample must be kept");
866 sample
867}
868
869#[cfg(test)]
870thread_local! {
871 static TCP_INFO_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
872}
873
874/// `getsockopt(TCP_INFO)` reads the calling thread has made through
875/// [`sample_rtt`] so far.
876#[cfg(test)]
877pub(super) fn tcp_info_reads() -> usize {
878 TCP_INFO_READS.with(std::cell::Cell::get)
879}
880
881/// `shutdown(SHUT_WR)` on a socket whose descriptor is about to be closed,
882/// unless the peer has already closed its side.
883///
884/// Every write-side shutdown of the HTTP and HTTPS close paths goes through
885/// here: the frontend in `HttpSession::close` (`lib/src/http.rs`) and
886/// `HttpsSession::close` (`lib/src/https.rs`), the backends in `Mux::close`
887/// and in the dead-backend sweep of `Mux::ready_inner`. A frontend's
888/// lingering close (`ConnectionH1::start_linger`, `h1.rs`, and
889/// `H2Shell::linger_instead_of_closing`, `h2.rs`) calls it earlier, before
890/// it drains what the client still sends; the session close then repeats
891/// it: a no-op once the FIN is queued, or `ENOTCONN` once the connection is
892/// gone, which both callers already ignore. `peer_closed` comes
893/// from [`Connection::peer_closed`], which `HttpsSession::close` widens with
894/// `FrontRustls::peer_disconnected`.
895///
896/// Skipping it changes nothing on the wire. None of these descriptors is
897/// shared (no `dup`, `try_clone` or fork; see `doc/lifetime_of_a_session.md`
898/// §11), so the `close(2)` that follows is the last one, and Linux's
899/// `tcp_close` then sends the FIN after the queued bytes itself when the
900/// receive queue is empty, or a RST when it is not — whether or not a
901/// `shutdown(SHUT_WR)` came first. Once the peer has closed, the shutdown is
902/// therefore a syscall that only reports what the close would do, and on a
903/// peer that has also reset it fails with `ENOTCONN`. HAProxy draws the same
904/// line: `conn_sock_shutw` (`include/haproxy/connection.h`) skips the socket
905/// shutdown once `CO_FL_SOCK_RD_SH` records the read0. The TLS
906/// `close_notify` is not affected: it is written before this call, and RFC
907/// 8446 §6.1 requires it before the write side closes whether or not the
908/// peer sent its own.
909///
910/// Under `cfg(test)` each shutdown actually issued bumps a thread-local
911/// counter, read back by [`shutdown_writes`].
912pub(crate) fn shutdown_write(socket: &TcpStream, peer_closed: bool) -> std::io::Result<()> {
913 if peer_closed {
914 return Ok(());
915 }
916 #[cfg(test)]
917 SHUTDOWN_WRITES.with(|writes| writes.set(writes.get() + 1));
918 socket.shutdown(Shutdown::Write)
919}
920
921#[cfg(test)]
922thread_local! {
923 static SHUTDOWN_WRITES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
924}
925
926/// `shutdown(SHUT_WR)` calls the calling thread has issued through
927/// [`shutdown_write`] so far.
928#[cfg(test)]
929pub(crate) fn shutdown_writes() -> usize {
930 SHUTDOWN_WRITES.with(std::cell::Cell::get)
931}
932
933/// Shared logic for half-close accounting: clear `bit` from `readiness.event` on
934/// socket errors/would-block, and return `true` (yield) when no bytes were
935/// transferred so the caller can park the half. Rationale for clearing only
936/// `bit` (not both halves) on `Closed`: the opposite half may still need one
937/// last pass to flush queued frames or the TLS close_notify.
938fn update_readiness(
939 size: usize,
940 status: SocketResult,
941 readiness: &mut Readiness,
942 bit: Ready,
943) -> bool {
944 trace!(
945 "{} size={}, status={:?}",
946 log_module_context!(),
947 size,
948 status
949 );
950 match status {
951 SocketResult::Continue => {}
952 SocketResult::Closed | SocketResult::Error | SocketResult::WouldBlock => {
953 readiness.event.remove(bit);
954 }
955 }
956 if size > 0 {
957 false
958 } else {
959 readiness.event.remove(bit);
960 true
961 }
962}
963
964/// [`update_readiness`] for a read, with one exception: a short read after HUP.
965///
966/// The socket layer stops on a short read and answers `(n, WouldBlock)`
967/// without reading on to EOF (`plain_socket_read`, `rustls_socket_read` in
968/// `lib/src/socket.rs`). A FIN that arrived with the data raised its edge
969/// already, and HUP is that edge (`Ready::from(&Event)` in
970/// `command/src/ready.rs`); no later event will announce the EOF, so dropping
971/// READABLE here would leave it unread. Keeping READABLE makes the next pass
972/// read it as `(0, Closed)`, which the H1 close-delimited response path
973/// needs. HAProxy reads `read0` on the same condition (`src/raw_sock.c`,
974/// `FD_POLL_HUP` after `ret < try`), and tokio keeps closed readiness through
975/// `clear_readiness` (`tokio/src/runtime/io/scheduled_io.rs`). The next read
976/// returns bytes or answers with `size == 0`, which drops READABLE, so the
977/// exception cannot spin.
978fn update_readiness_after_read(
979 size: usize,
980 status: SocketResult,
981 readiness: &mut Readiness,
982) -> bool {
983 if size > 0 && status == SocketResult::WouldBlock && readiness.event.is_hup() {
984 return false;
985 }
986 update_readiness(size, status, readiness, Ready::READABLE)
987}
988
989fn update_readiness_after_write(
990 size: usize,
991 status: SocketResult,
992 readiness: &mut Readiness,
993) -> bool {
994 update_readiness(size, status, readiness, Ready::WRITABLE)
995}
996pub struct Context<L: ListenerHandler + L7ListenerHandler> {
997 pub streams: Vec<Stream>,
998 /// Streams whose state is `StreamState::Link` and need backend connection.
999 /// Replaces the O(n) scan of `streams` in the ready loop.
1000 pub pending_links: VecDeque<GlobalStreamId>,
1001 /// Reverse index: backend token -> global stream IDs currently in
1002 /// `StreamState::Linked(token)`. Eliminates O(n) scans of `streams`
1003 /// when handling backend connect/disconnect/timeout/close events.
1004 ///
1005 /// An entry may be empty: `remove_backend_stream` keeps an emptied `Vec`
1006 /// for the next link on the same live connection (#1583), so "has linked
1007 /// streams" is the entry's emptiness, not the key's presence.
1008 pub backend_streams: InlineTokenMap<LinkedStreams>,
1009 /// Where every buffer this session's streams and connections need comes
1010 /// from, and the only thing allowed to refuse one.
1011 ///
1012 /// Boxed rather than concrete so the core names the contract rather than
1013 /// the worker's [`Pool`]: a refusal is the same event whatever stands
1014 /// behind it, and the `REFUSED_STREAM` answer to one is the same answer.
1015 /// [`Context::new`] installs the [`PoolBufferSource`] the proxy runs on.
1016 pub buffers: Box<dyn BufferSource>,
1017 pub listener: Rc<RefCell<L>>,
1018 /// Connection/session ULID — mirrors `Mux.session_ulid`. Stored here so
1019 /// per-stream `HttpContext` construction in [`Self::create_stream`] can
1020 /// stamp the session slot of the log-context bracket without reaching
1021 /// back into the parent [`Mux`].
1022 pub session_ulid: Ulid,
1023 pub session_address: Option<SocketAddr>,
1024 pub public_address: SocketAddr,
1025 pub debug: DebugHistory,
1026 /// Shrink threshold ratio for recycled stream slots.
1027 /// Vec is shrunk when total_slots > active_streams * ratio.
1028 pub h2_stream_shrink_ratio: usize,
1029 /// TLS SNI value negotiated at handshake, propagated to every
1030 /// per-stream [`HttpContext`] so the routing layer can enforce
1031 /// the SNI ↔ `:authority` binding on every H2 stream (and the
1032 /// single H1 request). `None` for plaintext listeners or when
1033 /// the client omitted the SNI extension. Stored pre-lowercased
1034 /// and without a port for cheap exact-match comparison.
1035 pub tls_server_name: Option<String>,
1036 /// Snapshot of the SAN set of the certificate Sōzu actually served at
1037 /// the TLS handshake. Captured once in `https.rs::upgrade_handshake`
1038 /// from the resolver and frozen for the connection lifetime so H2
1039 /// stream coalescing (RFC 7540 §9.1.1 / RFC 9113 §9.1.1) accepts any
1040 /// `:authority` covered by the certificate, with RFC 6125 §6.4.3
1041 /// wildcard handling. `None` for plaintext listeners or when SNI was
1042 /// absent. `Some(empty)` when the default cert was served — every
1043 /// `:authority` is rejected. `Arc` so the snapshot is shared across
1044 /// every per-stream `HttpContext` without re-allocation.
1045 pub tls_cert_names: Option<Arc<Vec<String>>>,
1046 /// Whether this session's listener speaks HTTPS or plaintext HTTP.
1047 ///
1048 /// The one listener value that is genuinely per-connection rather than
1049 /// per-request: a listener's protocol is fixed by its type and no
1050 /// `update_config` patch can change it, so capturing it once in
1051 /// [`Self::new`] frees both readers — [`Self::create_stream`] and the H2
1052 /// write pass's `:scheme` decision — from borrowing the listener at all.
1053 /// Contrast the per-request knobs, which [`Self::create_stream`] re-reads
1054 /// on every stream precisely because a reload *can* change them
1055 /// (`LIFECYCLE.md` §2.5).
1056 pub protocol: Protocol,
1057 /// Negotiated TLS protocol version short-form (e.g. `"TLSv1.3"`).
1058 /// Captured once at handshake completion in `https.rs` and propagated
1059 /// to every per-stream [`HttpContext`] so the access log can record it
1060 /// without reaching back into the rustls session per request. `None`
1061 /// for plaintext listeners.
1062 pub tls_version: Option<&'static str>,
1063 /// Negotiated TLS cipher suite short-form (e.g.
1064 /// `"TLS_AES_128_GCM_SHA256"`). Captured once at handshake completion
1065 /// and propagated to every per-stream [`HttpContext`]. `None` for
1066 /// plaintext listeners.
1067 pub tls_cipher: Option<&'static str>,
1068 /// Negotiated ALPN protocol short-form (e.g. `"h2"`, `"http/1.1"`).
1069 /// Captured once at handshake completion and propagated to every
1070 /// per-stream [`HttpContext`]. `None` for plaintext listeners or when
1071 /// no ALPN was negotiated.
1072 pub tls_alpn: Option<&'static str>,
1073 /// Clock snapshot for the pass currently executing.
1074 ///
1075 /// [`Mux`] is the only clock sampler in the mux: it refreshes this field
1076 /// once per outer [`Mux::ready`] pass and at the top of [`Mux::timeout`]
1077 /// and [`Mux::shutting_down`]. Every time-based decision in the H2 core
1078 /// reads this snapshot — mirrored into
1079 /// `h2::ConnectionH2::now` at each public entry point — instead of
1080 /// calling [`Instant::now`] itself, so one pass sees one consistent
1081 /// "now". Every deadline armed or evaluated inside a pass is therefore
1082 /// accurate to within that pass — **in either direction**. The error is
1083 /// not one-sided: an arm site runs at an arbitrary depth into its pass
1084 /// (a DATA or HEADERS refresh, an outbound-byte refresh) and stamps the
1085 /// snapshot taken at the START of that pass, so the stored instant is
1086 /// older than the event it records; an eval site runs near the top of a
1087 /// pass (`cancel_timed_out_streams` is the first thing
1088 /// `h2::ConnectionH2::poll_read_target` does). The measured age is therefore
1089 /// inflated by the arm site's depth, and a deadline can fire up to one
1090 /// pass EARLY as well as one pass late. At base both ends read the real
1091 /// clock and the comparison was exact; this is the cost of the snapshot.
1092 ///
1093 /// The flood and back-pressure windows are the one asymmetric case, and
1094 /// they fail closed: `now` is constant for the whole pass, so a window
1095 /// cannot decay part-way through one. A burst arriving during a pass is
1096 /// weighed in full against the window that was open when the pass
1097 /// started, where before the change a long pass could decay the counters
1098 /// under the burst. The window BOUNDARY still carries the same
1099 /// one-pass error in either direction as every other deadline.
1100 ///
1101 /// Note that [`crate::SessionMetrics`] and [`crate::timer::TimeoutContainer`]
1102 /// deliberately stay on the real clock: metrics want true elapsed time,
1103 /// and the timeout containers are the embedder's timer, held by the [`Mux`]
1104 /// adapter in [`Mux::timeouts`] and never touched by a core — which
1105 /// publishes a deadline stamped from this snapshot instead.
1106 pub now: Instant,
1107 /// Wall-clock sibling of [`Self::now`]: Unix milliseconds for the pass
1108 /// currently executing.
1109 ///
1110 /// [`Self::now`] is an [`Instant`] — an opaque monotonic point with no
1111 /// epoch and no defined conversion to civil time — so it cannot supply
1112 /// the 48-bit Unix-millisecond prefix a ULID carries in its high bits
1113 /// (see [`Self::next_request_id`]). This field is that value, and it is
1114 /// refreshed by [`Mux`] at exactly the three sites that refresh
1115 /// [`Self::now`], from the same pass. The two are siblings: a site that
1116 /// sets one and not the other lets a request id carry the timestamp of
1117 /// an older pass.
1118 ///
1119 /// It is a snapshot rather than a provider closure for the same reason
1120 /// [`Self::now`] is: the mux's discipline is one clock sample per pass,
1121 /// read by the core from a field, and a boxed provider would let the
1122 /// core reach the host at an arbitrary depth into a pass instead.
1123 pub now_wall_ms: u64,
1124 /// Entropy source for the 80-bit random suffix of a request ULID.
1125 ///
1126 /// `Ulid::generate()` is exactly
1127 /// `Ulid::from_timestamp_with_rng(unix_epoch_ms(), &mut rand::rng())` —
1128 /// two ambient reaches (the wall clock and a thread-local RNG) hidden
1129 /// inside a dependency, invisible to any grep over this module for
1130 /// `Instant::now` / `SystemTime::now` / `rand`. Holding the RNG here
1131 /// replaces the second with a field the embedder owns, exactly as
1132 /// [`Self::now_wall_ms`] replaces the first.
1133 ///
1134 /// Seeded once per session in [`Self::new`] from `rand::rng()` — the
1135 /// same OS-backed reseeding CSPRNG `Ulid::generate()` draws from, so the
1136 /// unpredictability of a request id is unchanged. A request id that
1137 /// repeats across sessions would be a regression, so nothing in the mux
1138 /// re-seeds it; a deterministic simulator assigns this field directly
1139 /// after construction instead.
1140 pub request_id_rng: StdRng,
1141 /// Backend accounting the core decided and the embedder has not performed
1142 /// yet (#1340, Question 12).
1143 ///
1144 /// The core's outbound channel for registry writes, in the same place
1145 /// `udp`'s `Output` queue sits: a site that used to reach a
1146 /// `Rc<RefCell<Backend>>` through `Position::Client` and mutate it pushes
1147 /// a [`BackendDelta`] here instead. `Mux::apply_backend_deltas` is the
1148 /// only drain, and the only writer of `Backend::active_requests` /
1149 /// `Backend::active_connections` on the mux path.
1150 ///
1151 /// Entries are applied in the order they were pushed, and the drain runs
1152 /// before anything reads backend load state — in practice, before each
1153 /// `Router::plan_connect`, whose load balancer is that sole reader. FIFO plus
1154 /// drain-before-read leaves the counters at every read point exactly
1155 /// where mutating in place left them, saturation included.
1156 pub backend_deltas: Vec<BackendDelta>,
1157 /// Response bytes this session's frontend has handed to its socket, over
1158 /// the whole session. Only ever grows (wrapping), unlike the per-stream
1159 /// `SessionMetrics::bout`, which `SessionMetrics::reset` zeroes when a
1160 /// stream ends and its slot is reused. `ConnectionH1::writable` and
1161 /// `H2Shell::write_streams` add every stream byte a server-position
1162 /// write accepted.
1163 pub(super) client_bytes_out: usize,
1164 /// [`Self::client_bytes_out`] when the inner loop of `Mux::ready_inner`
1165 /// last reached `MAX_LOOP_ITERATIONS` and waited for the client instead
1166 /// of closing. A second time with no byte written to the client since
1167 /// closes the session: waiting is only worth it while the client drains
1168 /// (`Mux::wait_for_client_at_loop_limit`).
1169 pub(super) loop_limit_bytes_out: Option<usize>,
1170 /// Stream bytes this session has handed to any socket, the client's or
1171 /// a backend's, over the whole session. Only ever grows (wrapping).
1172 /// `ConnectionH1::writable` and `H2Shell::write_streams` add every stream
1173 /// byte a write accepted, whatever the connection's position; H2 control
1174 /// frames are not stream bytes and do not count. When the inner loop of
1175 /// `Mux::ready_inner` reaches `MAX_LOOP_ITERATIONS` and this moved by at
1176 /// least one byte per budgeted iteration during the call
1177 /// (`progressed_through_loop_budget`), the session is progressing, not
1178 /// spinning: it yields to the event loop instead of closing
1179 /// (sozu-proxy/sozu#1906).
1180 pub(super) stream_bytes_out: usize,
1181 /// Set when the last `Mux::ready_inner` call yielded at
1182 /// `MAX_LOOP_ITERATIONS` with its readiness left as it was. The session
1183 /// reports it through `ProxySession::yielded`, which takes it, and
1184 /// `Server::resume_yielded_sessions` runs `ready` again on the next turn
1185 /// of the event loop: no socket edge would come back for work the loop
1186 /// left unread.
1187 pub(super) loop_budget_yielded: bool,
1188}
1189
1190/// Unix milliseconds for the wall clock, saturating to 0 before the epoch.
1191///
1192/// The single host wall-clock reach behind [`Context::now_wall_ms`]. Kept as
1193/// one function so the three [`Mux`] refresh sites and [`Context::new`] cannot
1194/// drift apart on the unit or on the pre-epoch case.
1195fn unix_epoch_ms() -> u64 {
1196 SystemTime::now()
1197 .duration_since(UNIX_EPOCH)
1198 .map(|d| d.as_millis() as u64)
1199 .unwrap_or(0)
1200}
1201
1202impl<L: ListenerHandler + L7ListenerHandler> Context<L> {
1203 pub fn new(
1204 session_ulid: Ulid,
1205 pool: Weak<RefCell<Pool>>,
1206 listener: Rc<RefCell<L>>,
1207 session_address: Option<SocketAddr>,
1208 public_address: SocketAddr,
1209 ) -> Self {
1210 let h2_stream_shrink_ratio = listener
1211 .borrow()
1212 .get_h2_connection_config()
1213 .stream_shrink_ratio as usize;
1214 let protocol = listener.borrow().protocol();
1215 Self {
1216 streams: Vec::new(),
1217 pending_links: VecDeque::new(),
1218 backend_streams: InlineTokenMap::new(),
1219 buffers: Box::new(PoolBufferSource::new(pool)),
1220 listener,
1221 session_ulid,
1222 session_address,
1223 public_address,
1224 debug: DebugHistory::new(),
1225 h2_stream_shrink_ratio,
1226 tls_server_name: None,
1227 tls_cert_names: None,
1228 protocol,
1229 tls_version: None,
1230 tls_cipher: None,
1231 tls_alpn: None,
1232 now: Instant::now(),
1233 now_wall_ms: unix_epoch_ms(),
1234 request_id_rng: StdRng::from_rng(&mut rand::rng()),
1235 backend_deltas: Vec::new(),
1236 client_bytes_out: 0,
1237 loop_limit_bytes_out: None,
1238 stream_bytes_out: 0,
1239 loop_budget_yielded: false,
1240 }
1241 }
1242
1243 /// Record one backend accounting change for the embedder to perform.
1244 ///
1245 /// The single push site, so every emitter reads the same way and a
1246 /// `grep` for `record_backend_delta` enumerates the ledger's inputs.
1247 pub(super) fn record_backend_delta(&mut self, backend: &BackendId, change: BackendChange) {
1248 self.backend_deltas.push(BackendDelta {
1249 slot: backend.slot(),
1250 change,
1251 });
1252 }
1253
1254 /// Mint the request ULID for a stream this session is about to open, or
1255 /// for the next request of an H1 keep-alive connection.
1256 ///
1257 /// The mux's two production ULID sites call it: `ConnectionH2::create_stream`
1258 /// (`lib/src/protocol/mux/h2.rs`) for each H2 stream, and the keep-alive
1259 /// branch of `ConnectionH1::writable` (`lib/src/protocol/mux/h1.rs`), which
1260 /// hands it to `HttpContext::reset`. It replaces `Ulid::generate()` there.
1261 /// `generate()` composes two ambient reaches — the wall clock and a
1262 /// thread-local RNG — inside `rusty_ulid`; this composes the same ULID
1263 /// from [`Self::now_wall_ms`] and [`Self::request_id_rng`], the two
1264 /// sources the embedder owns. Same bytes, same layout, no host reach
1265 /// from inside the core.
1266 ///
1267 /// `Ulid::from_timestamp_with_rng` panics above `0xFFFF_FFFF_FFFF` ms
1268 /// (year 10889); [`Self::now_wall_ms`] is a Unix-millisecond count, so
1269 /// reaching it requires a host clock set eight millennia ahead.
1270 pub fn next_request_id(&mut self) -> Ulid {
1271 Ulid::from_timestamp_with_rng(self.now_wall_ms, &mut self.request_id_rng)
1272 }
1273
1274 pub fn active_len(&self) -> usize {
1275 self.streams
1276 .iter()
1277 .filter(|s| !matches!(s.state, StreamState::Recycle))
1278 .count()
1279 }
1280
1281 /// Shared accessor for the [`HttpContext`] owned by a stream.
1282 ///
1283 /// Prefer this over `&self.streams[stream_id].context` at call sites
1284 /// that only need read access — it keeps the `Stream`/`HttpContext`
1285 /// relationship encapsulated and reads the same regardless of whether
1286 /// the caller is inside `Router::plan_connect`, the H2 mux, or a free
1287 /// helper. Panics on an out-of-bounds `stream_id`, which is the same
1288 /// behaviour as the raw `streams[sid]` indexing it replaces.
1289 pub fn http_context(&self, stream_id: GlobalStreamId) -> &HttpContext {
1290 &self.streams[stream_id].context
1291 }
1292
1293 /// Mutable sibling of [`Self::http_context`]. Use when routing
1294 /// decisions need to stamp `cluster_id` / `backend_id` on the stream's
1295 /// [`HttpContext`] (e.g. `Router::plan_connect` at the fill-cluster /
1296 /// fill-backend points).
1297 pub fn http_context_mut(&mut self, stream_id: GlobalStreamId) -> &mut HttpContext {
1298 &mut self.streams[stream_id].context
1299 }
1300
1301 /// Register a stream as linked to a backend token in the reverse index.
1302 pub fn link_stream(&mut self, stream_id: GlobalStreamId, token: Token) {
1303 self.streams[stream_id].state = StreamState::Linked(token);
1304 // An H1 backend carries one stream at a time, which a new entry
1305 // holds inline (#1613); an H2 backend spills into a `Vec`.
1306 self.backend_streams
1307 .get_or_insert_with(token, LinkedStreams::default)
1308 .push(stream_id);
1309 }
1310
1311 /// Remove a stream from the backend reverse index if it is currently
1312 /// `Linked`. Returns the backend token if one was removed.
1313 pub fn unlink_stream(&mut self, stream_id: GlobalStreamId) -> Option<Token> {
1314 if let StreamState::Linked(token) = self.streams[stream_id].state {
1315 remove_backend_stream(&mut self.backend_streams, token, stream_id);
1316 Some(token)
1317 } else {
1318 None
1319 }
1320 }
1321
1322 /// Open (or recycle) the stream slot for a request that has just arrived,
1323 /// capturing the listener configuration it will run on.
1324 ///
1325 /// This is the mux's single listener-snapshot point. `ConnectionH2` calls
1326 /// it from its HEADERS branch, so an H2 stream captures the listener as
1327 /// the operator had it configured when that request's HEADERS landed, and
1328 /// finishes on that capture whatever a concurrent reload does; the next
1329 /// HEADERS on the same connection takes a fresh one. The H1 sessions in
1330 /// `http.rs` / `https.rs` call it once per connection instead, because one
1331 /// H1 connection owns one slot that `HttpContext::reset` deliberately
1332 /// carries across keep-alive requests. `LIFECYCLE.md` §2.5 states the
1333 /// resulting staleness window for both.
1334 ///
1335 /// Every value read under the borrow below is re-read on each call for
1336 /// that reason. The two that are NOT read here — [`Self::protocol`] and
1337 /// the TLS fields — are connection-scoped by nature and captured once.
1338 ///
1339 /// `window` seeds [`Stream::front_window`], the stream's send window on
1340 /// the frontend connection. The backend leg's window is the backend's to
1341 /// size: `ConnectionH2::start_stream` sets [`Stream::back_window`].
1342 pub fn create_stream(&mut self, request_id: Ulid, window: u32) -> Option<GlobalStreamId> {
1343 let (http_context, answers) = {
1344 let listener = self.listener.borrow();
1345 let mut http_context = HttpContext::new(
1346 self.session_ulid,
1347 request_id,
1348 self.protocol,
1349 self.public_address,
1350 self.session_address,
1351 listener.get_sticky_name().to_string(),
1352 listener.get_sozu_id_header().to_string(),
1353 listener.get_elide_x_real_ip(),
1354 listener.get_send_x_real_ip(),
1355 );
1356 // Propagate the connection-scoped TLS SNI onto every per-stream
1357 // HttpContext so `route_from_request` can enforce the SNI ↔
1358 // `:authority` binding for each H2 stream independently.
1359 http_context.tls_server_name = self.tls_server_name.clone();
1360 // Mirror the frozen-at-handshake SAN snapshot. `Arc` clone is a
1361 // refcount bump, not a deep copy — every per-stream
1362 // `HttpContext` shares the same `Vec<String>`.
1363 http_context.tls_cert_names = self.tls_cert_names.clone();
1364 // Mirror the listener's strict_sni_binding flag onto each
1365 // HttpContext so the routing layer can honor operator opt-outs
1366 // without reaching back into the listener on every request.
1367 http_context.strict_sni_binding = listener.get_strict_sni_binding();
1368 // Mirror the listener's forwarding header family the same way:
1369 // a hot update reaches the next stream (H2) or connection (H1).
1370 http_context.forwarded_headers = listener.get_forwarded_headers();
1371 // The H1 trailer field bound is the listener's H2 one, so both
1372 // frontends admit the same number of trailer fields (#1701).
1373 http_context.max_trailer_fields = listener.get_h2_flood_config().max_header_fields();
1374 // Propagate the connection-scoped TLS metadata onto every
1375 // per-stream HttpContext so the access log can record it without
1376 // touching the rustls session on every request. These are
1377 // `&'static str` borrows from the rustls label tables — copy is
1378 // a pointer move.
1379 http_context.tls_version = self.tls_version;
1380 http_context.tls_cipher = self.tls_cipher;
1381 http_context.tls_alpn = self.tls_alpn;
1382 // The answer registry this request will render any default answer
1383 // from. Cloning the handle here — rather than borrowing the
1384 // listener again at each `set_default_answer` site — is what binds
1385 // a stream to the templates that were installed when it started.
1386 (http_context, listener.get_answers().clone())
1387 };
1388 let recycle_slot = self
1389 .streams
1390 .iter()
1391 .position(|s| s.state == StreamState::Recycle);
1392 if let Some(stream_id) = recycle_slot {
1393 let stream = &mut self.streams[stream_id];
1394 trace!("{} Reuse stream: {}", log_module_context!(), stream_id);
1395 stream.state = StreamState::Idle;
1396 stream.attempts = 0;
1397 stream.tried_backends.clear();
1398 stream.front_received_end_of_stream = false;
1399 stream.back_received_end_of_stream = false;
1400 stream.front_data_received = 0;
1401 stream.back_data_received = 0;
1402 // The ONE place `request_counted` is cleared without emitting the
1403 // paired `http.active_requests` decrement, and it must stay the
1404 // only one. A slot reaches `StreamState::Recycle` only through
1405 // `Stream::generate_access_log`, which already emitted the `-1`
1406 // and cleared the flag, so this is re-initialisation of an
1407 // already-false field rather than a decrement that went missing.
1408 // `Stream::new` asserts the same invariant for a fresh slot
1409 // ("new stream must not have a counted request"); this is its
1410 // reuse-path counterpart. A future site that clears this flag
1411 // without accounting for the gauge is a silent upward leak: the
1412 // `+1` stays in the aggregate with nothing left to pair it, and
1413 // an aggregate that only drifts up never underflows, so nothing
1414 // logs and nothing saturates.
1415 stream.request_counted = false;
1416 stream.front_window = i32::try_from(window).unwrap_or(i32::MAX);
1417 stream.back_window = i32::try_from(h2::DEFAULT_INITIAL_WINDOW_SIZE).unwrap_or(i32::MAX);
1418 stream.context = http_context;
1419 // A recycled slot takes the fresh capture too: the request that
1420 // released it ran on whatever the listener held then, and the one
1421 // taking it over must not inherit that.
1422 stream.answers = answers;
1423 stream.back.clear();
1424 stream.back.storage.clear();
1425 stream.front.clear();
1426 stream.front.storage.clear();
1427 // The request that released the slot may have been encoded for
1428 // an H2 backend; the one taking it over was not.
1429 stream.front_bound_to_backend = false;
1430 stream.forget_upstream_replay();
1431 stream.metrics.reset();
1432 stream.metrics.mark_request_start();
1433 // After recycling a slot, check if the Vec has excessive trailing
1434 // Recycle entries (more than 2x active streams of total capacity).
1435 let active = self.active_len();
1436 let total = self.streams.len();
1437 if total > 1 && active > 0 && total > active * self.h2_stream_shrink_ratio {
1438 self.shrink_trailing_recycle();
1439 }
1440 return Some(stream_id);
1441 }
1442 let stream = Stream::new(&mut *self.buffers, http_context, answers, window)?;
1443 self.streams.push(stream);
1444 Some(self.streams.len() - 1)
1445 }
1446
1447 /// Remove consecutive `Recycle` entries from the end of the streams Vec.
1448 ///
1449 /// This prevents unbounded growth when H2 streams are created and recycled
1450 /// over time, reclaiming memory from slots that are no longer needed.
1451 pub fn shrink_trailing_recycle(&mut self) {
1452 while self
1453 .streams
1454 .last()
1455 .is_some_and(|s| s.state == StreamState::Recycle)
1456 {
1457 self.streams.pop();
1458 }
1459 }
1460}
1461
1462/// A map keyed by [`Token`] that holds its lowest-token entry inline.
1463///
1464/// A session almost always has exactly one backend connection, so the two
1465/// per-backend indexes a mux session keeps — [`Context::backend_streams`] and
1466/// the backend half of [`MuxTimeouts`] — almost always hold one entry. A
1467/// `HashMap` allocated its table on that first insert, once per session and
1468/// so once per request on a one-request connection (#1613). Here the first
1469/// entry costs nothing and only a second backend reaches the overflow map.
1470///
1471/// The shape is [`BackendConnections`](router::BackendConnections)'s: `first`
1472/// is `None` only when `rest` is empty, and its token is below every token in
1473/// `rest`, so iteration runs in ascending `Token` order.
1474pub struct InlineTokenMap<V> {
1475 first: Option<(Token, V)>,
1476 rest: BTreeMap<Token, V>,
1477}
1478
1479impl<V> InlineTokenMap<V> {
1480 pub const fn new() -> Self {
1481 Self {
1482 first: None,
1483 rest: BTreeMap::new(),
1484 }
1485 }
1486
1487 pub fn is_empty(&self) -> bool {
1488 self.first.is_none()
1489 }
1490
1491 pub fn contains_key(&self, token: &Token) -> bool {
1492 self.get(token).is_some()
1493 }
1494
1495 pub fn get(&self, token: &Token) -> Option<&V> {
1496 match &self.first {
1497 Some((first, value)) if first == token => Some(value),
1498 _ => self.rest.get(token),
1499 }
1500 }
1501
1502 pub fn get_mut(&mut self, token: &Token) -> Option<&mut V> {
1503 match &mut self.first {
1504 Some((first, value)) if first == token => Some(value),
1505 _ => self.rest.get_mut(token),
1506 }
1507 }
1508
1509 /// The value under `token`, inserting `default()` first when there is none.
1510 pub fn get_or_insert_with(&mut self, token: Token, default: impl FnOnce() -> V) -> &mut V {
1511 let first_token = self.first.as_ref().map(|(first, _)| *first);
1512 if first_token.is_some_and(|first| token > first) {
1513 return self.rest.entry(token).or_insert_with(default);
1514 }
1515 if first_token.is_some_and(|first| token < first)
1516 && let Some((displaced_token, displaced)) = self.first.take()
1517 {
1518 self.rest.insert(displaced_token, displaced);
1519 }
1520 &mut self.first.get_or_insert_with(|| (token, default())).1
1521 }
1522
1523 /// Remove and return the value under `token`.
1524 pub fn remove(&mut self, token: &Token) -> Option<V> {
1525 match &self.first {
1526 Some((first, _)) if first == token => {
1527 let removed = std::mem::replace(&mut self.first, self.rest.pop_first());
1528 removed.map(|(_, value)| value)
1529 }
1530 _ => self.rest.remove(token),
1531 }
1532 }
1533
1534 /// Keep only the entries `keep` answers `true` for.
1535 pub fn retain(&mut self, mut keep: impl FnMut(&Token, &mut V) -> bool) {
1536 let keep_first = match &mut self.first {
1537 Some((token, value)) => keep(token, value),
1538 None => true,
1539 };
1540 self.rest.retain(|token, value| keep(token, value));
1541 if !keep_first {
1542 self.first = self.rest.pop_first();
1543 }
1544 }
1545
1546 pub fn clear(&mut self) {
1547 self.first = None;
1548 self.rest.clear();
1549 }
1550
1551 /// The entries in ascending `Token` order.
1552 pub fn iter(&self) -> impl Iterator<Item = (&Token, &V)> {
1553 self.first
1554 .iter()
1555 .map(|(token, value)| (token, value))
1556 .chain(self.rest.iter())
1557 }
1558
1559 pub fn values(&self) -> impl Iterator<Item = &V> {
1560 self.iter().map(|(_, value)| value)
1561 }
1562}
1563
1564impl<V> Default for InlineTokenMap<V> {
1565 fn default() -> Self {
1566 Self::new()
1567 }
1568}
1569
1570impl<V: Debug> Debug for InlineTokenMap<V> {
1571 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1572 f.debug_map().entries(self.iter()).finish()
1573 }
1574}
1575
1576/// The streams linked to one backend connection, in link order.
1577///
1578/// An H1 backend carries one stream at a time, so the first link is held
1579/// inline and allocates nothing; a second concurrent link (an H2 backend)
1580/// moves both into a `Vec` sized for four — the allocation the `Vec` this
1581/// replaces made at the same point — and the entry keeps that capacity for
1582/// the rest of the connection's life (#1583).
1583pub enum LinkedStreams {
1584 Inline(Option<GlobalStreamId>),
1585 Spilled(Vec<GlobalStreamId>),
1586}
1587
1588impl LinkedStreams {
1589 pub fn push(&mut self, stream_id: GlobalStreamId) {
1590 match self {
1591 LinkedStreams::Inline(None) => *self = LinkedStreams::Inline(Some(stream_id)),
1592 LinkedStreams::Inline(Some(held)) => {
1593 let mut ids = Vec::with_capacity(4);
1594 ids.push(*held);
1595 ids.push(stream_id);
1596 *self = LinkedStreams::Spilled(ids);
1597 }
1598 LinkedStreams::Spilled(ids) => ids.push(stream_id),
1599 }
1600 }
1601
1602 /// Drop `stream_id` from the list, keeping the order of the others.
1603 pub fn remove(&mut self, stream_id: GlobalStreamId) {
1604 match self {
1605 LinkedStreams::Inline(held) => {
1606 if *held == Some(stream_id) {
1607 *held = None;
1608 }
1609 }
1610 LinkedStreams::Spilled(ids) => ids.retain(|&id| id != stream_id),
1611 }
1612 }
1613}
1614
1615impl Default for LinkedStreams {
1616 fn default() -> Self {
1617 LinkedStreams::Inline(None)
1618 }
1619}
1620
1621impl std::ops::Deref for LinkedStreams {
1622 type Target = [GlobalStreamId];
1623
1624 fn deref(&self) -> &[GlobalStreamId] {
1625 match self {
1626 LinkedStreams::Inline(held) => held.as_slice(),
1627 LinkedStreams::Spilled(ids) => ids,
1628 }
1629 }
1630}
1631
1632impl Debug for LinkedStreams {
1633 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1634 f.debug_list().entries(self.iter()).finish()
1635 }
1636}
1637
1638/// The `Mux` adapter's timer-wheel handles: the frontend's, and one per live
1639/// backend connection.
1640///
1641/// The frontend's key never changes for the life of a session, so its handle
1642/// is a named field; the backends' are an open set, held in an
1643/// [`InlineTokenMap`] so the usual single backend costs no allocation. The
1644/// `HashMap<Token, TimeoutContainer>` this replaces allocated its table when
1645/// the session was built (#1613).
1646///
1647/// HAProxy goes further: a stream's deadlines are fixed fields (`conn_exp` in
1648/// `include/haproxy/stream-t.h`, each channel's `analyse_exp` in
1649/// `include/haproxy/channel-t.h`) that `process_stream` folds into its one
1650/// task's `expire` with `tick_first` (`src/stream.c`), so there is no table at
1651/// all. A sozu session keeps one wheel entry per connection instead, because
1652/// each connection's core publishes its own deadline.
1653#[derive(Debug)]
1654pub struct MuxTimeouts {
1655 pub frontend: TimeoutContainer,
1656 pub backends: InlineTokenMap<TimeoutContainer>,
1657}
1658
1659impl MuxTimeouts {
1660 pub const fn new(frontend: TimeoutContainer) -> Self {
1661 Self {
1662 frontend,
1663 backends: InlineTokenMap::new(),
1664 }
1665 }
1666}
1667
1668/// Remove `stream_id` from the backend-token reverse index for `token`.
1669/// Free function to allow split borrows when `context.streams` is already
1670/// mutably borrowed (preventing a `Context::unlink_stream` call).
1671///
1672/// An emptied entry is kept, not removed: the next `Context::link_stream` on
1673/// the same backend connection — every request on a keep-alive socket or an
1674/// H2 multiplex slot — pushes into its retained capacity instead of
1675/// allocating a fresh `Vec` (#1583). The key leaves with the connection
1676/// itself, in the dead-backend sweep of `Mux::ready_inner`, and `Mux::close`
1677/// clears the whole index, so the index stays bounded by the session's live
1678/// backend connections.
1679/// [`connect_outcome`] against the three states a dialled socket can be in
1680/// when sozu sees WRITABLE on it (sozu-proxy/sozu#1800).
1681#[cfg(test)]
1682mod connect_outcome_tests {
1683 use std::{
1684 net::{SocketAddr, TcpListener},
1685 os::fd::AsRawFd,
1686 time::{Duration, Instant},
1687 };
1688
1689 use super::{ConnectOutcome, connect_outcome};
1690
1691 /// Poll `connect_outcome` until it leaves `Pending` or `deadline` passes.
1692 fn settle(socket: &mio::net::TcpStream, deadline: Duration) -> ConnectOutcome {
1693 let started = Instant::now();
1694 loop {
1695 let outcome = connect_outcome(socket);
1696 if outcome != ConnectOutcome::Pending || started.elapsed() >= deadline {
1697 return outcome;
1698 }
1699 std::thread::sleep(Duration::from_millis(10));
1700 }
1701 }
1702
1703 #[test]
1704 fn an_accepted_dial_is_connected() {
1705 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1706 let socket = mio::net::TcpStream::connect(listener.local_addr().unwrap()).unwrap();
1707 assert_eq!(
1708 settle(&socket, Duration::from_secs(2)),
1709 ConnectOutcome::Connected
1710 );
1711 }
1712
1713 /// A `unix_path` backend's socket carries an AF_UNIX peer, which std's
1714 /// `peer_addr` refuses: the outcome must still read Connected.
1715 #[test]
1716 fn an_accepted_unix_dial_is_connected() {
1717 let path =
1718 std::env::temp_dir().join(format!("sozu-connect-outcome-{}.sock", std::process::id()));
1719 let _ = std::fs::remove_file(&path);
1720 let _listener = std::os::unix::net::UnixListener::bind(&path).unwrap();
1721 let unused: SocketAddr = "127.0.0.1:1".parse().unwrap();
1722 let socket = crate::backends::connect_backend(unused, path.to_str()).unwrap();
1723 let outcome = settle(&socket, Duration::from_secs(2));
1724 std::fs::remove_file(&path).unwrap();
1725 assert_eq!(outcome, ConnectOutcome::Connected);
1726 }
1727
1728 #[test]
1729 fn a_refused_dial_failed() {
1730 let address: SocketAddr = {
1731 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1732 listener.local_addr().unwrap()
1733 };
1734 // A loopback refusal may also be reported by `connect(2)` itself, in
1735 // which case there is no socket to read the outcome from.
1736 let Ok(socket) = mio::net::TcpStream::connect(address) else {
1737 return;
1738 };
1739 assert_eq!(
1740 settle(&socket, Duration::from_secs(2)),
1741 ConnectOutcome::Failed
1742 );
1743 }
1744
1745 #[test]
1746 fn a_dial_nobody_answers_is_pending() {
1747 // A full accept queue drops the SYN, so the dial stays in flight.
1748 let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1749 // SAFETY: shrinks the backlog of a socket this test owns.
1750 assert_eq!(unsafe { libc::listen(listener.as_raw_fd(), 0) }, 0);
1751 let address = listener.local_addr().unwrap();
1752 let mut queued = Vec::new();
1753 while let Ok(stream) =
1754 std::net::TcpStream::connect_timeout(&address, Duration::from_millis(300))
1755 {
1756 queued.push(stream);
1757 assert!(queued.len() < 16, "the accept queue never filled");
1758 }
1759 let socket = mio::net::TcpStream::connect(address).unwrap();
1760 std::thread::sleep(Duration::from_millis(100));
1761 assert_eq!(connect_outcome(&socket), ConnectOutcome::Pending);
1762 }
1763}
1764
1765/// Where the non-blocking `connect(2)` behind a connecting backend socket
1766/// stands when its readiness carries WRITABLE.
1767#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1768enum ConnectOutcome {
1769 /// The connection is established.
1770 Connected,
1771 /// The connect is still in progress: the WRITABLE was not the kernel's
1772 /// completion edge but one sozu raised itself (`Readiness::
1773 /// signal_pending_write` to flush queued output). The completion edge is
1774 /// still to come.
1775 Pending,
1776 /// The connect failed.
1777 Failed,
1778}
1779
1780/// Read the outcome of the non-blocking `connect(2)` behind `socket`.
1781///
1782/// The kernel reports it through the socket's pending error (`SO_ERROR`,
1783/// POSIX `connect(2)`): a WRITABLE edge means the attempt finished, not that
1784/// it succeeded, and reading the error clears it. An empty error proves
1785/// nothing alone, because the WRITABLE may be one sozu signalled itself while
1786/// the dial is still in flight, so `getpeername(2)` then tells an
1787/// established connection from a pending one (`ENOTCONN`). An error reading
1788/// either is a failed connect: nothing past it could use the socket
1789/// (sozu-proxy/sozu#1800).
1790fn connect_outcome(socket: &TcpStream) -> ConnectOutcome {
1791 match socket.take_error() {
1792 Ok(None) => {}
1793 Ok(Some(error)) | Err(error) => {
1794 // The level of the synchronous refusal the pending-links loop
1795 // of `Mux::ready_inner` logs before it retries.
1796 debug!(
1797 "{} backend connect failed, retrying: {}",
1798 log_module_context!(),
1799 error
1800 );
1801 return ConnectOutcome::Failed;
1802 }
1803 }
1804 // `socket2` rather than std's `peer_addr`, which refuses the AF_UNIX
1805 // address of a `unix_path` backend (`backends::connect_backend`).
1806 match socket2::SockRef::from(socket).peer_addr() {
1807 Ok(_) => ConnectOutcome::Connected,
1808 Err(error) if error.kind() == ErrorKind::NotConnected => ConnectOutcome::Pending,
1809 Err(_) => ConnectOutcome::Failed,
1810 }
1811}
1812
1813pub(super) fn remove_backend_stream(
1814 index: &mut InlineTokenMap<LinkedStreams>,
1815 token: Token,
1816 stream_id: GlobalStreamId,
1817) {
1818 if let Some(ids) = index.get_mut(&token) {
1819 ids.remove(stream_id);
1820 }
1821}
1822
1823pub struct Mux<Front: SocketHandler, L: ListenerHandler + L7ListenerHandler> {
1824 pub configured_frontend_timeout: Duration,
1825 pub frontend_token: Token,
1826 pub frontend: Connection<Front>,
1827 pub router: Router,
1828 pub context: Context<L>,
1829 /// The adapter's wheel handles, one per live token (the frontend plus
1830 /// every backend in `router.backends`).
1831 ///
1832 /// The H1/H2 cores no longer own a `TimeoutContainer`: they publish the
1833 /// instant they want to be called back at through `Connection::poll_timeout`
1834 /// and `Mux::reschedule` reflects that onto `crate::timer`, arming only
1835 /// when what the wheel holds differs from what the core wants. This map is
1836 /// the ONLY thing in the mux that talks to the timer wheel.
1837 ///
1838 /// A handle for a token that has left `router.backends` is dropped by
1839 /// `reschedule`'s `retain`, and `TimeoutContainer::drop` cancels its entry
1840 /// — which is why no backend-removal site has to remember to cancel.
1841 pub timeouts: MuxTimeouts,
1842 /// The registry handles the core's [`BackendSlot`]s stand for.
1843 ///
1844 /// The embedder's half of Question 12's perimeter: the core carries an
1845 /// opaque slot, this resolves it, and `Mux::apply_backend_deltas` is
1846 /// what turns the core's [`BackendDelta`]s into writes on the worker's
1847 /// `Backend` values. Indexed by slot.
1848 ///
1849 /// A dial reuses an existing slot when `Rc::ptr_eq` matches, so the
1850 /// vector is bounded by the number of DISTINCT backends a session dials
1851 /// rather than by its request count. Entries are never removed while the
1852 /// session lives: a delta pushed before a backend connection closed must
1853 /// still resolve when the drain reaches it, and an index that moved would
1854 /// silently mis-charge it.
1855 pub(crate) backend_registry: BackendRegistry,
1856 /// The worker's backend set, which `Mux::dial_backend` lends to
1857 /// `Router::backend_from_request` through a `RegistrySelector`.
1858 ///
1859 /// Handed over when the session is built rather than read through
1860 /// `L7Proxy` on every dial: `HttpProxy` and `HttpsProxy` never reassign
1861 /// their `backends` field, so this is the same map, and a dial no longer
1862 /// borrows the proxy or clones the handle to reach it.
1863 pub(crate) backends: Rc<RefCell<BackendMap>>,
1864 /// Per-session correlation ID generated at construction time. Included in
1865 /// every log line emitted from this module so all events for a single
1866 /// frontend connection can be reassembled (independent of the ephemeral
1867 /// per-stream request id used by access logs).
1868 pub session_ulid: Ulid,
1869}
1870
1871/// Answer the per-`(cluster, source-IP)` limit step the core paused at.
1872///
1873/// The embedder half of [`router::ConnectStep::CheckIpLimit`]. The core
1874/// cannot do this itself: it is a consult of
1875/// `SessionManager::cluster_ip_at_limit` **and**, on the admitting path, a
1876/// call to `SessionManager::track_cluster_ip`, which mutates
1877/// `connections_per_cluster_ip` — worker-global state keyed on
1878/// `(cluster, ip)` across every session, and read again by
1879/// `lib/src/tcp.rs`'s own gate for the TCP proxy.
1880///
1881/// **The track happens here, before the decision resumes, and that ordering
1882/// is the point.** An `Admitted` verdict asserts to the core that the slot
1883/// has been taken. Resuming as `Admitted` without having tracked would let
1884/// the stream through uncounted, and because the counter is worker-global
1885/// the next session from the same IP would find a slot that should already
1886/// be gone. `an_admitted_stream_is_counted_before_the_decision_resumes`
1887/// pins it.
1888///
1889/// A free function rather than a `Mux` method so it can be driven directly:
1890/// the ordering above is the thing worth testing, and testing it through a
1891/// copy of this logic would prove nothing about this logic.
1892fn consult_ip_gate(
1893 sessions: &Rc<RefCell<crate::server::SessionManager>>,
1894 frontend_token: Token,
1895 // The cluster routing resolved, borrowed from the stream's `HttpContext`
1896 // where `Router::plan_connect` stored it (#1583).
1897 cluster_id: &sozu_command::state::ClusterId,
1898 resume: &router::ConnectResume,
1899) -> router::IpGateVerdict {
1900 // BOTH caps are consulted here, through the one combined gate:
1901 // the pre-existing per-(cluster, source-IP) cap and the
1902 // per-(cluster, source-SUBNET) cap that stands beside it. A
1903 // connection is admitted only when both allow it, which is what
1904 // makes "10 per IP AND 100 per /64" expressible. The subnet cap is
1905 // `0` (disabled) by default, in which case this is byte-for-byte
1906 // the previous per-IP consult.
1907 let at_limit = sessions.borrow().cluster_connection_at_limit(
1908 frontend_token,
1909 cluster_id,
1910 &resume.ip(),
1911 resume.max_connections_per_ip(),
1912 resume.max_connections_per_subnet(),
1913 );
1914 if at_limit {
1915 let retry_after = sessions
1916 .borrow()
1917 .effective_retry_after(resume.cluster_retry_after());
1918 return router::IpGateVerdict::AtLimit { retry_after };
1919 }
1920 // Idempotent track — H2 streams to the same `(cluster, ip)` share a
1921 // single slot in the per-token set. The decrement happens wholesale on
1922 // session close, via `untrack_all_cluster_ip`.
1923 sessions.borrow_mut().track_cluster_connection(
1924 frontend_token,
1925 cluster_id.clone(),
1926 resume.ip(),
1927 resume.max_connections_per_subnet(),
1928 );
1929 router::IpGateVerdict::Admitted
1930}
1931
1932impl<Front: SocketHandler, L: ListenerHandler + L7ListenerHandler> Mux<Front, L> {
1933 pub fn front_socket(&self) -> &TcpStream {
1934 self.frontend.socket()
1935 }
1936
1937 /// Take the flag the last `ready` pass set when it yielded at
1938 /// `MAX_LOOP_ITERATIONS` with work left (`Context::loop_budget_yielded`).
1939 /// The session answers `ProxySession::yielded` with it.
1940 pub fn take_loop_budget_yield(&mut self) -> bool {
1941 std::mem::take(&mut self.context.loop_budget_yielded)
1942 }
1943
1944 /// Whether a backend connection carries a HUP or ERROR event that no
1945 /// socket readiness will deliver again: the flag `Mux::timeout_inner`
1946 /// sets on a dial whose connect timed out. The session answers
1947 /// `ProxySession::needs_ready_pass` with this, so `Server::timeout` runs
1948 /// the `ready` pass that counts the failure and fails the request over
1949 /// (sozu-proxy/sozu#1800).
1950 pub fn has_pending_backend_failure(&self) -> bool {
1951 self.router.backends.iter().any(|(_, backend)| {
1952 let event = backend.readiness().event;
1953 event.is_error() || event.is_hup()
1954 })
1955 }
1956
1957 /// Whether the client has closed its side of the frontend connection, the
1958 /// `peer_closed` argument the session's close path hands to
1959 /// `shutdown_write`. See [`Connection::peer_closed`].
1960 pub fn frontend_peer_closed(&self) -> bool {
1961 self.frontend.peer_closed()
1962 }
1963
1964 /// Perform every backend accounting change the core has decided since the
1965 /// last drain, in the order it decided them.
1966 ///
1967 /// The ONLY writer of `Backend::active_requests` and
1968 /// `Backend::active_connections` on the mux path. The drain leaves the
1969 /// queue empty, so running this twice in a row is idempotent — the second
1970 /// call finds nothing and performs nothing.
1971 ///
1972 /// Called from the four `Mux` wrappers that already owe the timer wheel a
1973 /// `reschedule` on every exit, and once more immediately before each
1974 /// `Router::plan_connect`, because that call's load balancer is the only
1975 /// reader of the counters this drains.
1976 pub(crate) fn apply_backend_deltas(&mut self) {
1977 self.backend_registry
1978 .apply_all(&mut self.context.backend_deltas, &self.backends);
1979 }
1980}
1981
1982impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHandler> Mux<Front, L> {
1983 fn sync_upgrade_buffers(&mut self) {
1984 for stream in &mut self.context.streams {
1985 stream
1986 .front
1987 .storage
1988 .buffer
1989 .sync(stream.front.storage.end, stream.front.storage.head);
1990 stream
1991 .back
1992 .storage
1993 .buffer
1994 .sync(stream.back.storage.end, stream.back.storage.head);
1995 }
1996 }
1997
1998 /// Consume the wheel entry that delivered a timeout and say whether it was
1999 /// really due.
2000 ///
2001 /// Three things happen here, in this order, and all three are load-bearing.
2002 ///
2003 /// 1. **Consume.** The wheel has already handed the entry over, so
2004 /// [`TimeoutContainer::triggered`] records that the handle holds
2005 /// nothing. This is the consume-then-reschedule step, the direct
2006 /// counterpart of `UdpManager::handle_timeout` clearing `armed_deadline`
2007 /// on entry (`lib/src/protocol/udp/manager.rs`). It matters because
2008 /// [`Mux::sync_timeout`] memoizes on "the handle already holds this
2009 /// deadline": without the clearing, a delivery that changes nothing is
2010 /// memoized away and the session is left with NO wheel entry at all — a
2011 /// lost wakeup.
2012 ///
2013 /// 2. **Re-validate.** `crate::timer` rounds a delay to the NEAREST tick
2014 /// (`timer.rs` `duration_to_tick`) and `Timer::poll` fires everything
2015 /// whose tick has come, so an entry armed for deadline `D` is delivered
2016 /// from `tick * round(D / tick) - tick/2` onwards. The earliness is
2017 /// `(D_ms + 50) mod 100` at the default 100 ms tick, i.e. up to **99 ms**
2018 /// — a full tick minus a millisecond, not half a tick; see
2019 /// `timer.rs::duration_to_tick` for the two halves of that bound. An
2020 /// early delivery is not an expiry: returning
2021 /// `false` here leaves the core's deadline untouched, and the
2022 /// `reschedule` on the way out of [`SessionState::timeout`] therefore
2023 /// puts the entry straight back at the SAME instant.
2024 ///
2025 /// 3. **Clear the core's deadline on a real expiry.** The core asked to be
2026 /// called back at `D` and has been; leaving `D` in place would have
2027 /// `reschedule` re-arm an already-elapsed instant, which the wheel
2028 /// re-delivers immediately — a busy loop. The timeout branches re-arm
2029 /// with `arm_timeout` on every path that keeps the session alive, which
2030 /// is what the old `triggered()` + `set(token)` pair did.
2031 ///
2032 /// A token with no core (an unknown token, or a backend already evicted)
2033 /// is reported due and handled by the caller's unknown-token branch.
2034 fn consume_timer_entry(&mut self, token: Token) -> bool {
2035 if token == self.frontend_token {
2036 self.timeouts.frontend.triggered();
2037 } else if let Some(container) = self.timeouts.backends.get_mut(&token) {
2038 container.triggered();
2039 }
2040 let core_deadline = if token == self.frontend_token {
2041 self.frontend.poll_timeout()
2042 } else {
2043 match self.router.backends.get(&token) {
2044 Some(backend) => backend.poll_timeout(),
2045 None => return true,
2046 }
2047 };
2048 if core_deadline.is_some_and(|deadline| self.context.now < deadline) {
2049 return false;
2050 }
2051 if token == self.frontend_token {
2052 self.frontend.clear_timeout();
2053 } else if let Some(backend) = self.router.backends.get_mut(&token) {
2054 backend.clear_timeout();
2055 }
2056 true
2057 }
2058
2059 /// Reflect every core's next deadline onto the timer wheel.
2060 ///
2061 /// This is the adapter half of the `poll_timeout()` split, and the direct
2062 /// analogue of `UdpManager::reschedule` (`lib/src/protocol/udp/manager.rs`).
2063 /// It runs at the end of every `SessionState` entry point that can change a
2064 /// deadline — `ready`, `timeout`, `shutting_down`, `cancel_timeouts` —
2065 /// through wrappers, so no early `return` inside those bodies can skip it.
2066 /// That structural placement is the point: a hand-placed re-arm at each
2067 /// exit is exactly what used to be forgotten.
2068 ///
2069 /// Departed tokens need no explicit cancel: `retain` drops their handle and
2070 /// `TimeoutContainer::drop` cancels the entry.
2071 fn reschedule(&mut self) {
2072 Self::sync_timeout(
2073 &mut self.timeouts.frontend,
2074 self.frontend_token,
2075 self.frontend.poll_timeout(),
2076 self.frontend.timeout_duration(),
2077 );
2078 for (token, backend) in &self.router.backends {
2079 let duration = backend.timeout_duration();
2080 Self::sync_timeout(
2081 self.timeouts
2082 .backends
2083 .get_or_insert_with(*token, || TimeoutContainer::new_empty(duration)),
2084 *token,
2085 backend.poll_timeout(),
2086 duration,
2087 );
2088 }
2089 let backends = &self.router.backends;
2090 self.timeouts
2091 .backends
2092 .retain(|token, _| backends.contains_key(token));
2093
2094 self.debug_assert_timer_coherence();
2095 }
2096
2097 /// Arm, re-arm or cancel one token's wheel entry so it matches `next`.
2098 ///
2099 /// The memoization — skip the wheel when nothing changed — is what makes
2100 /// this cheap enough to run on every pass, and it is also where the lost
2101 /// wakeup lives if it is written carelessly. It is guarded on BOTH halves:
2102 ///
2103 /// * `deadline() == next` — the instant has not moved; and
2104 /// * `is_armed() == next.is_some()` — the wheel actually still HOLDS it.
2105 ///
2106 /// The second is not redundant. A delivered entry is gone from the wheel,
2107 /// and `TimeoutContainer::triggered` clears the deadline alongside it, so
2108 /// the pair stays honest — that clearing is this design's
2109 /// consume-then-reschedule step, the same one `UdpManager::handle_timeout`
2110 /// performs by setting `armed_deadline = None` on entry. Memoize on the
2111 /// deadline alone, against a mirror that a firing does not clear, and every
2112 /// wheel delivery that changes nothing becomes a session with no timer.
2113 fn sync_timeout(
2114 container: &mut TimeoutContainer,
2115 token: Token,
2116 next: Option<Instant>,
2117 duration: Duration,
2118 ) {
2119 if container.duration() != duration {
2120 // Keep the handle's configured duration current without moving the
2121 // armed entry: the WebSocket upgrade hands these containers to
2122 // `Pipe`, which re-arms from `duration()`.
2123 container.retune(duration);
2124 }
2125 if container.deadline() == next && container.is_armed() == next.is_some() {
2126 return;
2127 }
2128 match next {
2129 Some(deadline) => container.set_at(token, deadline),
2130 None => {
2131 container.cancel();
2132 }
2133 }
2134 }
2135
2136 /// Timer coherence, carried over from `UdpManager::check_invariants` (6):
2137 /// for every live token the wheel handle must hold exactly what the core
2138 /// asked for — armed iff the core wants a timer, at the core's instant when
2139 /// it does — and no handle may outlive its connection.
2140 ///
2141 /// This is the invariant a missed `reschedule` breaks, and it is the reason
2142 /// the entry points are wrapped rather than trusted.
2143 fn debug_assert_timer_coherence(&self) {
2144 #[cfg(debug_assertions)]
2145 {
2146 let frontend_token = self.frontend_token;
2147 let frontend = std::iter::once((&frontend_token, &self.timeouts.frontend));
2148 for (token, container) in frontend.chain(self.timeouts.backends.iter()) {
2149 let core = if *token == frontend_token {
2150 self.frontend.poll_timeout()
2151 } else {
2152 match self.router.backends.get(token) {
2153 Some(backend) => backend.poll_timeout(),
2154 None => {
2155 panic!("timer handle for {token:?} outlived its connection")
2156 }
2157 }
2158 };
2159 debug_assert_eq!(
2160 container.is_armed(),
2161 core.is_some(),
2162 "timer coherence for {token:?}: armed={} but the core wants {core:?}",
2163 container.is_armed(),
2164 );
2165 debug_assert_eq!(
2166 container.deadline(),
2167 core,
2168 "timer coherence for {token:?}: the wheel holds {:?}, the core wants {core:?}",
2169 container.deadline(),
2170 );
2171 }
2172 }
2173 }
2174
2175 /// Whether a frontend HUP still leaves an exchange to serve.
2176 ///
2177 /// The frontend HUP is not always a hang-up: `Ready::from(&Event)`
2178 /// (`command/src/ready.rs`) also raises it for `is_read_closed()`, the
2179 /// `EPOLLRDHUP` a client's half-close (`shutdown(SHUT_WR)`, RFC 9293
2180 /// §3.6) produces. Such a client has stopped sending, not receiving, and
2181 /// still expects the response to what it sent:
2182 ///
2183 /// - input not read yet may hold a request, or the rest of one: the FIN
2184 /// can arrive with the request itself (`Connection::has_unread_input`);
2185 /// - a stream whose request was received whole is answered.
2186 ///
2187 /// Nothing else is: once the client's EOF is read, a request it left
2188 /// incomplete can never complete, and the session closes as on a hang-up
2189 /// instead of waiting for its backend.
2190 ///
2191 /// A full hang-up is never in flight: the client will read nothing more.
2192 /// It is ERROR (`EPOLLERR`, a reset) or WRITE_CLOSED (mio's
2193 /// `is_write_closed`: `EPOLLHUP` or `EPOLLERR`). ERROR alone is not
2194 /// enough: a reset whose error sozu's own `read` or `write` consumed
2195 /// first is reported as `EPOLLHUP` without `EPOLLERR`, which only
2196 /// WRITE_CLOSED tells from a half-close. A lingering frontend is the one
2197 /// exception: sozu shut its write side itself, so `EPOLLHUP` follows the
2198 /// client's FIN, and its drain still reads to that EOF.
2199 fn frontend_exchange_in_flight(&self) -> bool {
2200 let event = self.frontend.readiness().event;
2201 if event.is_error() || (event.is_write_closed() && !self.frontend.is_lingering()) {
2202 return false;
2203 }
2204 self.frontend.has_unread_input()
2205 || self
2206 .context
2207 .streams
2208 .iter()
2209 .any(|stream| stream.state.is_open() && stream.front.is_terminated())
2210 }
2211
2212 fn delay_close_for_frontend_flush(&mut self, reason: &'static str) -> bool {
2213 let _ = self.frontend.initiate_close_notify();
2214 // LIFECYCLE §9 invariant 16: consult per-stream back-buffers in
2215 // addition to the connection-level pending-write predicate so
2216 // shutdown does not close while any open H2 stream still has
2217 // kawa bytes queued after a voluntary scheduler yield.
2218 if self
2219 .frontend
2220 .has_pending_write_including_streams(&self.context)
2221 {
2222 let readiness = self.frontend.readiness_mut();
2223 readiness.interest = Ready::WRITABLE | Ready::HUP | Ready::ERROR;
2224 readiness.signal_pending_write();
2225 debug!(
2226 "{} Mux delaying close on {}: {:?}",
2227 log_context!(self),
2228 reason,
2229 self.frontend
2230 );
2231 true
2232 } else {
2233 false
2234 }
2235 }
2236
2237 /// The inner loop of `Mux::ready_inner` reached `MAX_LOOP_ITERATIONS`.
2238 /// Count and log it, then tell the loop whether to wait for the client
2239 /// (`true`) or close the session (`false`).
2240 ///
2241 /// Output still queued for the client, in the connection or in an open
2242 /// stream's response buffer, is not dropped while the client drains it.
2243 /// An H1 frontend's `has_pending_write_including_streams` does not see
2244 /// its stream's buffer, so the open streams are probed here
2245 /// (sozu-proxy/sozu#1819). The session then waits for the frontend's next
2246 /// writable event, with the frontend timeout as the bound if none comes.
2247 ///
2248 /// Waiting stays bounded: reaching the limit again with no byte written
2249 /// to the client since the last time (`Context::client_bytes_out`)
2250 /// closes the session, so a loop that spins on every wake-up without
2251 /// progress costs at most two budgets.
2252 fn wait_for_client_at_loop_limit(&mut self) -> bool {
2253 incr!(names::http::INFINITE_LOOP_ERROR);
2254 #[cfg(test)]
2255 LOOP_LIMIT_HITS.with(|hits| hits.set(hits.get() + 1));
2256 let bytes_out = self.context.client_bytes_out;
2257 let progressed = self.context.loop_limit_bytes_out != Some(bytes_out);
2258 let output_queued = progressed
2259 && (self
2260 .frontend
2261 .has_pending_write_including_streams(&self.context)
2262 || self.context.streams.iter().any(|stream| {
2263 stream.state.is_open()
2264 && (!stream.back.out.is_empty() || !stream.back.blocks.is_empty())
2265 }));
2266 warn!(
2267 "{} Mux loop reached {} iterations, {}: frontend={:?} backends={:?}",
2268 log_context!(self),
2269 MAX_LOOP_ITERATIONS,
2270 if output_queued {
2271 "waiting for the client to drain the queued output"
2272 } else if progressed {
2273 "closing"
2274 } else {
2275 "closing: no byte reached the client since the last time"
2276 },
2277 self.frontend.readiness(),
2278 self.router
2279 .backends
2280 .iter()
2281 .map(|(token, backend)| (*token, backend.readiness().clone()))
2282 .collect::<Vec<_>>()
2283 );
2284 if output_queued {
2285 self.context.loop_limit_bytes_out = Some(bytes_out);
2286 let readiness = self.frontend.readiness_mut();
2287 readiness.interest.insert(Ready::WRITABLE);
2288 readiness.event.remove(Ready::WRITABLE);
2289 self.frontend.arm_timeout(self.context.now);
2290 }
2291 output_queued
2292 }
2293
2294 /// Drive the frontend I/O path during shutdown, when the server is polling
2295 /// `shutting_down()` outside the normal epoll readiness loop.
2296 ///
2297 /// This is required for H2 graceful shutdown because a stream may still
2298 /// need one last readable pass to observe the peer's END_STREAM or one last
2299 /// writable pass to retire the stream, emit GOAWAY, or flush TLS records.
2300 ///
2301 /// An H2 frontend always gets that one writable pass. Further passes run
2302 /// only while the connection still has output queued AND the socket took
2303 /// the last write: the loop stops as soon as either is false. It must not
2304 /// wait for the WRITABLE *event* to clear, because the forced pass sets
2305 /// that bit itself (`signal_pending_write`) and `writable()` clears it only
2306 /// on `WouldBlock`. An idle draining session therefore never cleared it,
2307 /// and every `shut_down_sessions()` tick spun `MAX_LOOP_ITERATIONS` empty
2308 /// `writable()` calls per draining H2 session.
2309 fn drive_frontend_shutdown_io(&mut self) -> SessionIsToBeClosed {
2310 let force_h2_read = matches!(self.frontend, Connection::H2(_));
2311 let force_h2_write = matches!(self.frontend, Connection::H2(_));
2312 let readiness = self.frontend.readiness().clone();
2313 if !force_h2_read
2314 && !force_h2_write
2315 && readiness.event.is_empty()
2316 && !self.frontend.has_pending_write()
2317 {
2318 return false;
2319 }
2320
2321 if force_h2_read || self.frontend.readiness().event.is_readable() {
2322 self.frontend
2323 .readiness_mut()
2324 .interest
2325 .insert(Ready::READABLE);
2326 match self
2327 .frontend
2328 .readable(&mut self.context, EndpointClient(&mut self.router))
2329 {
2330 MuxResult::Continue => {}
2331 MuxResult::CloseSession | MuxResult::Upgrade => return true,
2332 }
2333 }
2334
2335 if !force_h2_write
2336 && !self.frontend.has_pending_write()
2337 && !self.frontend.readiness().event.is_writable()
2338 {
2339 return false;
2340 }
2341
2342 let mut iterations = 0;
2343 loop {
2344 self.frontend
2345 .readiness_mut()
2346 .interest
2347 .insert(Ready::WRITABLE);
2348 if force_h2_write {
2349 self.frontend.readiness_mut().signal_pending_write();
2350 }
2351 #[cfg(test)]
2352 SHUTDOWN_WRITABLE_PASSES.with(|passes| passes.set(passes.get() + 1));
2353 match self
2354 .frontend
2355 .writable(&mut self.context, EndpointClient(&mut self.router))
2356 {
2357 MuxResult::Continue => {}
2358 MuxResult::CloseSession | MuxResult::Upgrade => return true,
2359 }
2360
2361 iterations += 1;
2362 if iterations >= MAX_LOOP_ITERATIONS
2363 || !self.frontend.has_pending_write()
2364 || !self.frontend.readiness().event.is_writable()
2365 {
2366 break;
2367 }
2368 }
2369 false
2370 }
2371}
2372
2373impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHandler> SessionState
2374 for Mux<Front, L>
2375{
2376 /// Thin wrapper over `Mux::ready_inner` whose only job is to run
2377 /// `Mux::reschedule` on the way out. `ready_inner` has a dozen `return`
2378 /// sites; arming the wheel at each of them by hand is precisely the
2379 /// discipline this refactor exists to remove.
2380 fn ready(
2381 &mut self,
2382 session: Rc<RefCell<dyn ProxySession>>,
2383 proxy: Rc<RefCell<dyn L7Proxy>>,
2384 metrics: &mut SessionMetrics,
2385 ) -> SessionResult {
2386 let result = self.ready_inner(session, proxy, metrics);
2387 self.apply_backend_deltas();
2388 self.reschedule();
2389 result
2390 }
2391
2392 fn update_readiness(&mut self, token: Token, events: Ready) {
2393 trace!("{} EVENTS: {:?} on {:?}", log_context!(self), events, token);
2394 self.context.debug.push(DebugEvent::EV(token, events));
2395 if token == self.frontend_token {
2396 self.frontend.readiness_mut().event |= events;
2397 } else if let Some(c) = self.router.backends.get_mut(&token) {
2398 c.readiness_mut().event |= events;
2399 }
2400 }
2401
2402 /// Thin wrapper over `Mux::timeout_inner`: reschedule on the way out,
2403 /// then check the two properties carried over from `UdpManager`.
2404 fn timeout(&mut self, token: Token, metrics: &mut SessionMetrics) -> StateResult {
2405 let result = self.timeout_inner(token, metrics);
2406 self.apply_backend_deltas();
2407 self.reschedule();
2408
2409 // Strict-advance guard, carried over from `UdpManager::handle_timeout`
2410 // (`lib/src/protocol/udp/manager.rs`). After handling a firing at
2411 // `now`, the entry re-armed for the SAME token must be strictly in the
2412 // future. A deadline `<= now` makes the wheel re-deliver immediately
2413 // and the session spins — the canonical sans-io busy loop. The one
2414 // legitimate way to reach it is a configured timeout of zero, which is
2415 // an operator asking for exactly that.
2416 #[cfg(debug_assertions)]
2417 {
2418 let now = self.context.now;
2419 let (duration, container) = if token == self.frontend_token {
2420 (
2421 Some(self.frontend.timeout_duration()),
2422 Some(&self.timeouts.frontend),
2423 )
2424 } else {
2425 (
2426 self.router
2427 .backends
2428 .get(&token)
2429 .map(|b| b.timeout_duration()),
2430 self.timeouts.backends.get(&token),
2431 )
2432 };
2433 if let Some(container) = container
2434 && let Some(next) = container.deadline()
2435 && duration.is_some_and(|d| !d.is_zero())
2436 {
2437 debug_assert!(
2438 next > now,
2439 "Mux::timeout must strictly advance past a firing on {token:?}: \
2440 armed {next:?} <= fired_at {now:?} (busy-loop)"
2441 );
2442 }
2443 }
2444
2445 result
2446 }
2447
2448 fn cancel_timeouts(&mut self) {
2449 trace!("{} MuxState::cancel_timeouts", log_context!(self));
2450 self.frontend.clear_timeout();
2451 for backend in self.router.backends.values_mut() {
2452 backend.clear_timeout();
2453 }
2454 // Push the cleared deadlines onto the wheel in the same call: the
2455 // adapter's handles are the only thing holding real entries.
2456 self.reschedule();
2457 }
2458
2459 fn print_state(&self, _context: &str) {
2460 // The trait-required `context: &str` parameter (protocol tag like
2461 // "HTTPS"/"HTTP" passed by callers) predates the unified
2462 // `log_context!(self)` envelope. The canonical `MUX` tag lives
2463 // inside `log_context!`, so we ignore the parameter here and emit
2464 // the bracketed Session(...) block instead, mirroring the second
2465 // `error!` in this function.
2466 error!(
2467 "\
2468{} Session(Mux)
2469\tFrontend:
2470\t\ttoken: {:?}\treadiness: {:?}
2471\tBackend(s):",
2472 log_context!(self),
2473 self.frontend_token,
2474 self.frontend.readiness()
2475 );
2476 for (backend_token, backend) in &self.router.backends {
2477 error!(
2478 "{} \t\ttoken: {:?}\treadiness: {:?}",
2479 log_context!(self),
2480 backend_token,
2481 backend.readiness()
2482 )
2483 }
2484 }
2485
2486 fn close(&mut self, proxy: Rc<RefCell<dyn L7Proxy>>, _metrics: &mut SessionMetrics) {
2487 if self.context.debug.is_interesting() {
2488 warn!("{} {:?}", log_context!(self), self.context.debug.events);
2489 }
2490 debug!("{} MUX CLOSE", log_context!(self));
2491 trace!("{} FRONTEND: {:#?}", log_context!(self), self.frontend);
2492 trace!(
2493 "{} BACKENDS: {:#?}",
2494 log_context!(self),
2495 self.router.backends
2496 );
2497
2498 self.log_and_recycle_open_streams();
2499
2500 self.frontend
2501 .close(&mut self.context, EndpointClient(&mut self.router));
2502
2503 // Release every wheel handle: the session is going away and
2504 // `TimeoutContainer::drop` cancels each entry. This replaces the
2505 // per-backend `timeout_container().cancel()` the loop below used to do
2506 // and additionally covers the frontend, which it never did. The
2507 // frontend's handle is a field rather than a map entry, so it is
2508 // replaced by an unarmed one, which is what a later `reschedule`
2509 // would have inserted in its place.
2510 self.timeouts.frontend = TimeoutContainer::new_empty(self.timeouts.frontend.duration());
2511 self.timeouts.backends.clear();
2512
2513 self.close_backend_connections(&proxy);
2514
2515 // The session is going away, so this is the last chance to settle the
2516 // ledger against the worker's registry. Everything the frontend close
2517 // and the loop above decided is performed here.
2518 self.apply_backend_deltas();
2519 }
2520
2521 /// Thin wrapper over `Mux::shutting_down_inner`: it drives frontend I/O
2522 /// and can change a core deadline, so it owes the wheel a reschedule on
2523 /// every exit.
2524 fn shutting_down(&mut self) -> SessionIsToBeClosed {
2525 let result = self.shutting_down_inner();
2526 self.apply_backend_deltas();
2527 self.reschedule();
2528 result
2529 }
2530
2531 /// A mux session is targeted when one of its backend connections,
2532 /// connecting or connected, is: an H2 frontend multiplexing several
2533 /// clusters is targeted by each of them.
2534 fn targets(&self, target: &crate::SessionTarget) -> bool {
2535 self.router
2536 .backends
2537 .iter()
2538 .any(|(_, connection)| match connection.position() {
2539 Position::Client(cluster_id, backend, _) => {
2540 target.matches(cluster_id, Some(&backend.backend_id), Some(backend.address))
2541 }
2542 Position::Server => false,
2543 })
2544 }
2545}
2546
2547/// The bodies the `SessionState` wrappers above drive. They are inherent
2548/// methods rather than trait methods so the wrappers can own the one thing
2549/// every exit path needs: the [`Mux::reschedule`] that pushes the cores'
2550/// deadlines onto the timer wheel.
2551impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHandler> Mux<Front, L> {
2552 /// Emit the access log of every stream still open with a request start,
2553 /// with the connection overhead shared among them, and recycle it.
2554 /// Called by `Mux::close` at teardown and by
2555 /// [`Self::release_streams_and_backends_for_error_linger`].
2556 fn log_and_recycle_open_streams(&mut self) {
2557 // Log active streams at session teardown for timeout diagnosis
2558 let active_count = self
2559 .context
2560 .streams
2561 .iter()
2562 .filter(|s| s.state.is_open() && s.metrics.start.is_some())
2563 .count();
2564 if active_count > 0 {
2565 debug!(
2566 "{} Session close with {} active stream(s)",
2567 log_context!(self),
2568 active_count
2569 );
2570 for (idx, stream) in self
2571 .context
2572 .streams
2573 .iter()
2574 .enumerate()
2575 .filter(|(_, s)| s.state.is_open() && s.metrics.start.is_some())
2576 {
2577 let elapsed = stream.metrics.service_time();
2578 debug!(
2579 "{} active stream[{}]: state={:?} service_time={:?} method={:?} path={:?} status={:?}",
2580 log_context!(self),
2581 idx,
2582 stream.state,
2583 elapsed,
2584 stream.context.method,
2585 stream.context.path,
2586 stream.context.status,
2587 );
2588 }
2589 incr!(names::h2::CLOSE_WITH_ACTIVE_STREAMS);
2590 }
2591
2592 // Distribute H2 connection-level overhead (control frames) across in-flight
2593 // streams so that access log bytes_in/bytes_out reflect actual wire cost.
2594 // Integer division may lose up to (active_count - 1) bytes, which is acceptable.
2595 let active_count = active_count.max(1);
2596 let (total_overhead_in, total_overhead_out) = self.frontend.overhead_bytes();
2597 let share_in = total_overhead_in / active_count;
2598 let share_out = total_overhead_out / active_count;
2599
2600 // Generate access logs for in-flight streams on session teardown.
2601 // Skip streams that already had their access log emitted (metrics.start is
2602 // set to None by metrics.reset() after generate_access_log in the happy path).
2603 // Each connection's RTT is its one memoized sample (`memoized_rtt`):
2604 // asked for only for a stream that still owes a log, so the ordinary
2605 // close, where every stream has already been logged, reads nothing
2606 // (issue #1590), and a connection that logged before reads nothing
2607 // either.
2608 for stream in &mut self.context.streams {
2609 if stream.state.is_open() && stream.metrics.start.is_some() {
2610 let client_rtt = self.frontend.rtt();
2611 stream.metrics.bin += share_in;
2612 stream.metrics.bout += share_out;
2613 stream.metrics.service_stop();
2614 if stream.metrics.backend_stop.is_none() {
2615 stream.metrics.backend_stop();
2616 }
2617 // Only mark as error if the stream had an actual protocol/processing failure
2618 // (kawa parse error, backend error). Normal timeouts, client disconnects,
2619 // and graceful connection closures are not errors.
2620 let is_error = stream.front.is_error() || stream.back.is_error();
2621 let server_rtt = stream
2622 .linked_token()
2623 .and_then(|token| self.router.backends.get(&token).and_then(Connection::rtt));
2624 for event in stream
2625 .generate_access_log(
2626 is_error,
2627 Some("session close"),
2628 self.context.listener.clone(),
2629 client_rtt,
2630 server_rtt,
2631 )
2632 .into_iter()
2633 .flatten()
2634 {
2635 crate::protocol::mux::h2::record_metric(event);
2636 }
2637 stream.state = StreamState::Recycle;
2638 }
2639 }
2640 }
2641
2642 /// Shut down, deregister from the session slab and account for every
2643 /// backend connection (`StreamsEnded`, `ConnectionClosed`, the backend
2644 /// gauges), then clear the backend-stream reverse index. The connections
2645 /// stay in `router.backends`: at teardown they drop with the `Mux`.
2646 fn close_backend_connections(&mut self, proxy: &Rc<RefCell<dyn L7Proxy>>) {
2647 for (token, client) in &mut self.router.backends {
2648 let proxy_borrow = proxy.borrow();
2649 let peer_closed = client.peer_closed();
2650 let socket = client.socket_mut();
2651 // No `EPOLL_CTL_DEL`: the socket stays in `router.backends` until
2652 // this `Mux` drops with its session, before the event loop's next
2653 // `epoll_wait`, and its last close removes it from the epoll set —
2654 // the same invariant as the front socket in `HttpsSession::close`
2655 // (`lib/src/https.rs`).
2656 // invariant: write-only shutdown — Shutdown::Both on a TLS frontend
2657 // discards the receive buffer and elicits TCP RST, truncating the
2658 // already-queued response. Canonical write-up: `HttpsSession::close`
2659 // (`lib/src/https.rs`). Backend sockets follow the same discipline for symmetry.
2660 // Skipped once the backend has closed: see `shutdown_write`.
2661 if let Err(e) = shutdown_write(socket, peer_closed)
2662 && e.kind() != ErrorKind::NotConnected
2663 {
2664 error!(
2665 "{} error shutting down back socket({:?}): {:?}",
2666 log_context_lite!(self),
2667 socket,
2668 e
2669 );
2670 }
2671 if !proxy_borrow.remove_session(*token) {
2672 error!(
2673 "{} session {:?} was already removed!",
2674 log_context_lite!(self),
2675 token
2676 );
2677 }
2678
2679 match client.position() {
2680 Position::Client(cluster_id, backend, _) => {
2681 // Session teardown iterates the backends map directly
2682 // instead of routing through `Connection::close`, so it
2683 // emits the same two accounting changes that path emits.
2684 // Going through the ledger rather than mutating here is
2685 // what lets the invariant-14 balance be read off one
2686 // stream of deltas — see `Mux::apply_backend_deltas`.
2687 let count = self
2688 .context
2689 .backend_streams
2690 .get(token)
2691 .map_or(0, |ids| ids.len());
2692 let (slot, backend_id, metric_incarnation) = (
2693 backend.slot(),
2694 backend.backend_id.clone(),
2695 backend.metric_incarnation(),
2696 );
2697 self.context.backend_deltas.push(BackendDelta {
2698 slot,
2699 change: BackendChange::StreamsEnded(count),
2700 });
2701 self.context.backend_deltas.push(BackendDelta {
2702 slot,
2703 change: BackendChange::ConnectionClosed,
2704 });
2705 gauge_add!(names::backend::CONNECTIONS, -1);
2706 // Second `-1` site for `backend.pool.size` (the first is
2707 // in `connection.rs::pre_close_client_bookkeeping`). This
2708 // path runs during session teardown when the frontend
2709 // session iterates the backends map directly without
2710 // routing through `Connection::close`. Both `-1` sites
2711 // mirror the single `+1` in router.rs::connect and the
2712 // matching `backend.connections, -1` calls already
2713 // present here, so symmetry follows from
2714 // `backend.connections` correctness.
2715 gauge_add!(names::backend::POOL_SIZE, -1);
2716 gauge_add!(
2717 names::backend::CONNECTIONS_PER_BACKEND,
2718 -1,
2719 Some(cluster_id),
2720 Some(&backend_id),
2721 metric_incarnation
2722 );
2723 trace!(
2724 "{} connection (session) closed: {:?}",
2725 log_context_lite!(self),
2726 backend
2727 );
2728 }
2729 Position::Server => {
2730 error!(
2731 "{} close_backend called on Server position",
2732 log_context_lite!(self)
2733 );
2734 }
2735 }
2736 }
2737 // Clear the reverse index after all backends have charged their
2738 // `StreamsEnded` deltas (whose counts come from that index).
2739 self.context.backend_streams.clear();
2740 }
2741
2742 /// Release the streams and backends of an H2 frontend that lingers after
2743 /// a final GOAWAY carrying an error code, in the pass that decides the
2744 /// linger rather than at its end, the order
2745 /// `ngx_http_v2_finalize_connection` follows. A linger still pending on
2746 /// its TLS `close_notify` counts as decided
2747 /// (`Connection::lingers_after_error_goaway`): the pass that queues the
2748 /// alert can end with the kernel refusing it and nothing re-armed, and
2749 /// then reach the `pending_links` pass with its streams still open.
2750 ///
2751 /// Such a GOAWAY can leave streams open (a flood, a protocol or flow
2752 /// control error, a SETTINGS timeout), unlike a final GOAWAY(NO_ERROR),
2753 /// which waits for the stream table to empty. None of them will be
2754 /// answered, so their access logs are emitted now with the same outcome
2755 /// the immediate close gave them before (`session close`), the slots
2756 /// recycled, and the backend connections shut down, accounted for and
2757 /// dropped, closing their sockets. With every stream recycled, nothing
2758 /// can be re-linked: the `pending_links` pass of `Mux::ready_inner`
2759 /// skips any stream not in `StreamState::Link`, and no backend is left
2760 /// to report a failure that would queue one. The recycled slots keep
2761 /// their buffers until the `Mux` drops, at the latest when the linger's
2762 /// short deadline (`shared::ERROR_GOAWAY_LINGER_TIMEOUT`) closes it.
2763 ///
2764 /// Idempotent: a second call finds nothing open and no backend. A
2765 /// graceful linger is left alone: its stream table is already empty.
2766 fn release_streams_and_backends_for_error_linger(&mut self, proxy: &Rc<RefCell<dyn L7Proxy>>) {
2767 if !self.frontend.lingers_after_error_goaway() {
2768 return;
2769 }
2770 // Pre: the `metrics.start` filter below cannot skip a stream that
2771 // could reach a backend. Every entry into `StreamState::Link` arms
2772 // the start first (`SessionMetrics::service_start`) or re-links a
2773 // stream that was already linked, and no `SessionMetrics::reset`
2774 // leaves a stream in either state.
2775 debug_assert!(
2776 !self.context.streams.iter().any(|s| {
2777 matches!(s.state, StreamState::Link | StreamState::Linked(_))
2778 && s.metrics.start.is_none()
2779 }),
2780 "a stream queued for or linked to a backend carries its request start"
2781 );
2782 let has_open_stream = self
2783 .context
2784 .streams
2785 .iter()
2786 .any(|s| s.state.is_open() && s.metrics.start.is_some());
2787 if !has_open_stream && self.router.backends.is_empty() {
2788 return;
2789 }
2790 debug!(
2791 "{} H2 error GOAWAY linger: releasing {} backend connection(s) and the open streams",
2792 log_context!(self),
2793 self.router.backends.len()
2794 );
2795 self.log_and_recycle_open_streams();
2796 self.close_backend_connections(proxy);
2797 let tokens: Vec<Token> = (&self.router.backends)
2798 .into_iter()
2799 .map(|(token, _)| *token)
2800 .collect();
2801 for token in tokens {
2802 // Dropping the connection closes its socket, which leaves the
2803 // epoll set with its last close (see `Mux::close`).
2804 drop(self.router.backends.remove(&token));
2805 }
2806 self.timeouts.backends.clear();
2807 debug_assert!(
2808 self.router.backends.is_empty() && self.context.backend_streams.is_empty(),
2809 "an error-GOAWAY linger holds no backend connection"
2810 );
2811 debug_assert!(
2812 !self
2813 .context
2814 .streams
2815 .iter()
2816 .any(|s| matches!(s.state, StreamState::Link | StreamState::Linked(_))),
2817 "an error-GOAWAY linger keeps no stream that could reach a backend"
2818 );
2819 }
2820
2821 /// Fulfil a [`router::ConnectPlan::Dial`]: select a backend, dial it,
2822 /// build the connection, register it, and hand it to
2823 /// `Router::commit_dialed`.
2824 ///
2825 /// This is the embedder half of Question 6's split
2826 /// ([#1340](https://github.com/sozu-proxy/sozu/issues/1340)). Every step
2827 /// here reaches something the core has no business holding — the
2828 /// `BackendMap`, a `connect(2)`, `setsockopt`, the session slab, the mio
2829 /// registry — and the order is the one `Router::plan_connect` used, unchanged.
2830 ///
2831 /// The one side-effect taken before the dial is the reservation selection
2832 /// raises on the backend (`active_connections += 1`,
2833 /// `BackendMap::reserve_backend`), and this function releases it on
2834 /// every exit that returns `Err`: a failed `connect(2)` through
2835 /// `BackendChange::DialFailed`, which also records the backend's
2836 /// failure, and every later exit — all of [`Self::attach_dialed`]'s —
2837 /// through one `BackendChange::ConnectionClosed`, which records none,
2838 /// because there the backend did not fail (#1713).
2839 ///
2840 /// Taken as free functions over `&mut` fields rather than `&mut self` so
2841 /// the caller can keep its `&mut self.context` binding across the call.
2842 #[allow(clippy::too_many_arguments)]
2843 fn dial_backend(
2844 router: &mut Router,
2845 backend_registry: &mut BackendRegistry,
2846 backends: &RefCell<BackendMap>,
2847 stream_id: GlobalStreamId,
2848 context: &mut Context<L>,
2849 session: &Rc<RefCell<dyn ProxySession>>,
2850 proxy: &Rc<RefCell<dyn L7Proxy>>,
2851 // Owned: moved into the new connection's `Position::Client`, a handle
2852 // on the stream's routed id rather than a copy of it (#1583).
2853 cluster_id: sozu_command::state::ClusterId,
2854 h2: bool,
2855 frontend_should_stick: bool,
2856 ) -> Result<(), BackendConnectionError> {
2857 // One `borrow_mut` of the worker's `BackendMap` for exactly the
2858 // selection and its reservation, and no longer: nothing else on this
2859 // path borrows the map, and the `RefMut` drops before the dial and
2860 // before `add_session` and `register_socket` borrow the proxy below.
2861 let mut backend = {
2862 let mut backends = backends.borrow_mut();
2863 let stream = &mut context.streams[stream_id];
2864 let mut selector = RegistrySelector {
2865 backends: &mut backends,
2866 registry: backend_registry,
2867 now: context.now,
2868 exclude: &stream.tried_backends,
2869 };
2870 router.backend_from_request(
2871 &cluster_id,
2872 frontend_should_stick,
2873 &mut stream.context,
2874 &mut selector,
2875 )?
2876 };
2877 backend.set_metric_incarnation(
2878 context.streams[stream_id]
2879 .context
2880 .cluster_metrics_incarnation(),
2881 );
2882
2883 // Selection reserved a connection on `backend`; the dial is ours
2884 // (#1684). A `connect(2)` that fails releases that reservation and
2885 // records the failure through the ledger, which the drain before the
2886 // next selection applies — the backend is then exactly where a failed
2887 // `Backend::try_connect` used to leave it.
2888 // A backend with a `unix_path` is dialed over that UNIX socket, and
2889 // TCP_NODELAY means nothing there. An id without a registry handle
2890 // (an embedder's own selector) names a TCP backend.
2891 let (dialed, nodelay) = match backend_registry.handle(&backend) {
2892 Some(handle) => {
2893 let handle = handle.borrow();
2894 (handle.connect_socket(), handle.unix_path.is_none())
2895 }
2896 None => (mio::net::TcpStream::connect(backend.address), true),
2897 };
2898 let socket = match dialed {
2899 Ok(socket) => socket,
2900 Err(io_error) => {
2901 context.backend_deltas.push(BackendDelta {
2902 slot: backend.slot(),
2903 change: BackendChange::DialFailed(context.now),
2904 });
2905 context.streams[stream_id]
2906 .tried_backends
2907 .push(backend.address);
2908 // The count the backend will hold once the delta is applied,
2909 // as `Backend::try_connect`'s error reported it.
2910 let failures = backend_registry
2911 .handle(&backend)
2912 .map_or(1, |handle| handle.borrow().failures + 1);
2913 return Err(BackendConnectionError::Backend(
2914 BackendError::ConnectionFailures {
2915 cluster_id: cluster_id.to_string(),
2916 backend_address: backend.address,
2917 failures,
2918 error: io_error.to_string(),
2919 },
2920 ));
2921 }
2922 };
2923
2924 // Past the connect, the reservation belongs to the connection built
2925 // on `socket`. An exit that abandons it drops the connection
2926 // unregistered, and `Connection` has no `Drop` that would release
2927 // anything, so the release is taken here, once, for every such exit:
2928 // `Connection::new_h2_client` finding no buffer, a refused
2929 // `Connection::start_stream`, and the `register_socket` rollback
2930 // (#1713). It is `ConnectionClosed`, not `DialFailed`: each of those
2931 // is a local failure, and charging it to the backend's failure count
2932 // and backoff would steer traffic away from a healthy backend.
2933 let slot = backend.slot();
2934 let attached = Self::attach_dialed(
2935 router, stream_id, context, session, proxy, cluster_id, h2, backend, socket, nodelay,
2936 );
2937 if attached.is_err() {
2938 context.backend_deltas.push(BackendDelta {
2939 slot,
2940 change: BackendChange::ConnectionClosed,
2941 });
2942 debug_assert!(
2943 !matches!(context.streams[stream_id].state, StreamState::Linked(_)),
2944 "an abandoned dial must not have linked its stream"
2945 );
2946 }
2947 attached
2948 }
2949
2950 /// Build, start and register the connection of a dial whose `connect(2)`
2951 /// succeeded, and hand it to `Router::commit_dialed`.
2952 ///
2953 /// SECURITY (CWE-400): every stateful side-effect (the
2954 /// `backend.connections` / `backend.pool.size` /
2955 /// `connections_per_backend` gauges, the slab session, the mio
2956 /// registration, the router-map entry, `stream.metrics.backend_start`) is
2957 /// deferred until AFTER both the `Connection` constructor and
2958 /// `start_stream` have succeeded. If either fails this returns `Err`
2959 /// without leaking a slab entry, an epoll registration, a gauge counter
2960 /// or a router-map entry. The `TcpStream` lives on the stack and is moved
2961 /// into the `Connection`; on failure the `Connection` — or the raw
2962 /// `TcpStream`, for the pool-exhaustion branch that drops inside
2963 /// `Connection::new_h2_client` — is dropped, closing the fd. No token is
2964 /// allocated before that point, so there is nothing to roll back. The
2965 /// backend's reservation is the caller's to release: every `Err` here
2966 /// drops the connection, and [`Self::dial_backend`] releases it once for
2967 /// all of them.
2968 #[allow(clippy::too_many_arguments)]
2969 fn attach_dialed(
2970 router: &mut Router,
2971 stream_id: GlobalStreamId,
2972 context: &mut Context<L>,
2973 session: &Rc<RefCell<dyn ProxySession>>,
2974 proxy: &Rc<RefCell<dyn L7Proxy>>,
2975 cluster_id: sozu_command::state::ClusterId,
2976 h2: bool,
2977 backend: BackendId,
2978 socket: mio::net::TcpStream,
2979 nodelay: bool,
2980 ) -> Result<(), BackendConnectionError> {
2981 if nodelay && let Err(e) = socket.set_nodelay(true) {
2982 error!(
2983 "{} error setting nodelay on back socket({:?}): {:?}",
2984 log_module_context!(context.http_context(stream_id)),
2985 socket,
2986 e
2987 );
2988 }
2989
2990 // Cache the backend's configured address so SOCKET log lines fired on
2991 // ECONNREFUSED (or any failed async `connect()`) can still render
2992 // `peer=<backend>` — `getpeername(2)` returns ENOTCONN in that state,
2993 // so the live lookup path would show `peer=None` exactly when the
2994 // operator needs the backend id.
2995 let backend_peer = Some(backend.address);
2996 let socket = SessionTcpStream::new(socket, context.session_ulid, backend_peer);
2997
2998 let flood_config = context.listener.borrow().get_h2_flood_config();
2999 let connection_config = context.listener.borrow().get_h2_connection_config();
3000 let stream_idle_timeout = context.listener.borrow().get_h2_stream_idle_timeout();
3001 let graceful_shutdown_deadline = context
3002 .listener
3003 .borrow()
3004 .get_h2_graceful_shutdown_deadline();
3005 let backend_id_for_gauge = Rc::clone(&backend.backend_id);
3006 let metric_incarnation = backend.metric_incarnation();
3007 let mut connection = if h2 {
3008 match Connection::new_h2_client(
3009 context.session_ulid,
3010 socket,
3011 cluster_id,
3012 backend,
3013 &mut *context.buffers,
3014 router.configured_connect_timeout,
3015 flood_config,
3016 connection_config,
3017 stream_idle_timeout,
3018 graceful_shutdown_deadline,
3019 ) {
3020 Some(connection) => connection,
3021 // Pool exhaustion: the socket was already dropped by
3022 // `new_h2_client` and no side-effect has been committed.
3023 None => return Err(BackendConnectionError::MaxBuffers),
3024 }
3025 } else {
3026 Connection::new_h1_client(
3027 context.session_ulid,
3028 socket,
3029 cluster_id,
3030 backend,
3031 router.configured_connect_timeout,
3032 )
3033 };
3034
3035 // Check the backend can accept a new stream BEFORE committing any
3036 // registry state. `start_stream` records its `StreamsStarted` delta
3037 // only once the start has succeeded, so a refusal leaves the backend
3038 // accounting untouched rather than needing a rollback.
3039 if !connection.start_stream(stream_id, context) {
3040 error!(
3041 "{} Backend rejected stream start (max concurrent streams reached)",
3042 log_module_context!(context.http_context(stream_id))
3043 );
3044 // `connection` (socket + pending timeout deadline) drops here; no
3045 // wheel entry was ever armed for it.
3046 return Err(BackendConnectionError::MaxSessionsMemory);
3047 }
3048
3049 // --- Happy path: commit side-effects in one atomic-ish block ---
3050 let stream = &mut context.streams[stream_id];
3051 stream.metrics.backend_start();
3052 stream.metrics.backend_id = stream.context.backend_id.clone();
3053 gauge_add!(names::backend::CONNECTIONS, 1);
3054 // `backend.pool.size` mirrors `backend.connections` exactly: one entry
3055 // per `Router::backends` token. The `-1` partners live in
3056 // `connection.rs::pre_close_client_bookkeeping` (graceful close) and
3057 // in `Mux::close` (session teardown). Symmetric pairing with both
3058 // decrement sites is the only defence against the gauge underflow
3059 // class of bug fixed by ff401b54 / aadb3fa4.
3060 gauge_add!(names::backend::POOL_SIZE, 1);
3061 // The cluster id moved into the connection; the stream's own copy,
3062 // stored by `Router::plan_connect`, names the same cluster.
3063 gauge_add!(
3064 names::backend::CONNECTIONS_PER_BACKEND,
3065 1,
3066 stream.context.cluster_id.as_deref(),
3067 Some(&backend_id_for_gauge),
3068 metric_incarnation
3069 );
3070
3071 let token = proxy.borrow().add_session(session.clone());
3072
3073 {
3074 let socket_ref = connection.socket_mut();
3075 if let Err(e) = proxy.borrow().register_socket(
3076 socket_ref,
3077 token,
3078 Interest::READABLE | Interest::WRITABLE,
3079 ) {
3080 // SECURITY (CWE-400): treat mio registration failure as a hard
3081 // connect failure. Without this rollback the gauges
3082 // (`backend.connections`, `backend.pool.size`,
3083 // `connections_per_backend`) and the slab session leak until
3084 // the connect timeout fires. Under fd pressure
3085 // (EMFILE/ENFILE) this can occur in tight bursts and poison
3086 // capacity dashboards.
3087 error!(
3088 "{} error registering back socket: {:?} — rolling back",
3089 log_module_context!(context.http_context(stream_id)),
3090 e
3091 );
3092 // Undo the gauge increments committed above.
3093 gauge_add!(names::backend::CONNECTIONS, -1);
3094 gauge_add!(names::backend::POOL_SIZE, -1);
3095 gauge_add!(
3096 names::backend::CONNECTIONS_PER_BACKEND,
3097 -1,
3098 context.http_context(stream_id).cluster_id.as_deref(),
3099 Some(&backend_id_for_gauge),
3100 metric_incarnation
3101 );
3102 // Release the `active_requests` charge `start_stream` took.
3103 //
3104 // The comment this replaces claimed the drop released it "via
3105 // the regular session drop path
3106 // (`pre_close_client_bookkeeping`)". That was never true:
3107 // `Connection` has no `impl Drop`, so dropping it runs no
3108 // bookkeeping at all. It was harmless only because the charge
3109 // is guarded on `BackendStatus::Connected` and a
3110 // freshly-dialled connection is `Connecting`, so there was
3111 // nothing to release. Emitting the release explicitly, under
3112 // the same guard the charge uses, makes the pair hold however
3113 // the status is reached rather than by accident.
3114 connection.release_start_stream_charge(context);
3115 proxy.borrow().remove_session(token);
3116 return Err(BackendConnectionError::MaxSessionsMemory);
3117 }
3118 }
3119
3120 router.commit_dialed(stream_id, context, token, connection);
3121 Ok(())
3122 }
3123
3124 fn ready_inner(
3125 &mut self,
3126 session: Rc<RefCell<dyn ProxySession>>,
3127 proxy: Rc<RefCell<dyn L7Proxy>>,
3128 _metrics: &mut SessionMetrics,
3129 ) -> SessionResult {
3130 let mut counter = 0;
3131 // How far this call forwarded stream bytes, read at the loop budget:
3132 // a session that reaches `MAX_LOOP_ITERATIONS` while progressing
3133 // yields to the event loop instead of closing (sozu-proxy/sozu#1906).
3134 let bytes_out_at_entry = self.context.stream_bytes_out;
3135 self.context.loop_budget_yielded = false;
3136
3137 // A linger decided outside this function (a timeout, a soft stop)
3138 // releases at the next pass; see the call after the frontend write.
3139 self.release_streams_and_backends_for_error_linger(&proxy);
3140
3141 if self.frontend.is_lingering() && self.frontend.readiness().event.is_hup() {
3142 // A lingering frontend reads to the client's EOF before it closes:
3143 // a close with its last bytes unread would still reset.
3144 self.frontend.readiness_mut().event.insert(Ready::READABLE);
3145 } else if self.frontend.readiness().event.is_hup() && self.frontend_exchange_in_flight() {
3146 // A half-closed client still gets the response to what it sent:
3147 // neither close nor queue `close_notify` while it is in flight.
3148 // What is already pending is flushed as a delayed close would.
3149 if self
3150 .frontend
3151 .has_pending_write_including_streams(&self.context)
3152 {
3153 self.frontend.readiness_mut().arm_writable();
3154 }
3155 } else if self.frontend.readiness().event.is_hup()
3156 && !self.delay_close_for_frontend_flush("frontend HUP")
3157 {
3158 debug!(
3159 "{} Mux closing on frontend HUP: {:?}",
3160 log_context!(self),
3161 self.frontend
3162 );
3163 return SessionResult::Close;
3164 }
3165
3166 // Start service timers on all active streams after the HUP check.
3167 // This mirrors session-level service_start/service_stop in Http(s)Session::ready()
3168 // to measure only CPU processing time, excluding epoll wait between cycles.
3169 for stream in &mut self.context.streams {
3170 if stream.state.is_open() {
3171 stream.metrics.service_start();
3172 }
3173 }
3174
3175 let start = Instant::now();
3176 self.context.debug.push(DebugEvent::ReadyTimestamp(
3177 std::time::SystemTime::now()
3178 .duration_since(std::time::UNIX_EPOCH)
3179 .unwrap_or_default()
3180 .as_millis() as usize,
3181 ));
3182 trace!("{} {:?}", log_context!(self), start);
3183 // Set once the inner loop has spent its budget and waits for the
3184 // client. The outer loop then re-enters the inner one at most once
3185 // more, for a stream this sweep just linked (an attached pooled
3186 // backend is armed by a synthetic event no epoll edge repeats), and
3187 // that iteration stops at the spent budget without counting or
3188 // logging the same limit again.
3189 let mut budget_spent = false;
3190 let mut budget_extra_pass = false;
3191 loop {
3192 // The mux's clock sample for this pass. Everything time-based
3193 // below — flood and back-pressure windows, the SETTINGS-ACK
3194 // deadline, per-stream liveness and flow-control-stall deadlines
3195 // — reads `context.now` (mirrored into `ConnectionH2::now` at
3196 // each entry point) rather than calling `Instant::now()` itself.
3197 //
3198 // Sampled per OUTER iteration so the inner loop DELIBERATELY
3199 // shares one instant: that is the property the snapshot exists
3200 // for — every frame processed in one inner sweep is weighed
3201 // against the same "now", so a burst cannot decay a rate window
3202 // out from under itself mid-sweep. It is not a bound on how long
3203 // the sweep may take: `MAX_LOOP_ITERATIONS` is a count, and a
3204 // count bounds iterations, not wall clock. `counter` is declared
3205 // above BOTH loops, so that budget is shared across every outer
3206 // iteration of one `ready()` call rather than reset per pass.
3207 // `start` above stays a separate sample so the trace reports the
3208 // true entry instant.
3209 self.context.now = Instant::now();
3210 self.context.now_wall_ms = unix_epoch_ms();
3211 self.context.debug.push(DebugEvent::LoopStart);
3212 loop {
3213 self.context.debug.push(DebugEvent::LoopIteration(counter));
3214 if self.frontend.readiness().filter_interest().is_readable() {
3215 let res = {
3216 let context = &mut self.context;
3217 let res = self
3218 .frontend
3219 .readable(context, EndpointClient(&mut self.router));
3220 context.debug.push(DebugEvent::SR(
3221 self.frontend_token,
3222 res,
3223 self.frontend.readiness().clone(),
3224 ));
3225 res
3226 };
3227 match res {
3228 MuxResult::Continue => {}
3229 MuxResult::CloseSession => {
3230 if !self.delay_close_for_frontend_flush("frontend readable") {
3231 debug!(
3232 "{} Mux close from frontend readable: {:?}",
3233 log_context!(self),
3234 self.frontend
3235 );
3236 return SessionResult::Close;
3237 }
3238 }
3239 MuxResult::Upgrade => {
3240 self.sync_upgrade_buffers();
3241 return SessionResult::Upgrade;
3242 }
3243 }
3244 }
3245
3246 let mut all_backends_readiness_are_empty = true;
3247 self.router.dead_backends.clear();
3248 let mut backend_close: Option<(&'static str, Token)> = None;
3249 for (token, client) in self.router.backends.iter_mut() {
3250 // A WRITABLE on a connecting socket is not proof it
3251 // connected: a dial that failed can surface as WRITABLE
3252 // alone, its error in `SO_ERROR`, and sozu raises
3253 // WRITABLE itself to flush queued output, as an HTTP/2
3254 // backend's preface, while the dial is still in flight.
3255 // Read the outcome before the WRITABLE arm below would
3256 // mark the connection established: a failure takes the
3257 // same connect-failure branch as a refused dial, a dial
3258 // still in flight waits for the kernel's completion edge
3259 // (sozu-proxy/sozu#1800).
3260 if client.readiness().event.is_writable()
3261 && matches!(
3262 client.position(),
3263 Position::Client(_, _, BackendStatus::Connecting(_))
3264 )
3265 {
3266 match connect_outcome(client.socket()) {
3267 ConnectOutcome::Connected => {}
3268 ConnectOutcome::Pending => {
3269 client.readiness_mut().event.remove(Ready::WRITABLE);
3270 }
3271 ConnectOutcome::Failed => {
3272 client.readiness_mut().event.insert(Ready::ERROR);
3273 }
3274 }
3275 }
3276 let readiness = client.readiness_mut();
3277 // Check the raw event for HUP/ERROR, not filter_interest():
3278 // the event is what the poller reported, whatever the
3279 // interest holds.
3280 let dead = readiness.event.is_hup() || readiness.event.is_error();
3281 if dead {
3282 trace!(
3283 "{} Backend({:?}) -> {:?}",
3284 log_context_lite!(self),
3285 token,
3286 readiness
3287 );
3288 readiness.event.remove(Ready::WRITABLE);
3289 }
3290
3291 if client.readiness().filter_interest().is_writable() {
3292 let position = client.position_mut();
3293 match position {
3294 Position::Client(
3295 cluster_id,
3296 backend,
3297 BackendStatus::Connecting(start),
3298 ) => {
3299 #[cfg(debug_assertions)]
3300 self.context
3301 .debug
3302 .push(DebugEvent::CCS(*token, cluster_id.clone()));
3303
3304 // Health and latency are not invariant 14 and
3305 // the load balancer READS `retry_policy`, so
3306 // these three writes stay immediate, against
3307 // the handle the slot resolves to. Only the
3308 // `active_requests` charge below is a delta.
3309 let Some(handle) = self.backend_registry.get(backend.slot()) else {
3310 error!(
3311 "{} connected backend names a slot this session never minted: {:?}",
3312 log_context_lite!(self),
3313 backend
3314 );
3315 continue;
3316 };
3317 let mut backend_borrow = handle.borrow_mut();
3318 if backend_borrow.retry_policy.is_down() {
3319 info!(
3320 "{} backend server {} at {} is up",
3321 log_context_lite!(self),
3322 backend.backend_id,
3323 backend.address
3324 );
3325 incr!(
3326 names::backend::UP,
3327 Some(cluster_id),
3328 Some(&backend.backend_id),
3329 backend.metric_incarnation()
3330 );
3331 gauge!(
3332 names::backend::AVAILABLE,
3333 1,
3334 Some(cluster_id),
3335 Some(&backend.backend_id),
3336 backend.metric_incarnation()
3337 );
3338 push_event(Event {
3339 kind: EventKind::BackendUp as i32,
3340 backend_id: Some(backend.backend_id.to_string()),
3341 address: Some(backend.address.into()),
3342 cluster_id: Some(cluster_id.to_string()),
3343 metric_detail: None,
3344 });
3345 }
3346
3347 //successful connection, reset failure counter
3348 backend_borrow.failures = 0;
3349 backend_borrow
3350 .set_connection_time(start.elapsed(), self.context.now);
3351 backend_borrow.retry_policy.succeed(self.context.now);
3352 drop(backend_borrow);
3353
3354 // These streams linked while the connection
3355 // was still `Connecting`, so
3356 // `Connection::start_stream`'s `Connected`
3357 // guard skipped their charge. This is where
3358 // every first stream of a fresh dial is
3359 // charged — which is why the ledger cannot be
3360 // core-only and still balance.
3361 let mut started = 0;
3362 if let Some(ids) = self.context.backend_streams.get(token) {
3363 started = ids.len();
3364 for &stream_id in ids.iter() {
3365 self.context.streams[stream_id].metrics.backend_connected();
3366 }
3367 }
3368 if started > 0 {
3369 let slot = backend.slot();
3370 self.context.backend_deltas.push(BackendDelta {
3371 slot,
3372 change: BackendChange::StreamsStarted(started),
3373 });
3374 }
3375 trace!(
3376 "{} connection success: {:?}",
3377 log_context_lite!(self),
3378 backend
3379 );
3380 *position = Position::Client(
3381 std::mem::take(cluster_id),
3382 backend.clone(),
3383 BackendStatus::Connected,
3384 );
3385 client.set_timeout_duration(
3386 self.router.configured_backend_timeout,
3387 self.context.now,
3388 );
3389 }
3390 Position::Client(..) => {}
3391 Position::Server => {
3392 error!(
3393 "{} backend connection cannot be in Server position",
3394 log_context_lite!(self)
3395 );
3396 }
3397 }
3398 let res = {
3399 let context = &mut self.context;
3400 let res = client.writable(context, EndpointServer(&mut self.frontend));
3401 context.debug.push(DebugEvent::CW(
3402 *token,
3403 res,
3404 client.readiness().clone(),
3405 ));
3406 res
3407 };
3408 match res {
3409 MuxResult::Continue => {}
3410 MuxResult::Upgrade => {
3411 error!(
3412 "{} only frontend connections can trigger Upgrade",
3413 log_context_lite!(self)
3414 );
3415 }
3416 MuxResult::CloseSession => {
3417 backend_close = Some(("backend writable", *token));
3418 break;
3419 }
3420 }
3421 // Cross-readiness: backend wrote → wake frontend reader
3422 let context = &mut self.context;
3423 self.frontend.try_resume_reading(context);
3424 }
3425
3426 if client.readiness().filter_interest().is_readable() {
3427 let res = {
3428 let context = &mut self.context;
3429 let res = client.readable(context, EndpointServer(&mut self.frontend));
3430 context.debug.push(DebugEvent::CR(
3431 *token,
3432 res,
3433 client.readiness().clone(),
3434 ));
3435 res
3436 };
3437 match res {
3438 MuxResult::Continue => {}
3439 MuxResult::Upgrade => {
3440 error!(
3441 "{} only frontend connections can trigger Upgrade (readable)",
3442 log_context_lite!(self)
3443 );
3444 }
3445 MuxResult::CloseSession => {
3446 backend_close = Some(("backend readable", *token));
3447 break;
3448 }
3449 }
3450 }
3451
3452 if dead
3453 && !client.readiness().filter_interest().is_readable()
3454 && !client.has_buffer_pressure(&self.context)
3455 {
3456 self.context
3457 .debug
3458 .push(DebugEvent::CH(*token, client.readiness().clone()));
3459 trace!("{} Closing {:#?}", log_context_lite!(self), client);
3460 match client.position() {
3461 Position::Client(cluster_id, backend, BackendStatus::Connecting(_)) => {
3462 // A dial that never completed charged nothing
3463 // to `active_requests`, so this arm touches
3464 // only the health counters — not the ledger.
3465 let Some(handle) = self.backend_registry.get(backend.slot()) else {
3466 error!(
3467 "{} failed backend names a slot this session never minted: {:?}",
3468 log_context_lite!(self),
3469 backend
3470 );
3471 continue;
3472 };
3473 let mut backend_borrow = handle.borrow_mut();
3474 backend_borrow.failures += 1;
3475
3476 let already_unavailable = backend_borrow.retry_policy.is_down();
3477 // The jitter comes from the worker's backend
3478 // map, the one generator selection draws
3479 // from; nothing else borrows the map here.
3480 backend_borrow
3481 .retry_policy
3482 .fail(self.context.now, self.backends.borrow_mut().rng());
3483 incr!(
3484 names::backend::CONNECTIONS_ERROR,
3485 Some(cluster_id),
3486 Some(&backend.backend_id),
3487 backend.metric_incarnation()
3488 );
3489 if !already_unavailable && backend_borrow.retry_policy.is_down() {
3490 error!(
3491 "{} backend server {} at {} is down",
3492 log_context_lite!(self),
3493 backend.backend_id,
3494 backend.address
3495 );
3496 incr!(
3497 names::backend::DOWN,
3498 Some(cluster_id),
3499 Some(&backend.backend_id),
3500 backend.metric_incarnation()
3501 );
3502 gauge!(
3503 names::backend::AVAILABLE,
3504 0,
3505 Some(cluster_id),
3506 Some(&backend.backend_id),
3507 backend.metric_incarnation()
3508 );
3509 push_event(Event {
3510 kind: EventKind::BackendDown as i32,
3511 backend_id: Some(backend.backend_id.to_string()),
3512 address: Some(backend.address.into()),
3513 cluster_id: Some(cluster_id.to_string()),
3514 metric_detail: None,
3515 });
3516 }
3517 drop(backend_borrow);
3518 // Every stream this dial carried failed to
3519 // reach `backend`: a retry of any of them
3520 // prefers another one (sozu-proxy/sozu#1800).
3521 if let Some(ids) = self.context.backend_streams.get(token) {
3522 for &stream_id in ids.iter() {
3523 self.context.streams[stream_id]
3524 .tried_backends
3525 .push(backend.address);
3526 }
3527 }
3528 trace!(
3529 "{} connection fail: {:?}",
3530 log_context_lite!(self),
3531 backend
3532 );
3533 }
3534 Position::Client(_, backend, _) => {
3535 // A backend that dies past `Connecting` still
3536 // owes the charge every stream it carries
3537 // took — the same `StreamsEnded(count)` the
3538 // teardown path emits.
3539 let count = self
3540 .context
3541 .backend_streams
3542 .get(token)
3543 .map_or(0, |ids| ids.len());
3544 self.context.backend_deltas.push(BackendDelta {
3545 slot: backend.slot(),
3546 change: BackendChange::StreamsEnded(count),
3547 });
3548 }
3549 Position::Server => {
3550 error!(
3551 "{} dead backend cannot be in Server position",
3552 log_context_lite!(self)
3553 );
3554 }
3555 }
3556 client.close(&mut self.context, EndpointServer(&mut self.frontend));
3557 self.router.dead_backends.push(*token);
3558 }
3559
3560 if client.has_loop_work(&self.context) {
3561 all_backends_readiness_are_empty = false;
3562 }
3563 }
3564 // Remove dead backends from the map BEFORE handling
3565 // backend_close. client.close() already decremented
3566 // connections_per_backend / backend.connections gauges in
3567 // the loop above; if we return SessionResult::Close before
3568 // removing them, Mux::close() would decrement again
3569 // (double-decrement → gauge underflow).
3570 if !self.router.dead_backends.is_empty() {
3571 for token in &self.router.dead_backends {
3572 let proxy_borrow = proxy.borrow();
3573 // The reverse-index entry `remove_backend_stream`
3574 // kept for its capacity leaves with its connection.
3575 // Only an emptied one: an entry still naming a stream
3576 // is the drift the debug check at the end of this
3577 // pass reports, and removing it here would hide it.
3578 if self
3579 .context
3580 .backend_streams
3581 .get(token)
3582 .is_some_and(|ids| ids.is_empty())
3583 {
3584 self.context.backend_streams.remove(token);
3585 }
3586 if let Some(mut client) = self.router.backends.remove(token) {
3587 // No explicit timer cancel: the token has left
3588 // `router.backends`, so `Mux::reschedule` drops its
3589 // handle on the way out of `ready` and
3590 // `TimeoutContainer::drop` cancels the entry.
3591 // No `EPOLL_CTL_DEL`: `client` drops at the end of
3592 // this block, and closing its socket removes it from
3593 // the epoll set — see `HttpsSession::close`
3594 // (`lib/src/https.rs`).
3595 let peer_closed = client.peer_closed();
3596 let socket = client.socket_mut();
3597 // invariant: write-only shutdown — Shutdown::Both on a TLS frontend
3598 // discards the receive buffer and elicits TCP RST, truncating the
3599 // already-queued response. Canonical write-up: `HttpsSession::close`
3600 // (`lib/src/https.rs`). Backend sockets follow the same discipline for symmetry.
3601 // Skipped once the backend has closed: see `shutdown_write`.
3602 if let Err(e) = shutdown_write(socket, peer_closed)
3603 && e.kind() != ErrorKind::NotConnected
3604 {
3605 error!(
3606 "{} error shutting down back socket({:?}): {:?}",
3607 log_context!(self),
3608 socket,
3609 e
3610 );
3611 }
3612 } else {
3613 error!("{} session {:?} has no backend!", log_context!(self), token);
3614 }
3615 if !proxy_borrow.remove_session(*token) {
3616 error!(
3617 "{} session {:?} was already removed!",
3618 log_context!(self),
3619 token
3620 );
3621 }
3622 }
3623 trace!("{} FRONTEND: {:#?}", log_context!(self), self.frontend);
3624 trace!(
3625 "{} BACKENDS: {:#?}",
3626 log_context!(self),
3627 self.router.backends
3628 );
3629 }
3630 if let Some((reason, token)) = backend_close {
3631 if !self.delay_close_for_frontend_flush(reason) {
3632 debug!(
3633 "{} Mux close from {} token={:?}: frontend={:?}",
3634 log_context!(self),
3635 reason,
3636 token,
3637 self.frontend
3638 );
3639 return SessionResult::Close;
3640 }
3641 all_backends_readiness_are_empty = false;
3642 }
3643
3644 if self.frontend.readiness().filter_interest().is_writable() {
3645 let res = {
3646 let context = &mut self.context;
3647 let res = self
3648 .frontend
3649 .writable(context, EndpointClient(&mut self.router));
3650 context.debug.push(DebugEvent::SW(
3651 self.frontend_token,
3652 res,
3653 self.frontend.readiness().clone(),
3654 ));
3655 res
3656 };
3657 match res {
3658 MuxResult::Continue => {}
3659 MuxResult::CloseSession => {
3660 if !self.delay_close_for_frontend_flush("frontend writable") {
3661 debug!(
3662 "{} Mux close from frontend writable: {:?}",
3663 log_context!(self),
3664 self.frontend
3665 );
3666 return SessionResult::Close;
3667 }
3668 }
3669 MuxResult::Upgrade => {
3670 self.sync_upgrade_buffers();
3671 return SessionResult::Upgrade;
3672 }
3673 }
3674 // The pass that decides an error-GOAWAY linger, pending
3675 // on `close_notify` or already draining, releases its
3676 // streams and backends before anything below can touch
3677 // them, `pending_links` included.
3678 self.release_streams_and_backends_for_error_linger(&proxy);
3679 // Cross-readiness: the frontend write can hand work back to
3680 // a backend — a parked one resumes once its peer drained
3681 // buffer space, and forwarding an interim 1xx re-arms the
3682 // backend read so the response bytes already buffered
3683 // behind it get parsed (`ConnectionH1::writable`'s interim
3684 // arms, `ConnectionH2::handle_1xx_reset`). The
3685 // `all_backends_readiness_are_empty` flag was recorded
3686 // before this write, so it is refreshed from every
3687 // backend's readiness now: a stale `true` would exit the
3688 // loop with a backend armed and no socket event left to
3689 // bring it back.
3690 let context = &mut self.context;
3691 for backend in self.router.backends.values_mut() {
3692 if backend.try_resume_reading(context) || backend.has_loop_work(context) {
3693 all_backends_readiness_are_empty = false;
3694 }
3695 }
3696 }
3697
3698 // A pooled connection whose backend left the configuration
3699 // is never reused (`Router::decide_after_gate`); once it
3700 // carries no stream it has nothing left to do, so drop it
3701 // rather than leave it idle until the session ends. The HUP
3702 // `force_disconnect` raises sends it through the dead-backend
3703 // sweep above on the next iteration, which releases its
3704 // connection on the backend and lets a `Closing` backend
3705 // reach `Closed`.
3706 for (token, backend) in self.router.backends.iter_mut() {
3707 let retired = backend
3708 .idle_pooled_backend()
3709 .is_some_and(|id| self.backend_registry.is_retired(id))
3710 && self
3711 .context
3712 .backend_streams
3713 .get(token)
3714 .is_none_or(|ids| ids.is_empty());
3715 if retired {
3716 debug!(
3717 "{} closing idle backend connection {:?}: its backend was removed",
3718 log_context_lite!(self),
3719 token
3720 );
3721 backend.force_disconnect();
3722 all_backends_readiness_are_empty = false;
3723 }
3724 }
3725
3726 // A frontend HUP is handled here as it is on entry: nothing
3727 // else in this loop acts on one. Once no output is left to
3728 // flush, this check closes the session, including after a
3729 // flush the entry check delayed the close for and the
3730 // writable arm above completed. A lingering frontend is left
3731 // to its drain: `ConnectionH1::start_linger` raises READABLE,
3732 // which `Mux::frontend_exchange_in_flight` counts as unread
3733 // input until the drain reads the client's EOF, so the
3734 // client's last bytes are never left unread to reset the
3735 // connection. While output is
3736 // pending, the HUP alone is not work this loop can progress:
3737 // it stops when the frontend has nothing to read or write,
3738 // and the next readiness event resumes the flush. Counting
3739 // the HUP as work spun the loop to `MAX_LOOP_ITERATIONS`
3740 // after every client hang-up. A HUP that comes from a
3741 // half-close leaves the exchange in flight to be served
3742 // (`Mux::frontend_exchange_in_flight`): the session closes
3743 // here once its response is complete.
3744 if self.frontend.readiness().event.is_hup()
3745 && !self
3746 .frontend
3747 .has_pending_write_including_streams(&self.context)
3748 && !self.frontend_exchange_in_flight()
3749 && !self.delay_close_for_frontend_flush("frontend HUP")
3750 {
3751 debug!(
3752 "{} Mux closing on frontend HUP: {:?}",
3753 log_context!(self),
3754 self.frontend
3755 );
3756 return SessionResult::Close;
3757 }
3758 let frontend_work = self.frontend.readiness().filter_interest();
3759 if !frontend_work.is_readable()
3760 && !frontend_work.is_writable()
3761 && !frontend_work.is_error()
3762 && all_backends_readiness_are_empty
3763 {
3764 break;
3765 }
3766
3767 counter += 1;
3768 if counter >= MAX_LOOP_ITERATIONS {
3769 if budget_spent {
3770 break;
3771 }
3772 // A call that forwarded at least a byte per budgeted
3773 // iteration is a transfer whose peers
3774 // keep up, not a spin. Closing it cut a fast download or
3775 // upload at a random offset with a clean close
3776 // (sozu-proxy/sozu#1906); running on would hold the
3777 // single-threaded worker for the whole transfer. Leave
3778 // every readiness bit as it is and yield:
3779 // `Server::resume_yielded_sessions` calls `ready` again
3780 // after the next poll, which serves the other sessions
3781 // first.
3782 if progressed_through_loop_budget(
3783 bytes_out_at_entry,
3784 self.context.stream_bytes_out,
3785 ) {
3786 incr!(names::http::MUX_LOOP_BUDGET_YIELD);
3787 debug!(
3788 "{} Mux loop reached {} iterations while forwarding, yielding: frontend={:?}",
3789 log_context!(self),
3790 MAX_LOOP_ITERATIONS,
3791 self.frontend.readiness()
3792 );
3793 self.context.loop_budget_yielded = true;
3794 budget_spent = true;
3795 break;
3796 }
3797 if self.wait_for_client_at_loop_limit() {
3798 budget_spent = true;
3799 break;
3800 }
3801 return SessionResult::Close;
3802 }
3803 }
3804
3805 let context = &mut self.context;
3806 let mut dirty = false;
3807 while let Some(stream_id) = context.pending_links.pop_front() {
3808 let Some(stream) = context.streams.get(stream_id) else {
3809 continue;
3810 };
3811 if stream.state != StreamState::Link {
3812 continue;
3813 }
3814 // Before the first request triggers a stream Link, the frontend timeout is set
3815 // to a shorter request_timeout, here we switch to the longer nominal timeout.
3816 //
3817 // Only on a request's first link. A re-link — failover after a
3818 // failed dial, a stale-upstream replay — keeps the deadline armed
3819 // then, so `front_timeout` bounds the whole failover: re-arming it
3820 // on every attempt let a request spend `max_connection_attempts`
3821 // times `connect_timeout` before any answer. When it expires
3822 // during failover, the frontend-timeout branch of
3823 // `Mux::timeout_inner` answers as for any request still waiting on
3824 // a backend (sozu-proxy/sozu#1800).
3825 if stream.attempts == 0 {
3826 self.frontend
3827 .set_timeout_duration(self.configured_frontend_timeout, context.now);
3828 }
3829 let front_readiness = self.frontend.readiness_mut();
3830 dirty = true;
3831 // Settle the ledger before the ONLY reader of backend load
3832 // state on this path runs: the load balancer reached from
3833 // `Mux::dial_backend` below weighs `active_requests` and
3834 // `connection_time`. Draining here is what keeps a second
3835 // stream linking in the same pass seeing the first stream's
3836 // charge, exactly as it did when the charge was applied in
3837 // place. It is also what makes the view each selection reads
3838 // current: selection reserves `active_connections` itself
3839 // (`BackendMap::reserve_backend`), and a `connect(2)` that
3840 // failed in the previous iteration left a `DialFailed`
3841 // release on the ledger, applied here before the next
3842 // selection reads the counters (#1684).
3843 self.backend_registry
3844 .apply_all(&mut context.backend_deltas, &self.backends);
3845 // Build the routing view once for this decision, from a single
3846 // `proxy.borrow()`. Holding one `Ref` for the call is what
3847 // gives every read inside it the same cluster map; the two
3848 // sites used to take separate borrows. A second immutable
3849 // borrow inside (`plan_connect`'s limit gate, and
3850 // `attach_dialed`'s `add_session` / `register_socket`) is fine
3851 // — nothing on this path borrows the proxy mutably.
3852 let proxy_ref = proxy.borrow();
3853 let backend_registry = &self.backend_registry;
3854 let retired = |backend: &BackendId| backend_registry.is_retired(backend);
3855 let max_connection_attempts = proxy_ref.sessions().borrow().max_connection_attempts;
3856 let view = router::RoutingView::new(proxy_ref.clusters(), proxy_ref.kind())
3857 .with_backend_retired(&retired)
3858 .with_max_connection_attempts(max_connection_attempts);
3859 match self
3860 .router
3861 .plan_connect(stream_id, context, &view)
3862 .and_then(|step| match step {
3863 router::ConnectStep::Decided(plan) => Ok(plan),
3864 // The core resolved a cluster and paused: the
3865 // per-(cluster, source-IP) gate is a consult AND a
3866 // mutation of worker-global session state, so the core
3867 // cannot perform it. Answer it here, then resume.
3868 //
3869 // The track must land BEFORE the decision resumes, not
3870 // after: `cluster_ip_at_limit` is read by every other
3871 // session on this worker and by `lib/src/tcp.rs`'s own
3872 // gate, so a stream admitted but not yet counted is a
3873 // slot a concurrent stream can take twice.
3874 router::ConnectStep::CheckIpLimit(resume) => {
3875 // `plan_connect` stored the routed cluster before
3876 // it paused; an unset one is refused, not gated
3877 // against an empty key.
3878 let Some(cluster_id) =
3879 context.http_context(stream_id).cluster_id.as_ref()
3880 else {
3881 return Err(BackendConnectionError::MaxSessionsMemory);
3882 };
3883 let verdict = consult_ip_gate(
3884 &proxy.borrow().sessions(),
3885 self.frontend_token,
3886 cluster_id,
3887 &resume,
3888 );
3889 self.router
3890 .plan_connect_resume(stream_id, context, &view, resume, verdict)
3891 }
3892 })
3893 .and_then(|plan| match plan {
3894 router::ConnectPlan::Attached => Ok(()),
3895 router::ConnectPlan::Dial {
3896 cluster_id,
3897 h2,
3898 frontend_should_stick,
3899 } => Self::dial_backend(
3900 &mut self.router,
3901 &mut self.backend_registry,
3902 &self.backends,
3903 stream_id,
3904 context,
3905 &session,
3906 &proxy,
3907 cluster_id,
3908 h2,
3909 frontend_should_stick,
3910 ),
3911 }) {
3912 Ok(_) => {
3913 let state = context.streams[stream_id].state;
3914 context.debug.push(DebugEvent::CC(stream_id, state));
3915 }
3916 Err(error) => {
3917 trace!("{} Connection error: {}", log_module_context!(), error);
3918 let stream = &mut context.streams[stream_id];
3919 // The registry this stream captured when its request
3920 // arrived, not whatever the listener holds now.
3921 let answers_rc = stream.answers.clone();
3922 let answers = answers_rc.borrow();
3923 use BackendConnectionError as BE;
3924 match error {
3925 BE::MaxConnectionRetries(_)
3926 | BE::MaxSessionsMemory
3927 | BE::MaxBuffers => {
3928 warn!(
3929 "{} backend retry budget exhausted: {}",
3930 log_module_context!(stream.context),
3931 error
3932 );
3933 set_default_answer(stream, front_readiness, 503, &answers);
3934 }
3935 BE::Backend(BackendError::NoBackendForCluster(ref cluster_id)) => {
3936 // A cluster that has backends, none of them
3937 // selectable, is in an outage: its backends
3938 // failed their health checks or the
3939 // connections Sōzu opened to them. A cluster
3940 // with no backend at all is a routing miss.
3941 stream.context.backends_unavailable = self
3942 .backends
3943 .borrow()
3944 .backends
3945 .get(cluster_id.as_str())
3946 .is_some_and(|list| !list.backends.is_empty());
3947 set_default_answer(stream, front_readiness, 503, &answers);
3948 }
3949 BE::RetrieveClusterError(RetrieveClusterError::RetrieveFrontend(
3950 ref err,
3951 )) => {
3952 // RFC 9110 §15.5.1: a malformed authority is a
3953 // 400. A syntactically valid authority that
3954 // simply has no matching frontend stays on the
3955 // historical 404 path.
3956 let code = match err {
3957 FrontendFromRequestError::HostParse { .. }
3958 | FrontendFromRequestError::InvalidCharsAfterHost(_) => 400,
3959 FrontendFromRequestError::NoClusterFound(_) => 404,
3960 };
3961 set_default_answer(stream, front_readiness, code, &answers);
3962 }
3963 BE::RetrieveClusterError(RetrieveClusterError::UnauthorizedRoute) => {
3964 set_default_answer(stream, front_readiness, 401, &answers);
3965 }
3966 BE::RetrieveClusterError(RetrieveClusterError::SilentClose) => {
3967 // No response byte (sozu-proxy/sozu#767): an
3968 // H2 frontend resets this stream with
3969 // RST_STREAM(CANCEL) while the connection's
3970 // other streams continue; an H1 frontend
3971 // closes the connection without writing.
3972 // The RTT is sampled for this refusal only,
3973 // as the access-log sites do (#1590).
3974 let client_rtt = self.frontend.rtt();
3975 for event in silent_close_answer(
3976 stream,
3977 self.frontend.readiness_mut(),
3978 context.listener.clone(),
3979 client_rtt,
3980 )
3981 .into_iter()
3982 .flatten()
3983 {
3984 crate::protocol::mux::h2::record_metric(event);
3985 }
3986 }
3987 BE::RetrieveClusterError(
3988 RetrieveClusterError::SniAuthorityMismatch { .. },
3989 ) => {
3990 // RFC 9110 §15.5.20: 421 Misdirected Request is the
3991 // semantically correct status for an authority that
3992 // does not belong to this TLS connection. The
3993 // http.sni_authority_mismatch metric emitted in
3994 // `route_from_request` remains the durable signal;
3995 // the 421 body here is what a client sees and may
3996 // retry on a fresh TLS connection with a matching SNI.
3997 set_default_answer(stream, front_readiness, 421, &answers);
3998 }
3999 BE::RetrieveClusterError(RetrieveClusterError::HttpsRedirect) => {
4000 // Use the redirect status stashed by `Router::route_from_request`
4001 // (#1009). Falls back to 301 for the legacy
4002 // `cluster.https_redirect = true` path that does
4003 // not set the field.
4004 let code = stream.context.redirect_status.unwrap_or(301);
4005 set_default_answer(stream, front_readiness, code, &answers);
4006 }
4007
4008 // `connect(2)` itself failed: the dial was refused
4009 // before it could be registered. `Mux::dial_backend`
4010 // recorded the failure and the address; like a dial
4011 // refused later, the request goes to another
4012 // backend while its attempts last, and
4013 // `Router::plan_connect` answers 503 once they are
4014 // spent (sozu-proxy/sozu#1800). The stream is still
4015 // `Link`.
4016 BE::Backend(BackendError::ConnectionFailures { ref error, .. }) => {
4017 debug!(
4018 "{} backend connect failed, retrying: {}",
4019 log_module_context!(stream.context),
4020 error
4021 );
4022 debug_assert_eq!(stream.state, StreamState::Link);
4023 context.pending_links.push_back(stream_id);
4024 }
4025 BE::Backend(ref e) => {
4026 error!("{} backend connection error: {}", log_module_context!(), e);
4027 set_default_answer(stream, front_readiness, 503, &answers);
4028 }
4029 BE::RetrieveClusterError(ref other) => {
4030 error!(
4031 "{} unexpected RetrieveClusterError variant: {:?}",
4032 log_module_context!(),
4033 other
4034 );
4035 set_default_answer(stream, front_readiness, 503, &answers);
4036 }
4037 // TCP specific error
4038 BE::NotFound(ref msg) => {
4039 error!(
4040 "{} NotFound is TCP-specific, not reachable in mux: {:?}",
4041 log_module_context!(),
4042 msg
4043 );
4044 set_default_answer(stream, front_readiness, 503, &answers);
4045 }
4046 // Per-(cluster, source-IP) connection limit reached.
4047 // Emit HTTP 429 with the resolved `Retry-After`. The
4048 // value is computed in `Router::plan_connect` (where the
4049 // SessionManager + cluster override are reachable)
4050 // and stashed on the stream context just before the
4051 // error is returned, so the answer engine can render
4052 // (or elide) the header without re-deriving the
4053 // resolution chain here.
4054 BE::TooManyConnectionsPerIp { ref cluster_id } => {
4055 debug!(
4056 "{} per-(cluster, source-IP) limit hit for cluster {:?}",
4057 log_module_context!(),
4058 cluster_id
4059 );
4060 let retry_after = stream.context.retry_after_seconds;
4061 set_default_answer_with_retry_after(
4062 stream,
4063 front_readiness,
4064 429,
4065 &answers,
4066 retry_after,
4067 );
4068 }
4069 // A stale-upstream replay whose preconditions no
4070 // longer hold (sozu-proxy/sozu#1442). 502 is the
4071 // answer this stream received before the replay
4072 // existed, so refusing costs the client nothing it
4073 // would not already have paid — and costs it far
4074 // less than H1 wire bytes framed as an HTTP/2 DATA
4075 // payload, or a second partial copy of the request
4076 // trailing the first.
4077 BE::ReplayRefused(reason) => {
4078 warn!(
4079 "{} stale-upstream replay refused: {}",
4080 log_module_context!(stream.context),
4081 reason
4082 );
4083 set_default_answer(stream, front_readiness, 502, &answers);
4084 }
4085 }
4086 context.debug.push(DebugEvent::CCF(stream_id, error));
4087 }
4088 }
4089 // Every routing error arm sets a default answer, or for a
4090 // silent close an errored empty one, moving the stream out of
4091 // Link state, except a failed `connect(2)`, which re-queues it
4092 // above for another backend.
4093 }
4094 if !dirty || budget_extra_pass {
4095 break;
4096 }
4097 budget_extra_pass = budget_spent;
4098 }
4099
4100 // Stop service timers before yielding to epoll, so idle wait time is excluded
4101 // from the service_time metric. For Close/Upgrade returns, close() handles cleanup.
4102 for stream in &mut self.context.streams {
4103 if stream.state.is_open() {
4104 stream.metrics.service_stop();
4105 }
4106 }
4107
4108 #[cfg(debug_assertions)]
4109 {
4110 use std::collections::HashMap;
4111
4112 // Verify backend_streams index matches actual stream states.
4113 let mut expected: HashMap<Token, Vec<GlobalStreamId>> = HashMap::new();
4114 for (id, stream) in self.context.streams.iter().enumerate() {
4115 if let StreamState::Linked(token) = stream.state {
4116 expected.entry(token).or_default().push(id);
4117 }
4118 }
4119 // An emptied entry is capacity kept for the next link on the same
4120 // connection (#1583), not a link: it is left out of the count,
4121 // and it must name a connection the router still holds, which is
4122 // what keeps the index bounded by the live backends.
4123 for (token, ids) in self.context.backend_streams.iter() {
4124 assert!(
4125 !ids.is_empty() || self.router.backends.contains_key(token),
4126 "backend_streams keeps an emptied entry for {token:?}, \
4127 which is not a live backend connection"
4128 );
4129 }
4130 assert_eq!(
4131 expected.len(),
4132 self.context
4133 .backend_streams
4134 .values()
4135 .filter(|ids| !ids.is_empty())
4136 .count(),
4137 "backend_streams index key count mismatch: expected={:?}, actual={:?}",
4138 expected,
4139 self.context.backend_streams
4140 );
4141 for (token, mut expected_ids) in expected {
4142 let mut actual_ids = self
4143 .context
4144 .backend_streams
4145 .get(&token)
4146 .map_or_else(Vec::new, |ids| ids.to_vec());
4147 expected_ids.sort();
4148 actual_ids.sort();
4149 assert_eq!(
4150 expected_ids, actual_ids,
4151 "backend_streams index mismatch for token {token:?}",
4152 );
4153 }
4154 }
4155
4156 SessionResult::Continue
4157 }
4158
4159 fn timeout_inner(&mut self, token: Token, _metrics: &mut SessionMetrics) -> StateResult {
4160 trace!("{} MuxState::timeout({:?})", log_context!(self), token);
4161 // `timeout` runs outside `ready()`, so it is its own sampling point.
4162 // `cancel_timed_out_streams` below reaps on this snapshot.
4163 self.context.now = Instant::now();
4164 self.context.now_wall_ms = unix_epoch_ms();
4165 // Consume the wheel entry and re-validate it BEFORE the per-token
4166 // branches, so both of them share one gate. See
4167 // `Mux::consume_timer_entry`.
4168 if !self.consume_timer_entry(token) {
4169 trace!(
4170 "{} MuxState::timeout: early wheel delivery on {:?}, re-armed",
4171 log_context!(self),
4172 token
4173 );
4174 return StateResult::Continue;
4175 }
4176 let front_is_h2 = match self.frontend {
4177 Connection::H1(_) => false,
4178 Connection::H2(_) => true,
4179 };
4180 if self.frontend_token == token && self.frontend.is_lingering() {
4181 // The lingering close's deadline: stop draining and close.
4182 debug!(
4183 "{} Mux lingering close deadline reached: {:?}",
4184 log_context!(self),
4185 self.frontend
4186 );
4187 return StateResult::CloseSession;
4188 }
4189 let mut should_close = true;
4190 let mut should_write = false;
4191 if self.frontend_token == token {
4192 trace!(
4193 "{} MuxState::timeout_frontend({:#?})",
4194 log_context!(self),
4195 self.frontend
4196 );
4197 // The per-stream reaper (bidirectional-idle + outbound
4198 // flow-control-stall guards) normally runs only from `readable()`,
4199 // but a fully-silent peer never triggers a read event. Run it on the
4200 // connection-timeout path too so a window-stalled stream — a buffered
4201 // response the peer refuses to drain by holding its receive window
4202 // shut — is reaped and its MAX_CONCURRENT_STREAMS slot freed, instead
4203 // of lingering until the 30-minute zombie checker. The reaper queues
4204 // an `RST_STREAM(CANCEL)`; because `has_pending_write()` does NOT
4205 // observe the `H2ControlTx` queue (it gates connection close, so a
4206 // queued RST must not read as "keep open"), set `should_write` via
4207 // the dedicated `has_pending_control_write()` probe so the reset is
4208 // actually flushed to the peer before the connection closes — without
4209 // it, a fully-silent peer's stalled stream is freed but the peer sees
4210 // only EOF, never the RST(CANCEL).
4211 if let Connection::H2(h2) = &mut self.frontend {
4212 h2.cancel_timed_out_streams(
4213 &mut self.context,
4214 &mut EndpointClient(&mut self.router),
4215 );
4216 if h2.core.has_pending_control_write() {
4217 should_write = true;
4218 }
4219 }
4220 if self.frontend.has_pending_write() {
4221 should_write = true;
4222 }
4223 // Stream-specific on an H1 frontend only; see
4224 // `response_waits_on_client`.
4225 let front_pending_write = !front_is_h2 && self.frontend.has_pending_write();
4226 // Streams whose response waits on their backend, not on the
4227 // client: the frontend timeout leaves them linked, to their
4228 // backend's timeout (sozu-proxy/sozu#1906).
4229 let mut waiting_on_backend: Vec<GlobalStreamId> = Vec::new();
4230 let front_readiness = self.frontend.readiness_mut();
4231 for stream_id in 0..self.context.streams.len() {
4232 match self.context.streams[stream_id].state {
4233 StreamState::Idle => {
4234 // In h1 an Idle stream is always the first request, so we can send a 408
4235 // In h2 an Idle stream doesn't necessarily hold a request yet,
4236 // in most cases it was just reserved, so we can just ignore them.
4237 if !front_is_h2 {
4238 // Each stream answers out of the registry it
4239 // captured when its request arrived; a listener
4240 // reload since then belongs to the next one.
4241 let answers_rc = self.context.streams[stream_id].answers.clone();
4242 let answers = answers_rc.borrow();
4243 let stream = &mut self.context.streams[stream_id];
4244 stream.context.access_log_message = Some("client_timeout");
4245 set_default_answer(stream, front_readiness, 408, &answers);
4246 should_write = true;
4247 }
4248 }
4249 StreamState::Link => {
4250 // This is an unusual case, as we have both a complete request and no
4251 // available backend yet. For now, we answer with 503.
4252 // Not a timeout-driven outcome from the operator's
4253 // perspective — leave access_log_message as None.
4254 let answers_rc = self.context.streams[stream_id].answers.clone();
4255 let answers = answers_rc.borrow();
4256 let stream = &mut self.context.streams[stream_id];
4257 set_default_answer(stream, front_readiness, 503, &answers);
4258 should_write = true;
4259 }
4260 StreamState::Linked(_) => {
4261 // The frontend timed out while a stream is linked to a backend.
4262 // The backend timeout should handle this, but in case the backend
4263 // is also stalled, send a 504 and terminate the stream.
4264 if !self.context.streams[stream_id].back.consumed {
4265 self.context.unlink_stream(stream_id);
4266 let answers_rc = self.context.streams[stream_id].answers.clone();
4267 let answers = answers_rc.borrow();
4268 let stream = &mut self.context.streams[stream_id];
4269 stream.context.access_log_message =
4270 Some("client_timeout_during_response");
4271 set_default_answer(stream, front_readiness, 504, &answers);
4272 should_write = true;
4273 } else if self.context.streams[stream_id].back.is_completed()
4274 && (self.context.streams[stream_id].back.is_terminated()
4275 || self.context.streams[stream_id].back.is_error())
4276 {
4277 // Response fully proxied, stream can be closed.
4278 // kawa's `is_completed` only says nothing is queued:
4279 // without the terminated/error half, a response
4280 // still waiting on its backend took this arm and
4281 // was cut, silently (sozu-proxy/sozu#1906).
4282 } else if self.context.streams[stream_id].back.is_terminated()
4283 || self.context.streams[stream_id].back.is_error()
4284 {
4285 // Response is terminated/error but not fully written to frontend.
4286 // Keep the session alive briefly to flush remaining data.
4287 should_close = false;
4288 } else if !response_waits_on_client(
4289 &self.context.streams[stream_id],
4290 front_pending_write,
4291 ) {
4292 // The client has taken every byte so far: the
4293 // response waits on its backend, whose own
4294 // timeout bounds that wait. Cutting it here
4295 // charged a slow backend to the client.
4296 trace!(
4297 "{} frontend timeout while stream {} waits on its backend, deferring to back_timeout",
4298 log_module_context!(),
4299 stream_id
4300 );
4301 waiting_on_backend.push(stream_id);
4302 should_close = false;
4303 } else {
4304 // Partial response in progress — forcefully terminate
4305 self.context.unlink_stream(stream_id);
4306 let stream = &mut self.context.streams[stream_id];
4307 stream.context.access_log_message =
4308 Some("client_timeout_during_response");
4309 forcefully_terminate_answer(
4310 stream,
4311 front_readiness,
4312 H2Error::InternalError,
4313 );
4314 should_write = true;
4315 }
4316 // end_stream is called in a second pass below to avoid
4317 // borrow conflicts on context.streams.
4318 }
4319 StreamState::Unlinked => {
4320 // A stream Unlinked already has a response and its backend closed.
4321 // In case it hasn't finished proxying we wait. Otherwise it is a stream
4322 // kept alive for a new request, which can be killed.
4323 if !self.context.streams[stream_id].back.is_completed() {
4324 should_close = false;
4325 }
4326 }
4327 StreamState::Recycle => {
4328 // A recycled stream is an h2 stream which doesn't hold a request anymore.
4329 // We can ignore it.
4330 }
4331 }
4332 }
4333 // Second pass: end streams that were linked to backends.
4334 // This is done separately to avoid borrow conflicts on context.streams.
4335 let linked_streams: Vec<(GlobalStreamId, Token)> = self
4336 .context
4337 .streams
4338 .iter()
4339 .enumerate()
4340 .filter_map(|(id, stream)| {
4341 if let StreamState::Linked(back_token) = stream.state
4342 && !waiting_on_backend.contains(&id)
4343 {
4344 Some((id, back_token))
4345 } else {
4346 None
4347 }
4348 })
4349 .collect();
4350 for (stream_id, back_token) in linked_streams {
4351 if let Some(backend) = self.router.backends.get_mut(&back_token) {
4352 #[cfg(debug_assertions)]
4353 debug_assert_h1_owns_stream(backend, stream_id);
4354 backend.end_stream(stream_id, &mut self.context);
4355 }
4356 }
4357 } else if let Some(backend) = self.router.backends.get_mut(&token) {
4358 // Captured before the backend borrow so the re-arm below can reach
4359 // the pass's clock snapshot.
4360 let now = self.context.now;
4361 trace!(
4362 "{} MuxState::timeout_backend({:#?})",
4363 log_context_lite!(self),
4364 backend
4365 );
4366 // A dial still `Connecting` when its timer fires is the
4367 // `connect_timeout` armed by `Mux::dial_backend`: the backend
4368 // accepted nothing, so no byte of any request reached it. That is
4369 // a backend connect failure exactly like a refused dial, and it
4370 // takes the same path: flag the connection dead as the kernel
4371 // flags a refused one, and let `ready_inner`'s dead-backend branch
4372 // count the failure (`failures`, `retry_policy`, `backend.down`),
4373 // close the connection and re-queue its streams for another
4374 // backend through `end_stream_decision`'s `Reconnect`. Answering
4375 // 504 here left the failure uncounted, so the load balancer kept
4376 // choosing the dead backend (sozu-proxy/sozu#1800).
4377 //
4378 // The pass that does this runs right after this timeout:
4379 // `Mux::has_pending_backend_failure` reports the flag and
4380 // `Server::timeout` calls `ready` on it (`ProxySession::
4381 // needs_ready_pass`). Nothing is re-armed for this backend: that
4382 // pass removes it.
4383 if matches!(
4384 backend.position(),
4385 Position::Client(_, _, BackendStatus::Connecting(_))
4386 ) {
4387 debug!(
4388 "{} backend connect timeout on {:?}, failing over",
4389 log_context_lite!(self),
4390 token
4391 );
4392 backend.readiness_mut().event.insert(Ready::ERROR);
4393 return StateResult::Continue;
4394 }
4395 // Stream-specific on an H1 frontend only; see
4396 // `response_waits_on_client`.
4397 let front_pending_write = !front_is_h2 && self.frontend.has_pending_write();
4398 // Set when a stream's response waits on the client: the backend
4399 // timeout leaves it linked, to the frontend's timeout
4400 // (sozu-proxy/sozu#1906).
4401 let mut waiting_on_client = false;
4402 let front_readiness = self.frontend.readiness_mut();
4403 let linked_ids: Vec<GlobalStreamId> = self
4404 .context
4405 .backend_streams
4406 .get(&token)
4407 .map_or_else(Vec::new, |ids| ids.to_vec());
4408 for stream_id in linked_ids {
4409 // This stream is linked to the backend that timedout
4410 if self.context.streams[stream_id].back.is_terminated()
4411 || self.context.streams[stream_id].back.is_error()
4412 {
4413 trace!(
4414 "{} Stream terminated or in error, do nothing, just wait a bit more",
4415 log_module_context!()
4416 );
4417 // Nothing to do, simply wait for the remaining bytes to be proxied
4418 if !self.context.streams[stream_id].back.is_completed() {
4419 should_close = false;
4420 }
4421 } else if !self.context.streams[stream_id].back.consumed {
4422 // The response has not started yet
4423 trace!(
4424 "{} Stream still waiting for response, send 504",
4425 log_module_context!()
4426 );
4427 self.context.unlink_stream(stream_id);
4428 let answers_rc = self.context.streams[stream_id].answers.clone();
4429 let answers = answers_rc.borrow();
4430 let stream = &mut self.context.streams[stream_id];
4431 stream.context.access_log_message = Some("backend_timeout");
4432 set_default_answer(stream, front_readiness, 504, &answers);
4433 should_write = true;
4434 } else if response_waits_on_client(
4435 &self.context.streams[stream_id],
4436 front_pending_write,
4437 ) {
4438 // Sōzu holds response bytes the client has not taken, so
4439 // it stopped reading this backend itself: the backend is
4440 // not inactive, the client is slow. The frontend timeout
4441 // bounds that; cutting here truncated a large download to
4442 // a client that paused longer than `back_timeout`.
4443 trace!(
4444 "{} backend timeout while stream {} waits on the client, deferring to front_timeout",
4445 log_module_context!(),
4446 stream_id
4447 );
4448 waiting_on_client = true;
4449 should_close = false;
4450 continue;
4451 } else {
4452 trace!(
4453 "{} Stream waiting for end of response, forcefully terminate it",
4454 log_module_context!()
4455 );
4456 self.context.unlink_stream(stream_id);
4457 let stream = &mut self.context.streams[stream_id];
4458 stream.context.access_log_message = Some("backend_response_timeout");
4459 forcefully_terminate_answer(stream, front_readiness, H2Error::InternalError);
4460 should_write = true;
4461 }
4462 #[cfg(debug_assertions)]
4463 debug_assert_h1_owns_stream(backend, stream_id);
4464 backend.end_stream(stream_id, &mut self.context);
4465 }
4466 // Re-arm the backend timeout if the session stays alive (draining streams).
4467 // Without this, the timeout is consumed and the session becomes immortal
4468 // until the zombie checker runs.
4469 //
4470 // The `!should_close` condition looks like a hole, and is not. On
4471 // `should_close`, control falls through to the shared tail below,
4472 // which can still return `Continue` — through the `should_write`
4473 // writable loop, or through `delay_close_for_frontend_flush` —
4474 // having re-armed only the FRONTEND container. That would be a
4475 // lost wakeup if this backend could still be carrying work with no
4476 // timer of its own. It cannot, and the chain is four links, all in
4477 // this file and `connection.rs`/`h1.rs`/`h2.rs`:
4478 //
4479 // 1. the loop above iterates `context.backend_streams[&token]` —
4480 // every stream currently linked to this backend, and nothing
4481 // else — and calls `Connection::end_stream` on EVERY iteration,
4482 // outside the if/else chain, so no arm can skip it;
4483 // 2. `ConnectionH2::end_stream` (`h2.rs`) starts with
4484 // `context.unlink_stream(...)` unconditionally.
4485 // `ConnectionH1::end_stream` (`h1.rs`) does NOT: it first guards
4486 // `if self.stream != Some(stream) { error!(..); return; }` and
4487 // that early return skips the unlink. The two are dispatched by
4488 // `Connection::end_stream` in `connection.rs`;
4489 // 3. the H1 guard cannot reject a stream this loop passes it,
4490 // because an H1 connection carries at most one linked stream and
4491 // it is exactly `self.stream`. That holds because
4492 // `self.stream = Some(..)` occurs only in
4493 // `ConnectionH1::start_stream`, paired with
4494 // `Context::link_stream`, and `self.stream = None` occurs only
4495 // inside `end_stream` itself, after the unlink has run. So while
4496 // `backend_streams[&token]` names a stream, `self.stream` names
4497 // the same one — and `debug_assert_h1_owns_stream`, called
4498 // just above every `end_stream` in this function, is the
4499 // tripwire that fires if it ever stops being true;
4500 // 4. so by the time the tail runs, this backend has no linked
4501 // stream left. The index-consistency `assert_eq!` at the end of
4502 // `Mux::ready` pins the other direction (`backend_streams` is
4503 // exactly the set of `StreamState::Linked` streams).
4504 //
4505 // `Context::unlink_stream` is the eviction point this path relies
4506 // on, not the only one in the module: `remove_backend_stream` also
4507 // has direct callers in `h1.rs` and `h2.rs`. Extra eviction can
4508 // only strengthen the conclusion, but the premise is "this loop
4509 // evicts", not "nothing else can".
4510 //
4511 // Work attached to this backend AFTERWARDS does not depend on
4512 // this site either, and since the adapter landed it no longer even
4513 // depends on the connection doing I/O: a pooled backend stays in
4514 // `router.backends`, so `Mux::reschedule` keeps its handle armed at
4515 // whatever deadline the core holds, every pass. (`arm_timeout()` at
4516 // the top of `ConnectionH1::{readable,writable}` and in
4517 // `ConnectionH2::{handle_write,handle_headers_frame}` pushes that
4518 // deadline out on the first pass that moves a stream's bytes — not
4519 // on every pass, LIFECYCLE §9 invariant 9 — but nothing hangs on
4520 // it.) That is what covers the pool-reuse branch of
4521 // `Router::plan_connect` never arming a timeout, unlike the fresh-dial
4522 // branch.
4523 //
4524 // The assertion below is the cheap tripwire for link 3: if the
4525 // impossible becomes possible (a new arm that skips `end_stream`,
4526 // or an `end_stream` that stops unlinking), it fires here rather
4527 // than as an immortal backend in production.
4528 #[cfg(debug_assertions)]
4529 debug_assert!(
4530 waiting_on_client
4531 || self
4532 .context
4533 .backend_streams
4534 .get(&token)
4535 .is_none_or(|ids| ids.is_empty()),
4536 "backend {token:?} still holds linked streams {:?} after its \
4537 timeout reaped them: on the should_close path nothing re-arms \
4538 its container, so those streams would have no backend timer",
4539 self.context.backend_streams.get(&token),
4540 );
4541 if !should_close {
4542 backend.arm_timeout(now);
4543 }
4544 // A deferral must not reach the shared tail, which re-arms the
4545 // FRONTEND timer: re-armed on every backend timeout, a client that
4546 // stopped reading would never time out at all.
4547 if waiting_on_client && !should_write {
4548 return StateResult::Continue;
4549 }
4550 } else {
4551 // Session received a timeout for an unknown token, ignore it
4552 return StateResult::Continue;
4553 }
4554 if should_write {
4555 // Drain as much pending data as possible before closing.
4556 // A single writable() call is insufficient for large responses —
4557 // the TLS buffer may need multiple flushes. Without this loop,
4558 // the session is killed with unflushed TLS data, causing the
4559 // client to receive a truncated TLS record ("decode error").
4560 //
4561 // The constant 16 is empirical: it papers over a missing
4562 // invariant-15 hop in the H2 mux state machine where the
4563 // writable readiness signal is not always re-armed after a
4564 // partial flush. Long-term plan: reach invariant-15 closure
4565 // and remove this loop. See `lib/src/protocol/mux/LIFECYCLE.md`.
4566 let mut result = StateResult::Continue;
4567 for _ in 0..16 {
4568 result = match self
4569 .frontend
4570 .writable(&mut self.context, EndpointClient(&mut self.router))
4571 {
4572 MuxResult::Continue => StateResult::Continue,
4573 MuxResult::Upgrade => StateResult::Upgrade,
4574 MuxResult::CloseSession => StateResult::CloseSession,
4575 };
4576 if result != StateResult::Continue
4577 || !self.frontend.readiness_mut().interest.is_writable()
4578 {
4579 break;
4580 }
4581 }
4582 // Re-arm the frontend timeout so the session doesn't become immortal.
4583 // The writable call may have partially flushed the response — we need
4584 // the timeout to fire again if the flush stalls.
4585 if result == StateResult::Continue {
4586 self.frontend.arm_timeout(self.context.now);
4587 }
4588 return result;
4589 }
4590 if should_close {
4591 if self.delay_close_for_frontend_flush("timeout") {
4592 debug!(
4593 "{} Mux timeout delaying close for frontend flush: token={:?}, frontend={:?}",
4594 log_context!(self),
4595 token,
4596 self.frontend
4597 );
4598 self.frontend.arm_timeout(self.context.now);
4599 return StateResult::Continue;
4600 }
4601 if front_is_h2 {
4602 debug!(
4603 "{} Mux timeout returning CloseSession: token={:?}, frontend={:?}",
4604 log_context!(self),
4605 token,
4606 self.frontend
4607 );
4608 for (idx, stream) in self.context.streams.iter().enumerate() {
4609 if stream.state != StreamState::Recycle {
4610 debug!(
4611 "{} timeout stream[{}]: state={:?}, front_phase={:?}, back_phase={:?}, front_completed={}, back_completed={}",
4612 log_context!(self),
4613 idx,
4614 stream.state,
4615 stream.front.parsing_phase,
4616 stream.back.parsing_phase,
4617 stream.front.is_completed(),
4618 stream.back.is_completed()
4619 );
4620 }
4621 }
4622 }
4623 StateResult::CloseSession
4624 } else {
4625 // Re-arm the frontend timeout. Without this, the timeout is consumed
4626 // by triggered() and the session stays alive indefinitely until the
4627 // zombie checker runs (default: 30 minutes).
4628 self.frontend.arm_timeout(self.context.now);
4629 StateResult::Continue
4630 }
4631 }
4632
4633 fn shutting_down_inner(&mut self) -> SessionIsToBeClosed {
4634 // `shut_down_sessions()` drives this outside `ready()`, so it is its
4635 // own sampling point, and it is load-bearing rather than belt-and-braces.
4636 // The chain is: this line refreshes `context.now`;
4637 // `drive_frontend_shutdown_io` below always reaches `readable()` for an
4638 // H2 frontend (`force_h2_read` is unconditionally true, so the
4639 // early return above it cannot fire), and `readable` mirrors
4640 // `context.now` into `ConnectionH2::now`; the forced-close check that
4641 // follows then reads a fresh snapshot. Drop this line and a silent
4642 // draining session propagates the snapshot of its last `ready()` pass
4643 // forever, so the graceful-shutdown budget never expires. Pinned by
4644 // `shutting_down_refreshes_the_snapshot_so_the_drain_budget_expires`.
4645 let now = Instant::now();
4646 self.context.now = now;
4647 self.context.now_wall_ms = unix_epoch_ms();
4648 // RFC 9113 §6.8: initiate graceful shutdown with double-GOAWAY pattern.
4649 // Only send the initial GOAWAY once. The final GOAWAY (with the real
4650 // last_stream_id) is handled by finalize_write() when all streams drain.
4651 // Calling graceful_goaway() again would send the final GOAWAY
4652 // prematurely and force-disconnect before in-flight streams complete.
4653 if !self.frontend.is_draining() {
4654 match self.frontend.graceful_goaway(now) {
4655 MuxResult::CloseSession => return true,
4656 MuxResult::Continue => {
4657 // graceful_goaway() queued a GOAWAY frame. Flush it directly
4658 // since the event loop uses edge-triggered epoll and won't
4659 // deliver a new WRITABLE event for an already-writable socket.
4660 self.frontend.flush_output_buffer();
4661 }
4662 _ => {}
4663 }
4664 } else {
4665 trace!(
4666 "{} shutting_down: already draining, skipping duplicate GOAWAY",
4667 log_context!(self)
4668 );
4669 // A peer GOAWAY drains without arming the graceful-shutdown
4670 // budget, which only `graceful_goaway` arms: arm it here, once,
4671 // so the forced-close check below still bounds this session.
4672 self.frontend.arm_graceful_shutdown_deadline(now);
4673 // shut_down_sessions() runs outside ready(), so retry flushing any
4674 // previously-buffered GOAWAY/TLS records on each pass.
4675 self.frontend.flush_output_buffer();
4676 }
4677 if self.drive_frontend_shutdown_io() {
4678 return true;
4679 }
4680 // Forced-close deadline: once the H2 listener's
4681 // `h2_graceful_shutdown_deadline_seconds` budget has elapsed from
4682 // the moment `graceful_goaway` armed `drain.started_at`, stop
4683 // waiting for streams and tear the session down. `drive_frontend_
4684 // shutdown_io` above already had a chance to flush any pending
4685 // TLS/GOAWAY records; this branch accepts that some bytes may be
4686 // lost in exchange for honoring the operator-configured SLA.
4687 // Listeners that disable the knob (`= 0` → `None`) short-circuit
4688 // the check inside `graceful_shutdown_deadline_elapsed`.
4689 //
4690 // RFC 9113 §6.8: the close is still preceded by the final GOAWAY, so
4691 // the peer learns which streams were processed; the advisory one at
4692 // `2^31-1` says nothing about it (sozu-proxy/sozu#1647). It is flushed
4693 // once, best effort, like the advisory GOAWAY above.
4694 if self.frontend.graceful_shutdown_deadline_elapsed() {
4695 debug!(
4696 "{} Mux shutting_down: graceful-shutdown deadline elapsed, forcing close",
4697 log_context!(self)
4698 );
4699 // A lingering connection already sent its final GOAWAY and shut
4700 // its write side down: the operator's budget closes it as is.
4701 if !self.frontend.is_lingering() {
4702 let _ = self.frontend.goaway_before_forced_close(&self.context);
4703 self.frontend.flush_output_buffer();
4704 }
4705 return true;
4706 }
4707 if self.frontend.is_lingering() {
4708 // Bounded by its own deadline (`request_timeout` from the
4709 // linger's start), which the timer enforces. For H2 also by the
4710 // graceful-shutdown budget just above, once that budget is armed:
4711 // `graceful_goaway` arms it when this soft stop sends the first
4712 // GOAWAY, and the lingering passes keep the connection's clock
4713 // fresh so it can elapse. A drain the client's own GOAWAY started
4714 // never armed it, so such a linger runs to its own deadline.
4715 return false;
4716 }
4717 if matches!(self.frontend, Connection::H2(_)) && self.frontend.is_draining() {
4718 for stream in &mut self.context.streams {
4719 if stream.front_received_end_of_stream {
4720 continue;
4721 }
4722 if !matches!(stream.state, StreamState::Linked(_) | StreamState::Unlinked) {
4723 continue;
4724 }
4725 if stream.front.consumed
4726 && stream.front.storage.is_empty()
4727 && stream.front.is_completed()
4728 {
4729 stream.front_received_end_of_stream = true;
4730 self.frontend
4731 .readiness_mut()
4732 .interest
4733 .insert(Ready::WRITABLE);
4734 self.frontend.readiness_mut().signal_pending_write();
4735 }
4736 }
4737 }
4738 // An H2 request that has not reached a backend yet still holds the
4739 // session, bounded by the forced-close deadline above: a field block
4740 // still being reassembled (its stream is `StreamState::Idle`), and a
4741 // complete one awaiting `Mux::ready`'s link pass (`StreamState::Link`),
4742 // which is where a CONTINUATION read by `drive_frontend_shutdown_io`
4743 // leaves it. Closing on either loses a request the advisory GOAWAY
4744 // invited, and skips the final GOAWAY `ConnectionH2::finalize_write`
4745 // sends once the stream is done (sozu-proxy/sozu#1647).
4746 let front_is_h2 = matches!(self.frontend, Connection::H2(_));
4747 let mut can_stop = !self.frontend.peer_header_block_in_progress();
4748 for stream in &mut self.context.streams {
4749 match stream.state {
4750 StreamState::Linked(_) => {
4751 // An HTTP/1.1 response still to come announces the close
4752 // (`Connection: close`, in `HttpContext::on_response_headers`
4753 // of `lib/src/protocol/kawa_h1/editor.rs`), and the
4754 // frontend closes once it is sent instead of waiting for
4755 // the next pass. HTTP/2 announces it with the GOAWAY.
4756 if !front_is_h2 {
4757 stream.context.closing = true;
4758 }
4759 can_stop = false;
4760 }
4761 StreamState::Link if front_is_h2 => {
4762 can_stop = false;
4763 }
4764 StreamState::Unlinked => {
4765 kawa::debug_kawa(&stream.front);
4766 kawa::debug_kawa(&stream.back);
4767 if stream.is_quiesced() {
4768 continue;
4769 }
4770 stream.context.closing = true;
4771 can_stop = false;
4772 }
4773 _ => {}
4774 }
4775 }
4776 if self.frontend.has_pending_write() {
4777 return false;
4778 }
4779 if can_stop {
4780 let active_h2_streams = self
4781 .context
4782 .streams
4783 .iter()
4784 .enumerate()
4785 .filter(|(_, s)| {
4786 if s.state == StreamState::Recycle {
4787 return false;
4788 }
4789 if s.state == StreamState::Unlinked && s.is_quiesced() {
4790 return false;
4791 }
4792 true
4793 })
4794 .collect::<Vec<_>>();
4795 if matches!(self.frontend, Connection::H2(_)) && !active_h2_streams.is_empty() {
4796 debug!(
4797 "{} Mux shutting_down returning true with active H2 streams: {:?}",
4798 log_context!(self),
4799 self.frontend
4800 );
4801 for (idx, stream) in active_h2_streams {
4802 debug!(
4803 "{} shutdown stream[{}]: state={:?}, front_phase={:?}, back_phase={:?}, front_completed={}, back_completed={}",
4804 log_context!(self),
4805 idx,
4806 stream.state,
4807 stream.front.parsing_phase,
4808 stream.back.parsing_phase,
4809 stream.front.is_completed(),
4810 stream.back.is_completed()
4811 );
4812 }
4813 }
4814 }
4815 can_stop
4816 }
4817}
4818
4819/// The embedder's [`router::BackendSelector`]: the worker's backend set and
4820/// this session's registry, lent together for one selection.
4821///
4822/// It is the one place a registry handle becomes an opaque [`BackendId`]: the
4823/// backend is chosen and reserved by the `BackendMap`, named by the session's
4824/// table, and the `Rc` goes no further. Everything past
4825/// [`router::BackendSelector::select`] — the dial in `Mux::dial_backend`, and
4826/// every `Position::Client` built from it — holds the id, never the handle.
4827struct RegistrySelector<'a> {
4828 backends: &'a mut BackendMap,
4829 registry: &'a mut BackendRegistry,
4830 /// The pass's clock sample, [`Context::now`]: selection judges backoff
4831 /// windows and connection-time decay against it rather than reading a
4832 /// clock (#1684).
4833 now: Instant,
4834 /// The backends the stream being linked already failed to connect to,
4835 /// [`Stream::tried_backends`]: selection prefers any other one
4836 /// (sozu-proxy/sozu#1800).
4837 exclude: &'a [std::net::SocketAddr],
4838}
4839
4840impl router::BackendSelector for RegistrySelector<'_> {
4841 fn select(
4842 &mut self,
4843 cluster_id: &str,
4844 affinity: router::Affinity<'_>,
4845 key: Option<u64>,
4846 ) -> Result<router::SelectedBackend, BackendError> {
4847 let handle = match affinity {
4848 router::Affinity::Sticky(Some(cookie)) => {
4849 self.backends.reserve_sticky_backend_excluding(
4850 cluster_id,
4851 cookie,
4852 key,
4853 self.now,
4854 self.exclude,
4855 )?
4856 }
4857 router::Affinity::Sticky(None) | router::Affinity::Unpinned => self
4858 .backends
4859 .reserve_backend_excluding(cluster_id, key, self.now, self.exclude)?,
4860 };
4861 let sticky_session = match affinity {
4862 router::Affinity::Sticky(_) => {
4863 let chosen = handle.borrow();
4864 Some(
4865 chosen
4866 .sticky_id
4867 .clone()
4868 .unwrap_or_else(|| chosen.backend_id.to_owned()),
4869 )
4870 }
4871 router::Affinity::Unpinned => None,
4872 };
4873 let backend = self.registry.id_for(&handle);
4874 // The id names the handle the map just reserved, not a copy of it.
4875 debug_assert!(
4876 self.registry
4877 .get(backend.slot())
4878 .is_some_and(|known| Rc::ptr_eq(known, &handle)),
4879 "the minted id must resolve to the very backend that was reserved"
4880 );
4881 Ok(router::SelectedBackend {
4882 backend,
4883 sticky_session,
4884 })
4885 }
4886}
4887
4888/// Minimal listener + `Mux` scaffolding shared by the `mod.rs` and `h2.rs`
4889/// test modules. Lives here rather than in either test module because
4890/// `Context<L>` is generic over the listener and both need the same `L`.
4891#[cfg(test)]
4892pub(crate) mod test_support {
4893 use std::{
4894 cell::RefCell,
4895 collections::BTreeMap,
4896 net::{SocketAddr, TcpListener as StdTcpListener},
4897 rc::Rc,
4898 };
4899
4900 use sozu_command::logging::CachedTags;
4901
4902 use super::*;
4903 use crate::{
4904 FrontendFromRequestError, L7ListenerHandler, ListenerHandler, Protocol,
4905 protocol::http::{answers::HttpAnswers, parser::Method},
4906 router::RouteResult,
4907 };
4908
4909 /// Implements exactly the nine required methods of `ListenerHandler` +
4910 /// `L7ListenerHandler`; every other knob keeps its trait default.
4911 pub(crate) struct TestListener {
4912 address: SocketAddr,
4913 answers: Rc<RefCell<HttpAnswers>>,
4914 sticky_name: String,
4915 elide_x_real_ip: bool,
4916 }
4917
4918 pub(crate) const TEST_STICKY_NAME: &str = "SOZUBALANCEID";
4919
4920 impl TestListener {
4921 pub(crate) fn new() -> Self {
4922 Self {
4923 address: "127.0.0.1:1".parse().expect("test address must parse"),
4924 answers: test_answers(),
4925 sticky_name: TEST_STICKY_NAME.to_owned(),
4926 elide_x_real_ip: false,
4927 }
4928 }
4929
4930 /// Stand in for an operator listener reload: change the per-request
4931 /// knobs and publish a brand-new answer registry under a new `Rc`,
4932 /// exactly as `HttpListener::update_config` does. Returns the newly
4933 /// published registry.
4934 ///
4935 /// The registry must be *published*, not edited: a test that mutated
4936 /// the existing one in place would prove nothing about the per-stream
4937 /// capture, because both streams would still be holding the same
4938 /// object.
4939 pub(crate) fn reload(
4940 &mut self,
4941 sticky_name: &str,
4942 elide_x_real_ip: bool,
4943 ) -> Rc<RefCell<HttpAnswers>> {
4944 self.sticky_name = sticky_name.to_owned();
4945 self.elide_x_real_ip = elide_x_real_ip;
4946 self.answers = test_answers();
4947 self.answers.clone()
4948 }
4949 }
4950
4951 /// A fresh, empty answer registry — what a test `Stream` renders from
4952 /// when the test is not about templates.
4953 pub(crate) fn test_answers() -> Rc<RefCell<HttpAnswers>> {
4954 Rc::new(RefCell::new(
4955 HttpAnswers::new(&BTreeMap::new()).expect("default answers must build"),
4956 ))
4957 }
4958
4959 impl ListenerHandler for TestListener {
4960 fn get_addr(&self) -> &SocketAddr {
4961 &self.address
4962 }
4963 fn get_tags(&self, _key: &str) -> Option<&CachedTags> {
4964 None
4965 }
4966 fn set_tags(&mut self, _key: String, _tags: Option<BTreeMap<String, String>>) {}
4967 fn protocol(&self) -> Protocol {
4968 Protocol::HTTP
4969 }
4970 fn public_address(&self) -> SocketAddr {
4971 self.address
4972 }
4973 }
4974
4975 impl L7ListenerHandler for TestListener {
4976 fn get_sticky_name(&self) -> &str {
4977 &self.sticky_name
4978 }
4979 fn get_elide_x_real_ip(&self) -> bool {
4980 self.elide_x_real_ip
4981 }
4982 fn get_connect_timeout(&self) -> u32 {
4983 10
4984 }
4985 fn frontend_from_request(
4986 &self,
4987 _host: &str,
4988 _uri: &str,
4989 _method: &Method,
4990 _headers: &dyn crate::router::RequestHeaders,
4991 ) -> Result<RouteResult, FrontendFromRequestError> {
4992 Err(FrontendFromRequestError::InvalidCharsAfterHost(
4993 "test listener routes nothing".to_owned(),
4994 ))
4995 }
4996 fn get_answers(&self) -> &Rc<RefCell<HttpAnswers>> {
4997 &self.answers
4998 }
4999 }
5000
5001 /// Whether the epoll instance behind `epoll` watches descriptor `fd`,
5002 /// read from the `tfd:` lines Linux lists in `/proc/self/fdinfo`.
5003 ///
5004 /// Reading the kernel's own table, rather than counting calls, is what
5005 /// lets a test tell "never deregistered" from "removed by the close".
5006 #[cfg(target_os = "linux")]
5007 pub(crate) fn epoll_watches(epoll: &impl std::os::fd::AsRawFd, fd: std::os::fd::RawFd) -> bool {
5008 let fdinfo = std::fs::read_to_string(format!("/proc/self/fdinfo/{}", epoll.as_raw_fd()))
5009 .expect("the epoll instance must expose its fdinfo");
5010 fdinfo.lines().any(|line| {
5011 line.strip_prefix("tfd:")
5012 .and_then(|rest| rest.split_whitespace().next())
5013 .and_then(|target| target.parse::<std::os::fd::RawFd>().ok())
5014 == Some(fd)
5015 })
5016 }
5017
5018 /// A live, connected, non-blocking loopback socket. Reads return
5019 /// `WouldBlock` rather than `ECONNREFUSED`, so a `readable()` pass over it
5020 /// is a no-op instead of a forced disconnect. The accepted peer is returned
5021 /// so the caller can keep it alive for the duration of the test.
5022 pub(crate) fn connected_socket() -> (mio::net::TcpStream, std::net::TcpStream) {
5023 let listener = StdTcpListener::bind("127.0.0.1:0").expect("bind test listener");
5024 let address = listener.local_addr().expect("listener local addr");
5025 let client = mio::net::TcpStream::connect(address).expect("connect test client");
5026 let (peer, _) = listener.accept().expect("accept test peer");
5027 peer.set_nonblocking(true).expect("peer nonblocking");
5028 (client, peer)
5029 }
5030
5031 /// A `Context` backed by [`TestListener`] and a small buffer pool.
5032 pub(crate) fn test_context(pool: &Rc<RefCell<Pool>>) -> Context<TestListener> {
5033 Context::new(
5034 Ulid::generate(),
5035 Rc::downgrade(pool),
5036 Rc::new(RefCell::new(TestListener::new())),
5037 None,
5038 "127.0.0.1:1".parse().expect("public address must parse"),
5039 )
5040 }
5041}
5042
5043// Counts `writable()` calls made by `Mux::drive_frontend_shutdown_io`, so a
5044// test can pin how much work one shutdown pass does. Thread-local: each test
5045// runs on its own thread.
5046#[cfg(test)]
5047thread_local! {
5048 static SHUTDOWN_WRITABLE_PASSES: std::cell::Cell<u32> = const { std::cell::Cell::new(0) };
5049 /// Times `Mux::wait_for_client_at_loop_limit` ran on this thread, in
5050 /// every build (the debug history is empty in release builds).
5051 static LOOP_LIMIT_HITS: std::cell::Cell<u32> = const { std::cell::Cell::new(0) };
5052}
5053
5054#[cfg(test)]
5055mod tests {
5056 use std::collections::HashMap;
5057
5058 use super::test_support::{TEST_STICKY_NAME, connected_socket, test_context};
5059 use super::*;
5060 use crate::{
5061 pool::Pool,
5062 protocol::http::parser::Method,
5063 protocol::mux::{
5064 h2::{H2ConnectionConfig, H2State},
5065 h2_flood_detector::H2FloodConfig,
5066 },
5067 timer::TimeoutContainer,
5068 };
5069
5070 /// After its final GOAWAY, an H2 frontend closes while the client may
5071 /// still be sending frames. A close with those frames unread, or followed
5072 /// by more, makes the kernel reset the connection and discard the part of
5073 /// the response still queued in the frontend's socket. The connection
5074 /// must shut its write side down and drain instead (RFC 9112 §9.6, the
5075 /// same lingering close as H1), so the client reads all of it, then EOF.
5076 ///
5077 /// The test emulates the session close `HttpsSession::close` performs —
5078 /// `shutdown_write`, then `close(2)` — whenever `writable` asks for it.
5079 #[test]
5080 fn an_h2_frontend_lingers_after_its_final_goaway_so_the_client_reads_everything() {
5081 use std::io::{Read, Write};
5082
5083 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5084 let (socket, mut peer) = connected_socket();
5085 // A client that is slow to read: most of the response stays queued
5086 // in the frontend's socket.
5087 socket2::SockRef::from(&peer)
5088 .set_recv_buffer_size(32 * 1024)
5089 .expect("the client receive buffer must be settable");
5090
5091 let mut h2 = h2::H2Shell::new(
5092 Ulid::generate(),
5093 socket,
5094 Position::Server,
5095 &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5096 H2FloodConfig::default(),
5097 H2ConnectionConfig::default(),
5098 Duration::from_secs(30),
5099 None,
5100 Duration::from_secs(10),
5101 None,
5102 Ready::WRITABLE | Ready::HUP | Ready::ERROR,
5103 )
5104 .expect("a pool with free buffers must yield an H2 connection");
5105 // The final GOAWAY is out: the next writable pass closes.
5106 h2.core.state = H2State::GoAway;
5107 h2.core.drain.__test_set_draining();
5108 h2.core.graceful_goaway = true;
5109
5110 // The response, already handed to the kernel.
5111 let mut queued = 0;
5112 let chunk = [b'r'; 16 * 1024];
5113 loop {
5114 match (&h2.socket).write(&chunk) {
5115 Ok(size) => queued += size,
5116 Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break,
5117 Err(e) => panic!("the response write failed: {e}"),
5118 }
5119 if queued >= 1024 * 1024 {
5120 break;
5121 }
5122 }
5123 // The client still sends frames after the GOAWAY.
5124 peer.write_all(&[0u8; 1024])
5125 .expect("the client may still send after the GOAWAY");
5126 std::thread::sleep(Duration::from_millis(20));
5127
5128 let mut frontend = Connection::H2(h2);
5129 let mut context = test_context(&pool);
5130 let mut router = Router::new(Duration::from_secs(30), Duration::from_secs(30));
5131 context.now = Instant::now();
5132 let result = frontend.writable(&mut context, EndpointClient(&mut router));
5133 let lingering = frontend.is_lingering();
5134 let mut frontend = Some(frontend);
5135 if matches!(result, MuxResult::CloseSession)
5136 && let Some(closed) = frontend.take()
5137 {
5138 let _ = shutdown_write(closed.socket(), false);
5139 drop(closed);
5140 }
5141
5142 peer.set_nonblocking(false)
5143 .expect("the client reads blocking");
5144 peer.set_read_timeout(Some(Duration::from_secs(5)))
5145 .expect("the client read is bounded");
5146 let mut received = 0;
5147 let mut buf = [0u8; 16 * 1024];
5148 let end = loop {
5149 match peer.read(&mut buf) {
5150 Ok(0) => break Ok(()),
5151 Ok(size) => received += size,
5152 Err(e) => break Err(e),
5153 }
5154 };
5155 assert_eq!(
5156 received, queued,
5157 "the client reads the whole response (end: {end:?})"
5158 );
5159 assert!(
5160 end.is_ok(),
5161 "the response ends with a clean EOF, got {end:?}"
5162 );
5163 assert!(
5164 lingering && matches!(result, MuxResult::Continue),
5165 "the frontend lingers instead of closing, got {result:?}"
5166 );
5167
5168 // The client closes: the drain meets its EOF and the session closes.
5169 let mut frontend = frontend.expect("a lingering frontend is kept");
5170 drop(peer);
5171 let mut after_client_close = MuxResult::Continue;
5172 for _ in 0..1000 {
5173 frontend.readiness_mut().event.insert(Ready::READABLE);
5174 after_client_close = frontend.readable(&mut context, EndpointClient(&mut router));
5175 if matches!(after_client_close, MuxResult::CloseSession) {
5176 break;
5177 }
5178 std::thread::sleep(Duration::from_millis(1));
5179 }
5180 assert!(
5181 matches!(after_client_close, MuxResult::CloseSession),
5182 "the client's EOF ends the drain, got {after_client_close:?}"
5183 );
5184 }
5185
5186 /// Start the lingering close of an H2 frontend that queued its final
5187 /// GOAWAY through `ConnectionH2::goaway` — `NO_ERROR` when `graceful`,
5188 /// `ENHANCE_YOUR_CALM` (a flood) otherwise — with
5189 /// a 10 s `request_timeout`; then have its client send `sent` bytes
5190 /// without closing, move the drain's clock `elapsed` past the linger's
5191 /// start, and drive the drain until it closes or 200 passes go by. Answers the linger the close started with, the instants that
5192 /// bracket it, and whether the drain closed.
5193 fn h2_linger_after_goaway(
5194 graceful: bool,
5195 sent: usize,
5196 elapsed: Duration,
5197 ) -> (shared::Linger, Instant, Instant, bool) {
5198 use std::io::Write;
5199
5200 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5201 let (socket, peer) = connected_socket();
5202 let mut h2 = h2::H2Shell::new(
5203 Ulid::generate(),
5204 socket,
5205 Position::Server,
5206 &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5207 H2FloodConfig::default(),
5208 H2ConnectionConfig::default(),
5209 Duration::from_secs(30),
5210 None,
5211 Duration::from_secs(10),
5212 None,
5213 Ready::WRITABLE | Ready::HUP | Ready::ERROR,
5214 )
5215 .expect("a pool with free buffers must yield an H2 connection");
5216 // The real entry point: `goaway` queues the final GOAWAY, records
5217 // its error code and asks for the write that flushes it.
5218 let queued = h2.core.goaway(if graceful {
5219 H2Error::NoError
5220 } else {
5221 H2Error::EnhanceYourCalm
5222 });
5223 assert!(
5224 matches!(queued, MuxResult::Continue),
5225 "the GOAWAY is queued"
5226 );
5227
5228 let mut frontend = Connection::H2(h2);
5229 let mut context = test_context(&pool);
5230 let mut router = Router::new(Duration::from_secs(30), Duration::from_secs(30));
5231 let before = Instant::now();
5232 context.now = before;
5233 let result = frontend.writable(&mut context, EndpointClient(&mut router));
5234 let after = Instant::now();
5235 assert!(
5236 frontend.is_lingering() && matches!(result, MuxResult::Continue),
5237 "premise: the frontend lingers after its final GOAWAY, got {result:?}"
5238 );
5239 let Connection::H2(h2) = &frontend else {
5240 unreachable!("built as H2")
5241 };
5242 let linger = h2.core.linger;
5243
5244 // The client keeps sending and never closes; it is kept open until
5245 // the drain has decided.
5246 peer.set_nonblocking(false)
5247 .expect("the client writes blocking");
5248 let mut writer = peer.try_clone().expect("the client socket clones");
5249 let sender = std::thread::spawn(move || {
5250 let _ = writer.write_all(&vec![0u8; sent]);
5251 });
5252 context.now = after + elapsed;
5253 let mut closed = false;
5254 for _ in 0..200 {
5255 frontend.readiness_mut().event.insert(Ready::READABLE);
5256 if matches!(
5257 frontend.readable(&mut context, EndpointClient(&mut router)),
5258 MuxResult::CloseSession
5259 ) {
5260 closed = true;
5261 break;
5262 }
5263 std::thread::sleep(Duration::from_millis(1));
5264 }
5265 drop(frontend);
5266 let _ = sender.join();
5267 drop(peer);
5268 (linger, before, after, closed)
5269 }
5270
5271 /// The lingering close after a GOAWAY carrying an error code exists only
5272 /// to deliver that GOAWAY (sozu-proxy/sozu#1861): its deadline is
5273 /// `shared::ERROR_GOAWAY_LINGER_TIMEOUT` from its start and its byte
5274 /// budget `shared::ERROR_GOAWAY_LINGER_MAX_BYTES`, so a client that keeps
5275 /// sending is cut off once that much is read. A graceful GOAWAY keeps the
5276 /// listener's `request_timeout` and `shared::LINGER_MAX_BYTES`, and the
5277 /// same traffic does not close it. With no traffic at all, the short
5278 /// deadline closes the error linger and leaves the graceful one open.
5279 ///
5280 /// TO SEE THIS RED: in `ConnectionH2::linger_budget` (`h2.rs`), return
5281 /// the graceful pair for every GOAWAY.
5282 #[test]
5283 fn an_error_goaway_lingers_under_the_short_budget_and_a_graceful_one_keeps_its_own() {
5284 let sent = shared::ERROR_GOAWAY_LINGER_MAX_BYTES + 64 * 1024;
5285
5286 let (linger, before, after, closed) = h2_linger_after_goaway(false, sent, Duration::ZERO);
5287 let shared::Linger::Draining {
5288 deadline,
5289 remaining,
5290 } = linger
5291 else {
5292 panic!("an error GOAWAY drains at once on a plain socket, got {linger:?}");
5293 };
5294 assert_eq!(remaining, shared::ERROR_GOAWAY_LINGER_MAX_BYTES);
5295 assert!(
5296 deadline >= before + shared::ERROR_GOAWAY_LINGER_TIMEOUT
5297 && deadline <= after + shared::ERROR_GOAWAY_LINGER_TIMEOUT,
5298 "the error linger's deadline is the short one, {:?} past its start",
5299 deadline.saturating_duration_since(before)
5300 );
5301 assert!(
5302 closed,
5303 "the byte budget ends the error linger while the client still sends"
5304 );
5305
5306 let (linger, before, after, closed) = h2_linger_after_goaway(true, sent, Duration::ZERO);
5307 let shared::Linger::Draining {
5308 deadline,
5309 remaining,
5310 } = linger
5311 else {
5312 panic!("a graceful GOAWAY drains at once on a plain socket, got {linger:?}");
5313 };
5314 assert_eq!(remaining, shared::LINGER_MAX_BYTES);
5315 assert!(
5316 deadline >= before + Duration::from_secs(10)
5317 && deadline <= after + Duration::from_secs(10),
5318 "the graceful linger keeps request_timeout, {:?} past its start",
5319 deadline.saturating_duration_since(before)
5320 );
5321 assert!(
5322 !closed,
5323 "a graceful linger keeps draining a client that is still sending"
5324 );
5325
5326 let (.., closed) = h2_linger_after_goaway(false, 0, shared::ERROR_GOAWAY_LINGER_TIMEOUT);
5327 assert!(closed, "the short deadline ends a silent error linger");
5328 let (.., closed) = h2_linger_after_goaway(true, 0, shared::ERROR_GOAWAY_LINGER_TIMEOUT);
5329 assert!(!closed, "the short deadline does not end a graceful linger");
5330 }
5331
5332 /// Run one `Mux::ready` pass over an H2 frontend that queued its final
5333 /// GOAWAY (`NO_ERROR` when `graceful`, `ENHANCE_YOUR_CALM` otherwise)
5334 /// while one stream is still linked to a backend. Answers the `Mux`
5335 /// after the pass and the backend's peer socket.
5336 fn mux_after_final_goaway_with_a_linked_stream(
5337 pool: &Rc<RefCell<Pool>>,
5338 graceful: bool,
5339 ) -> (
5340 Mux<mio::net::TcpStream, test_support::TestListener>,
5341 std::net::TcpStream,
5342 std::net::TcpStream,
5343 ) {
5344 let (socket, front_peer) = connected_socket();
5345 let mut h2 = h2::H2Shell::new(
5346 Ulid::generate(),
5347 socket,
5348 Position::Server,
5349 &mut PoolBufferSource::new(Rc::downgrade(pool)),
5350 H2FloodConfig::default(),
5351 H2ConnectionConfig::default(),
5352 Duration::from_secs(30),
5353 None,
5354 Duration::from_secs(10),
5355 None,
5356 Ready::WRITABLE | Ready::HUP | Ready::ERROR,
5357 )
5358 .expect("a pool with free buffers must yield an H2 connection");
5359 let queued = h2.core.goaway(if graceful {
5360 H2Error::NoError
5361 } else {
5362 H2Error::EnhanceYourCalm
5363 });
5364 assert!(
5365 matches!(queued, MuxResult::Continue),
5366 "the GOAWAY is queued"
5367 );
5368
5369 let mut context = test_context(pool);
5370 context
5371 .create_stream(Ulid::generate(), 1 << 16)
5372 .expect("test context must create a stream");
5373 let mut mux = Mux {
5374 configured_frontend_timeout: Duration::from_secs(10),
5375 frontend_token: Token(0),
5376 frontend: Connection::H2(h2),
5377 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5378 context,
5379 session_ulid: Ulid::generate(),
5380 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(10))),
5381 backend_registry: BackendRegistry::default(),
5382 backends: Rc::default(),
5383 };
5384 let (backend, back_peer) = test_backend_connection(&mut mux, Duration::from_secs(30));
5385 mux.router.backends.insert(Token(1), backend);
5386 mux.context.link_stream(0, Token(1));
5387 mux.timeouts.backends.get_or_insert_with(Token(1), || {
5388 TimeoutContainer::new_empty(Duration::from_secs(30))
5389 });
5390
5391 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
5392 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
5393 let mut metrics = SessionMetrics::new(None);
5394 let result = mux.ready(session, proxy, &mut metrics);
5395 assert!(
5396 matches!(result, SessionResult::Continue) && mux.frontend.is_lingering(),
5397 "premise: the pass that flushes the final GOAWAY starts the linger, got {result:?}"
5398 );
5399 (mux, front_peer, back_peer)
5400 }
5401
5402 /// A final GOAWAY carrying an error code can leave streams open, and none
5403 /// of them will be answered: the pass that starts its linger logs and
5404 /// recycles them and shuts down, accounts for and drops their backend
5405 /// connections, instead of holding them until the linger ends. A graceful
5406 /// linger is left as it was.
5407 ///
5408 /// TO SEE THIS RED: remove both calls to
5409 /// `Mux::release_streams_and_backends_for_error_linger` from
5410 /// `Mux::ready_inner`.
5411 #[test]
5412 fn an_error_goaway_linger_releases_its_streams_and_backends_when_it_starts() {
5413 use std::io::Read;
5414
5415 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
5416 let (mux, _front_peer, mut back_peer) =
5417 mux_after_final_goaway_with_a_linked_stream(&pool, false);
5418 assert!(
5419 mux.router.backends.is_empty(),
5420 "the backend connection is released when the error linger starts"
5421 );
5422 assert!(
5423 mux.context.backend_streams.is_empty() && mux.timeouts.backends.is_empty(),
5424 "no reverse-index entry or backend timer survives the release"
5425 );
5426 assert_eq!(
5427 mux.context.streams[0].state,
5428 StreamState::Recycle,
5429 "the stream that will get no answer is logged and recycled"
5430 );
5431 back_peer
5432 .set_nonblocking(false)
5433 .expect("the backend peer reads blocking");
5434 back_peer
5435 .set_read_timeout(Some(Duration::from_secs(5)))
5436 .expect("the backend peer read is bounded");
5437 let mut buf = [0u8; 64];
5438 assert_eq!(
5439 back_peer.read(&mut buf).ok(),
5440 Some(0),
5441 "the backend socket is closed: its peer reads EOF"
5442 );
5443
5444 let (mux, _front_peer, _back_peer) =
5445 mux_after_final_goaway_with_a_linked_stream(&pool, true);
5446 assert_eq!(
5447 mux.router.backends.len(),
5448 1,
5449 "a graceful linger keeps its backend, as before"
5450 );
5451 assert_eq!(mux.context.streams[0].state, StreamState::Linked(Token(1)));
5452 }
5453
5454 /// A TLS frontend whose final GOAWAY carries an error code and leaves a
5455 /// stream open sends that GOAWAY without `close_notify` behind it
5456 /// (`ConnectionH2::output_flush_closes_connection` wants an empty stream
5457 /// table). The close its next writable pass decides queues the alert and
5458 /// turns into a pending linger (`H2Shell::linger_instead_of_closing`);
5459 /// when the kernel then refuses the alert's record, nothing re-arms the
5460 /// write and `Mux::ready_inner`'s loop ends with the linger still
5461 /// pending. The release must already have run by then: a stream queued
5462 /// in `pending_links` by the same pass is never routed nor dialed.
5463 ///
5464 /// TO SEE THIS RED: gate `Connection::lingers_after_error_goaway` on
5465 /// `is_lingering()` alone, as before. The `pending_links` pass then
5466 /// reaches the routing view for the stream and panics in
5467 /// `NoRoutingProxy` ("pending_links routed a stream of an error-GOAWAY
5468 /// linger").
5469 #[test]
5470 fn an_error_goaway_linger_pending_on_close_notify_releases_before_pending_links() {
5471 use std::io::Write;
5472
5473 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
5474 let (mut h2, _peer, _client) = h2::tests::rustls_h2_connection(&pool, H2State::Header);
5475 let mut context = test_context(&pool);
5476 let stream_id = context
5477 .create_stream(Ulid::generate(), 1 << 16)
5478 .expect("test context must create a stream");
5479 // A request the same read pass parsed whole and queued for a backend.
5480 context.streams[stream_id].state = StreamState::Link;
5481 context.pending_links.push_back(stream_id);
5482 h2.core.__test_insert_wire_mapping_only(1, stream_id);
5483
5484 assert!(
5485 matches!(
5486 h2.core.goaway(H2Error::EnhanceYourCalm),
5487 MuxResult::Continue
5488 ),
5489 "the GOAWAY is queued"
5490 );
5491 h2.flush_output_buffer();
5492 assert!(
5493 h2.core.output_pending().is_empty() && !h2.socket.socket_wants_write(),
5494 "premise: the GOAWAY reached the kernel, with no close_notify behind it"
5495 );
5496 // The client stops reading: fill the frontend's send queue so the
5497 // kernel refuses the close_notify record.
5498 let junk = [0u8; 4096];
5499 let mut filled = false;
5500 for _ in 0..4096 {
5501 match h2.socket.stream.write(&junk) {
5502 Ok(_) => {}
5503 Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
5504 filled = true;
5505 break;
5506 }
5507 Err(e) => panic!("filling the frontend send queue failed: {e:?}"),
5508 }
5509 }
5510 assert!(filled, "premise: the frontend send queue is full");
5511 h2.core.readiness.event.insert(Ready::WRITABLE);
5512
5513 let mut mux = Mux {
5514 configured_frontend_timeout: Duration::from_secs(10),
5515 frontend_token: Token(0),
5516 frontend: Connection::H2(h2),
5517 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5518 context,
5519 session_ulid: Ulid::generate(),
5520 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(10))),
5521 backend_registry: BackendRegistry::default(),
5522 backends: Rc::default(),
5523 };
5524 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
5525 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(NoRoutingProxy));
5526 let mut metrics = SessionMetrics::new(None);
5527 let result = mux.ready(session, proxy, &mut metrics);
5528
5529 let Connection::H2(h2) = &mux.frontend else {
5530 unreachable!("the frontend was built as H2")
5531 };
5532 assert!(
5533 matches!(result, SessionResult::Continue)
5534 && matches!(h2.core.linger, shared::Linger::Pending { .. }),
5535 "premise: the pass ends with the linger pending on close_notify, got \
5536 {result:?} with {:?}",
5537 h2.core.linger
5538 );
5539 assert_eq!(
5540 mux.context.streams[stream_id].state,
5541 StreamState::Recycle,
5542 "the stream that will get no answer is released, not linked"
5543 );
5544 assert!(
5545 mux.context.pending_links.is_empty() && mux.router.backends.is_empty(),
5546 "nothing is left to link and no backend was dialed"
5547 );
5548 }
5549
5550 /// A soft stop must still close a lingering H2 session once the H2
5551 /// listener's graceful-shutdown budget has elapsed, even when the linger
5552 /// began before the budget ran out. The lingering `readable`/`writable`
5553 /// passes must adopt the pass's clock: `graceful_shutdown_deadline_elapsed`
5554 /// reads the connection's own snapshot, and one frozen at the linger's
5555 /// start would only ever see the budget as it stood then.
5556 ///
5557 /// TO SEE THIS RED: drop the `self.core.adopt_now(context.now)` from the
5558 /// lingering early returns of `H2Shell::readable` and `H2Shell::writable`.
5559 /// `shutting_down` then keeps the session.
5560 #[test]
5561 fn a_soft_stop_closes_a_lingering_h2_session_once_its_budget_elapses() {
5562 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5563 let (socket, _peer) = connected_socket();
5564
5565 let mut h2 = h2::H2Shell::new(
5566 Ulid::generate(),
5567 socket,
5568 Position::Server,
5569 &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5570 H2FloodConfig::default(),
5571 H2ConnectionConfig::default(),
5572 Duration::from_secs(30),
5573 Some(Duration::from_secs(1)),
5574 Duration::from_secs(30),
5575 None,
5576 Ready::READABLE | Ready::HUP | Ready::ERROR,
5577 )
5578 .expect("a pool with free buffers must yield an H2 connection");
5579
5580 // A 1 s budget armed 3 s ago; the linger began 100 ms into it, with a
5581 // linger deadline far away.
5582 let armed_at = Instant::now() - Duration::from_secs(3);
5583 let linger_started = armed_at + Duration::from_millis(100);
5584 h2.core.state = H2State::Error;
5585 h2.core.drain.__test_arm_draining(armed_at);
5586 h2.core.adopt_now(linger_started);
5587 h2.core.linger = shared::Linger::Draining {
5588 deadline: linger_started + Duration::from_secs(60),
5589 remaining: shared::LINGER_MAX_BYTES,
5590 };
5591 let frontend = Connection::H2(h2);
5592 assert!(frontend.is_lingering(), "premise: the session lingers");
5593
5594 let mut mux = Mux {
5595 configured_frontend_timeout: Duration::from_secs(30),
5596 frontend_token: Token(0),
5597 frontend,
5598 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5599 context: test_context(&pool),
5600 session_ulid: Ulid::generate(),
5601 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
5602 backend_registry: BackendRegistry::default(),
5603 backends: Rc::default(),
5604 };
5605
5606 assert!(
5607 mux.shutting_down(),
5608 "the elapsed graceful-shutdown budget closes the lingering session"
5609 );
5610 }
5611
5612 /// `Mux::shutting_down` runs outside `ready()`, driven by
5613 /// `shut_down_sessions()`. It is therefore its own clock-sampling point,
5614 /// and that sample is load-bearing: it refreshes `context.now`, which
5615 /// `drive_frontend_shutdown_io` -> `readable()` mirrors into
5616 /// `ConnectionH2::now`, which the forced-close check then reads.
5617 ///
5618 /// A silent draining session gets no read/write events, so nothing else
5619 /// refreshes the snapshot. Without this sample the connection would carry
5620 /// the snapshot of its last `ready()` pass forever and the
5621 /// graceful-shutdown budget would never expire — the session would hang
5622 /// past the operator-configured SLA, which is the exact failure the
5623 /// forced-close deadline exists to prevent.
5624 ///
5625 /// The frontend is already draining, so `graceful_goaway` is NOT called on
5626 /// this path — that is what makes the test discriminating, because the
5627 /// `now` parameter of `graceful_goaway` cannot cover it.
5628 ///
5629 /// To SEE THIS RED: delete `self.context.now = now;` from
5630 /// `Mux::shutting_down` (keeping `let now` so `graceful_goaway(now)` still
5631 /// compiles). `readable()` then mirrors the frozen snapshot, the budget
5632 /// reads as 0s elapsed against a 5s deadline, and the final assertion fails
5633 /// with `assertion failed: mux.shutting_down()`.
5634 /// One `shutting_down()` pass on an idle draining H2 session makes exactly
5635 /// one `writable()` call: the forced pass, with nothing queued after it.
5636 ///
5637 /// `shut_down_sessions()` calls `shutting_down()` on every draining session
5638 /// at each 100 ms shutdown tick, so the work in one pass is multiplied by
5639 /// both. The loop in `Mux::drive_frontend_shutdown_io` used to wait for the
5640 /// WRITABLE event bit to clear, a bit that its own `signal_pending_write`
5641 /// had just set and that `writable()` clears only on `WouldBlock`, so an
5642 /// idle session with a writable socket ran all `MAX_LOOP_ITERATIONS`.
5643 /// Measured on the e2e drain test in a debug build: 10 000 calls and about
5644 /// 29 ms of CPU per pass, uncontended.
5645 ///
5646 /// To SEE THIS RED: restore the old exit condition,
5647 /// `!has_pending_write() && !readiness().event.is_writable()`; the count
5648 /// becomes `MAX_LOOP_ITERATIONS`.
5649 #[test]
5650 fn an_idle_draining_h2_session_makes_one_writable_pass_per_shutdown_tick() {
5651 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5652 let (socket, _peer) = connected_socket();
5653
5654 let mut h2 = h2::H2Shell::new(
5655 Ulid::generate(),
5656 socket,
5657 Position::Server,
5658 &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5659 H2FloodConfig::default(),
5660 H2ConnectionConfig::default(),
5661 Duration::from_secs(30),
5662 Some(Duration::from_secs(3600)),
5663 Duration::from_secs(30),
5664 None,
5665 Ready::READABLE | Ready::HUP | Ready::ERROR,
5666 )
5667 .expect("a pool with free buffers must yield an H2 connection");
5668
5669 // Already draining, budget far from elapsed: the pass must neither
5670 // send a GOAWAY nor force-close, only drive the frontend I/O.
5671 let armed_at = Instant::now();
5672 h2.core.state = H2State::Header;
5673 h2.core.drain.__test_arm_draining(armed_at);
5674 let mut frontend = Connection::H2(h2);
5675
5676 let mut context = test_context(&pool);
5677 context.now = armed_at;
5678 // One live stream, so the session keeps draining instead of closing.
5679 let stream_id = context
5680 .create_stream(Ulid::generate(), 1 << 16)
5681 .expect("test context must create a stream");
5682 context.streams[stream_id].state = StreamState::Linked(Token(1));
5683 let Connection::H2(h2) = &mut frontend else {
5684 unreachable!("frontend was built as H2")
5685 };
5686 h2.core.__test_insert_wire_mapping_only(1, stream_id);
5687
5688 let mut mux = Mux {
5689 configured_frontend_timeout: Duration::from_secs(30),
5690 frontend_token: Token(0),
5691 frontend,
5692 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5693 context,
5694 session_ulid: Ulid::generate(),
5695 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
5696 backend_registry: BackendRegistry::default(),
5697 backends: Rc::default(),
5698 };
5699 assert!(
5700 !mux.frontend.has_pending_write(),
5701 "precondition: the idle draining session has nothing queued"
5702 );
5703
5704 SHUTDOWN_WRITABLE_PASSES.with(|passes| passes.set(0));
5705 assert!(
5706 !mux.shutting_down(),
5707 "a draining session with a live stream and budget left must stay open"
5708 );
5709 let passes = SHUTDOWN_WRITABLE_PASSES.with(|passes| passes.get());
5710 assert_eq!(
5711 passes, 1,
5712 "one shutdown pass on an idle draining H2 session must make exactly the \
5713 one forced writable() call, not spin until MAX_LOOP_ITERATIONS \
5714 ({MAX_LOOP_ITERATIONS}); got {passes}"
5715 );
5716 assert!(
5717 !mux.frontend.has_pending_write(),
5718 "the pass must leave nothing queued behind"
5719 );
5720 }
5721
5722 #[test]
5723 fn shutting_down_refreshes_the_snapshot_so_the_drain_budget_expires() {
5724 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5725 let (socket, _peer) = connected_socket();
5726 let deadline = Duration::from_secs(5);
5727
5728 let mut h2 = h2::H2Shell::new(
5729 Ulid::generate(),
5730 socket,
5731 Position::Server,
5732 &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5733 H2FloodConfig::default(),
5734 H2ConnectionConfig::default(),
5735 Duration::from_secs(30),
5736 Some(deadline),
5737 Duration::from_secs(30),
5738 None,
5739 Ready::READABLE | Ready::HUP | Ready::ERROR,
5740 )
5741 .expect("a pool with free buffers must yield an H2 connection");
5742
5743 // A connection that already sent its initial GOAWAY and armed the
5744 // budget well over the deadline ago. `shutting_down` takes the
5745 // already-draining branch, which never calls `graceful_goaway`.
5746 let armed_at = Instant::now() - Duration::from_secs(60);
5747 h2.core.state = H2State::Header;
5748 h2.core.drain.__test_arm_draining(armed_at);
5749 let mut frontend = Connection::H2(h2);
5750
5751 let mut context = test_context(&pool);
5752 // Freeze the snapshot at the instant the budget was armed, as a
5753 // session that has seen no event since then would carry it.
5754 context.now = armed_at;
5755 // One live stream, so `can_stop` stays false and the forced-close
5756 // deadline is the only path that can return `true`.
5757 let stream_id = context
5758 .create_stream(Ulid::generate(), 1 << 16)
5759 .expect("test context must create a stream");
5760 context.streams[stream_id].state = StreamState::Linked(Token(1));
5761 // Give the connection a live WIRE stream too: with an empty
5762 // `ConnectionH2::streams`, `finalize_write` sends the FINAL GOAWAY and
5763 // force-disconnects, so `drive_frontend_shutdown_io` would return true
5764 // and short-circuit `shutting_down` before the deadline check. The
5765 // per-stream activity maps are deliberately left empty so
5766 // `cancel_timed_out_streams` early-returns and cannot queue an RST.
5767 let Connection::H2(h2) = &mut frontend else {
5768 unreachable!("frontend was built as H2")
5769 };
5770 h2.core.__test_insert_wire_mapping_only(1, stream_id);
5771
5772 let mut mux = Mux {
5773 configured_frontend_timeout: Duration::from_secs(30),
5774 frontend_token: Token(0),
5775 frontend,
5776 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5777 context,
5778 session_ulid: Ulid::generate(),
5779 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
5780 backend_registry: BackendRegistry::default(),
5781 backends: Rc::default(),
5782 };
5783
5784 assert!(
5785 mux.frontend.is_draining(),
5786 "precondition: the frontend must already be draining, so this test \
5787 exercises the branch that never reaches graceful_goaway"
5788 );
5789
5790 assert!(
5791 mux.shutting_down(),
5792 "shutting_down must refresh the clock snapshot itself, so a silent \
5793 draining session's forced-close budget still expires"
5794 );
5795 }
5796
5797 /// sozu-proxy/sozu#1647: a CONTINUATION that `drive_frontend_shutdown_io`
5798 /// reads completes the block outside `Mux::ready`, so its stream is left
5799 /// `StreamState::Link` until the next `ready` pass links it. A draining
5800 /// H2 session must stay open for it, within budget, rather than drop a
5801 /// request the advisory GOAWAY invited.
5802 #[test]
5803 fn a_draining_h2_session_waits_for_a_stream_awaiting_its_link() {
5804 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5805 let (socket, _peer) = connected_socket();
5806
5807 let mut h2 = h2::H2Shell::new(
5808 Ulid::generate(),
5809 socket,
5810 Position::Server,
5811 &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5812 H2FloodConfig::default(),
5813 H2ConnectionConfig::default(),
5814 Duration::from_secs(30),
5815 Some(Duration::from_secs(60)),
5816 Duration::from_secs(30),
5817 None,
5818 Ready::READABLE | Ready::HUP | Ready::ERROR,
5819 )
5820 .expect("a pool with free buffers must yield an H2 connection");
5821 // Already draining, budget armed now: nothing but the stream below
5822 // can keep the session open.
5823 h2.core.state = H2State::Header;
5824 h2.core.drain.__test_arm_draining(Instant::now());
5825 let mut frontend = Connection::H2(h2);
5826
5827 let mut context = test_context(&pool);
5828 let stream_id = context
5829 .create_stream(Ulid::generate(), 1 << 16)
5830 .expect("test context must create a stream");
5831 context.streams[stream_id].state = StreamState::Link;
5832 // A live wire stream, so `finalize_write` does not send the final
5833 // GOAWAY and close through `drive_frontend_shutdown_io`.
5834 let Connection::H2(h2) = &mut frontend else {
5835 unreachable!("frontend was built as H2")
5836 };
5837 h2.core.__test_insert_wire_mapping_only(1, stream_id);
5838
5839 let mut mux = Mux {
5840 configured_frontend_timeout: Duration::from_secs(30),
5841 frontend_token: Token(0),
5842 frontend,
5843 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5844 context,
5845 session_ulid: Ulid::generate(),
5846 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
5847 backend_registry: BackendRegistry::default(),
5848 backends: Rc::default(),
5849 };
5850 assert!(
5851 !mux.frontend.graceful_shutdown_deadline_elapsed(),
5852 "precondition: the forced-close budget has not elapsed"
5853 );
5854
5855 assert!(
5856 !mux.shutting_down(),
5857 "a draining H2 session must stay open for a stream awaiting its link"
5858 );
5859 assert_eq!(
5860 mux.context.streams[stream_id].state,
5861 StreamState::Link,
5862 "the shutdown pass must leave the stream for `Mux::ready` to link"
5863 );
5864 }
5865
5866 #[test]
5867 fn a_short_read_after_hup_keeps_readable_to_read_the_eof() {
5868 let mut readiness = Readiness {
5869 event: Ready::READABLE | Ready::HUP,
5870 interest: Ready::READABLE,
5871 };
5872
5873 let should_yield =
5874 update_readiness_after_read(17, SocketResult::WouldBlock, &mut readiness);
5875
5876 assert!(!should_yield);
5877 assert!(
5878 readiness.event.is_readable(),
5879 "no edge will announce the queued EOF: READABLE must stay"
5880 );
5881 }
5882
5883 /// The case the HUP exception exists for, on a real loopback socket
5884 /// registered edge-triggered with mio: the data and the FIN arrive before
5885 /// the worker polls, so ONE event carries both `readable` and
5886 /// `read_closed`. The short read returns the data and stops before the
5887 /// EOF, and no later event will ever announce it. READABLE must survive
5888 /// `update_readiness_after_read` so the next pass reads `(0, Closed)`.
5889 ///
5890 /// To SEE THIS RED: remove the `is_hup()` exception from
5891 /// `update_readiness_after_read`; READABLE is dropped after the short read
5892 /// and the queued EOF is never read.
5893 #[test]
5894 fn data_and_fin_in_one_edge_keep_readable_until_the_eof_is_read() {
5895 use std::io::Write as _;
5896
5897 use crate::socket::SocketHandler as _;
5898
5899 let token = mio::Token(9);
5900 let mut poll = mio::Poll::new().expect("a poll instance must open");
5901 let mut events = mio::Events::with_capacity(8);
5902 let listener = std::net::TcpListener::bind("127.0.0.1:0")
5903 .expect("test listener must bind to a loopback port");
5904 let mut client = std::net::TcpStream::connect(
5905 listener
5906 .local_addr()
5907 .expect("test listener must report its local address"),
5908 )
5909 .expect("loopback connect must complete");
5910 let (accepted, _) = listener.accept().expect("the connection must be accepted");
5911 accepted
5912 .set_nonblocking(true)
5913 .expect("mio requires a nonblocking stream");
5914 let mut server = mio::net::TcpStream::from_std(accepted);
5915
5916 // Data and FIN are both queued before the socket is registered, so
5917 // the first poll reports them in a single edge.
5918 client.write_all(b"DATA").expect("the client must write");
5919 client
5920 .shutdown(std::net::Shutdown::Write)
5921 .expect("the client must half-close");
5922 poll.registry()
5923 .register(&mut server, token, mio::Interest::READABLE)
5924 .expect("the stream must register");
5925 poll.poll(&mut events, Some(std::time::Duration::from_secs(5)))
5926 .expect("poll must succeed");
5927 let event = events
5928 .iter()
5929 .find(|event| event.token() == token)
5930 .expect("the arrival must raise an event");
5931 assert!(
5932 event.is_readable() && event.is_read_closed(),
5933 "premise: data and FIN must share one edge, got {event:?}"
5934 );
5935 let mut readiness = Readiness {
5936 event: Ready::from(event),
5937 interest: Ready::READABLE,
5938 };
5939
5940 let mut buf = vec![0u8; 16 * 1024];
5941 let (size, status) = server.socket_read(&mut buf);
5942 assert_eq!((size, status), (4, SocketResult::WouldBlock));
5943 assert!(!update_readiness_after_read(size, status, &mut readiness));
5944 assert!(
5945 readiness.event.is_readable(),
5946 "HUP was seen: READABLE must stay so the EOF is read"
5947 );
5948
5949 poll.poll(&mut events, Some(std::time::Duration::from_millis(300)))
5950 .expect("poll must succeed");
5951 assert!(
5952 events.iter().all(|event| event.token() != token),
5953 "edge-triggered: no further event announces the queued EOF"
5954 );
5955
5956 let (size, status) = server.socket_read(&mut buf);
5957 assert_eq!((size, status), (0, SocketResult::Closed));
5958 assert!(update_readiness_after_read(size, status, &mut readiness));
5959 assert!(!readiness.event.is_readable());
5960 }
5961
5962 #[test]
5963 fn a_short_read_without_hup_drops_readable() {
5964 let mut readiness = Readiness {
5965 event: Ready::READABLE,
5966 interest: Ready::READABLE,
5967 };
5968
5969 let should_yield =
5970 update_readiness_after_read(17, SocketResult::WouldBlock, &mut readiness);
5971
5972 assert!(!should_yield);
5973 assert!(!readiness.event.is_readable(), "the next edge re-arms it");
5974 }
5975
5976 #[test]
5977 fn an_empty_read_after_hup_still_drops_readable() {
5978 let mut readiness = Readiness {
5979 event: Ready::READABLE | Ready::HUP,
5980 interest: Ready::READABLE,
5981 };
5982
5983 let should_yield = update_readiness_after_read(0, SocketResult::WouldBlock, &mut readiness);
5984
5985 assert!(should_yield);
5986 assert!(
5987 !readiness.event.is_readable(),
5988 "the exception must not spin"
5989 );
5990 }
5991
5992 #[test]
5993 fn update_readiness_after_read_closed_keeps_writable() {
5994 let mut readiness = Readiness {
5995 event: Ready::READABLE | Ready::WRITABLE | Ready::HUP,
5996 interest: Ready::READABLE | Ready::WRITABLE | Ready::HUP,
5997 };
5998
5999 let should_yield = update_readiness_after_read(17, SocketResult::Closed, &mut readiness);
6000
6001 assert!(!should_yield);
6002 assert!(!readiness.event.is_readable());
6003 assert!(readiness.event.is_writable());
6004 assert!(readiness.event.is_hup());
6005 }
6006
6007 /// Build a minimal H1 frontend `Mux` with one `Idle` stream. The returned
6008 /// peer socket must be kept alive for the duration of the test, otherwise
6009 /// the loopback connection is torn down and `readable()` sees a forced
6010 /// disconnect.
6011 ///
6012 /// An H1 frontend is deliberate: the `StreamState::Idle` arm of
6013 /// `Mux::timeout` writes a 408 and stamps
6014 /// `stream.context.access_log_message = Some("client_timeout")`, which is
6015 /// the crispest available witness that the timeout BODY ran. The H2 `Idle`
6016 /// arm is silently ignored and would witness nothing.
6017 fn h1_mux_with_idle_stream(
6018 pool: &Rc<RefCell<Pool>>,
6019 frontend_timeout: Duration,
6020 ) -> (
6021 Mux<mio::net::TcpStream, test_support::TestListener>,
6022 std::net::TcpStream,
6023 ) {
6024 let (socket, peer) = connected_socket();
6025 let frontend = Connection::new_h1_server(Ulid::generate(), socket, frontend_timeout);
6026 let mut context = test_context(pool);
6027 context
6028 .create_stream(Ulid::generate(), 1 << 16)
6029 .expect("test context must create a stream");
6030 let mux = Mux {
6031 configured_frontend_timeout: frontend_timeout,
6032 frontend_token: Token(0),
6033 frontend,
6034 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
6035 context,
6036 session_ulid: Ulid::generate(),
6037 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(frontend_timeout)),
6038 backend_registry: BackendRegistry::default(),
6039 backends: Rc::default(),
6040 };
6041 (mux, peer)
6042 }
6043
6044 /// Stage the exact shape of sozu-proxy/sozu#1442: a request fully written
6045 /// onto a REUSED keep-alive upstream that then closed without answering.
6046 ///
6047 /// `front.consumed` is what `end_stream_decision` reads to mean "the
6048 /// request left the front buffer"; the armed `retry_buffer` is what
6049 /// `ConnectionH1::writable` fills, and only ever on a connection whose
6050 /// `reused_from_pool` is set.
6051 fn stale_pooled_upstream_stream(
6052 pool: &Rc<RefCell<Pool>>,
6053 method: Method,
6054 ) -> (
6055 Mux<mio::net::TcpStream, test_support::TestListener>,
6056 std::net::TcpStream,
6057 ) {
6058 let (mut mux, peer) = h1_mux_with_idle_stream(pool, Duration::from_secs(60));
6059 let stream = &mut mux.context.streams[0];
6060 // The capture is armed while the method is still `Get`, and the method
6061 // under test is set afterwards. Since sozu-proxy/sozu#1450
6062 // `arm_upstream_replay` refuses to capture a non-idempotent request at
6063 // all, so arming under `method` would leave the POST and `PATCH` cases
6064 // below with no capture — and they would then be observing the ARM
6065 // guard rather than the conjunct they exist to pin. Staging it this way
6066 // keeps `Stream::can_replay_on_fresh_upstream`'s own idempotence
6067 // conjunct as the thing under test: dropping that conjunct still
6068 // reddens them. Production cannot reach this state any more, because a
6069 // request's method does not change after its headers are parsed, which
6070 // is exactly why the conjunct is now defense in depth and why these
6071 // tests are what keep it honest.
6072 stream.context.method = Some(Method::Get);
6073 stream.arm_upstream_replay();
6074 stream
6075 .retry_buffer
6076 .as_mut()
6077 .expect("arm_upstream_replay must install a buffer")
6078 .extend_from_slice(b"GET /api HTTP/1.1\r\nHost: localhost\r\n\r\n");
6079 stream.context.method = Some(method);
6080 stream.front.consumed = true;
6081 // The upstream produced nothing: not in body phase, nothing forwarded,
6082 // nothing buffered.
6083 assert!(
6084 !stream.back.is_main_phase() && !stream.back.consumed && stream.back.storage.is_empty(),
6085 "precondition: the upstream must have produced no response at all"
6086 );
6087 (mux, peer)
6088 }
6089
6090 /// Concatenate every byte `ConnectionH1::writable` would hand the socket,
6091 /// in queue order. Mirrors its own `kawa.out` walk.
6092 fn queued_request_bytes(front: &GenericHttpStream) -> Vec<u8> {
6093 let buffer = front.storage.buffer();
6094 front
6095 .out
6096 .iter()
6097 .map(|block| match block {
6098 kawa::OutBlock::Store(store) => store.data(buffer).to_vec(),
6099 kawa::OutBlock::Delimiter => Vec::new(),
6100 })
6101 .collect::<Vec<_>>()
6102 .concat()
6103 }
6104
6105 /// Stage the partial-write shape of sozu-proxy/sozu#1442 on the front
6106 /// kawa: a real request, parsed and serialized exactly as
6107 /// `ConnectionH1::writable` serializes it, of which the socket accepted
6108 /// everything but the last `unwritten_tail` bytes.
6109 ///
6110 /// `kawa::Kawa::consume` pushes the partially consumed store back to the
6111 /// FRONT of `out` (kawa-0.7.1 `storage/repr.rs`), so the refused tail
6112 /// stays queued ahead of anything appended afterwards. That is what makes
6113 /// the queue position of the replay load-bearing.
6114 ///
6115 /// Returns the complete serialization, which a correct replay reproduces
6116 /// byte-for-byte.
6117 fn stage_partially_written_pooled_request(
6118 stream: &mut Stream,
6119 unwritten_tail: usize,
6120 ) -> Vec<u8> {
6121 let request: &[u8] = b"GET /api HTTP/1.1\r\nHost: localhost\r\nX-Tail: unwritten\r\n\r\n";
6122 stream.front.storage.space()[..request.len()].copy_from_slice(request);
6123 stream.front.storage.fill(request.len());
6124 kawa::h1::parse(&mut stream.front, &mut stream.context);
6125 assert!(
6126 stream.front.is_main_phase(),
6127 "the staged request must parse"
6128 );
6129 stream.front.prepare(&mut kawa::h1::BlockConverter);
6130 assert!(
6131 stream.front.blocks.is_empty(),
6132 "prepare must drain blocks, as it has by the time writable writes"
6133 );
6134
6135 let serialized = queued_request_bytes(&stream.front);
6136 let accepted = serialized
6137 .len()
6138 .checked_sub(unwritten_tail)
6139 .expect("the unwritten tail must fit inside the serialized request");
6140 stream.context.method = Some(Method::Get);
6141 // Through `arm_upstream_replay` rather than by assigning the field:
6142 // the capture owns a charge against `MAX_ARMED_REPLAY_CAPTURES`, so a
6143 // hand-built one would release a charge it never took.
6144 stream.arm_upstream_replay();
6145 stream
6146 .retry_buffer
6147 .as_mut()
6148 .expect("arm_upstream_replay must install a buffer")
6149 .extend_from_slice(&serialized[..accepted]);
6150 stream.front.consume(accepted);
6151 stream.front.consumed = true;
6152 serialized
6153 }
6154
6155 /// The replay must be queued AHEAD of the bytes the kernel refused.
6156 ///
6157 /// `ConnectionH1::writable` handles a partial `socket_write_vectored` by
6158 /// calling `signal_pending_write()` and returning `MuxResult::Continue`,
6159 /// which leaves the unwritten remainder queued: `kawa::Kawa::consume`
6160 /// pushes the partially consumed store back to the FRONT of `out`
6161 /// (kawa-0.7.1 `storage/repr.rs`). `Kawa::push_out` APPENDS, so queueing
6162 /// the capture with it would hand the healthy backend `[tail][head]` — a
6163 /// request mangled mid-token, on the wire, to a backend that never saw
6164 /// the first attempt.
6165 ///
6166 /// Reachable on an idempotent request large enough that the kernel
6167 /// accepts only part of one write to a pooled upstream, followed by that
6168 /// stale peer EOFing without answering.
6169 ///
6170 /// The remainder spans zero, one, three and thirteen `out` entries across
6171 /// the cases below: `consume` pushes back at most ONE partially consumed
6172 /// store and drops every entry before it, so how many survive depends on
6173 /// where the split falls. Prepending is correct for all of them because
6174 /// `VecDeque::push_front` preserves the relative order of what is already
6175 /// queued, and the capture is exactly the complementary prefix.
6176 ///
6177 /// To SEE THIS RED: in `Stream::queue_upstream_replay`, swap the
6178 /// `front.out.push_front(...)` back for `self.front.push_out(...)`.
6179 #[test]
6180 fn a_replay_is_queued_ahead_of_the_unwritten_tail() {
6181 for (unwritten_tail, queued_entries) in [(0, 0), (2, 1), (21, 3), (100, 13)] {
6182 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6183 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6184 let stream = &mut mux.context.streams[0];
6185 let serialized = stage_partially_written_pooled_request(stream, unwritten_tail);
6186 assert_eq!(
6187 stream.front.out.len(),
6188 queued_entries,
6189 "staging a {unwritten_tail}-byte tail must leave {queued_entries} queued entries"
6190 );
6191 assert_eq!(
6192 shared::end_stream_decision(stream),
6193 shared::EndStreamAction::ReplayOnFreshBackend,
6194 "a partially written request on a stale pooled upstream is still replayable"
6195 );
6196
6197 stream
6198 .queue_upstream_replay()
6199 .expect("the staged stream carries a capture");
6200
6201 assert_eq!(
6202 queued_request_bytes(&stream.front),
6203 serialized,
6204 "with a {unwritten_tail}-byte unwritten tail the fresh backend must \
6205 receive the request in its original order, not the tail first"
6206 );
6207 }
6208 }
6209
6210 /// The defect of sozu-proxy/sozu#1442. A request written onto a pooled
6211 /// keep-alive upstream that the peer had already closed must be replayed
6212 /// on a fresh backend, not answered `502 Bad Gateway`: nothing was
6213 /// observed by the client, so re-issuing it is unobservable. Measured
6214 /// 2026-09-22 with only the replay decision reverted: 25 failures over
6215 /// 576 pooled `test_issue_806` trials (4.3% per trial), 25 red runs of
6216 /// 25; green on 25 of 25 with the replay in place.
6217 ///
6218 /// To SEE THIS RED: in `super::shared::end_stream_decision`, replace the
6219 /// `can_replay_on_fresh_upstream()` branch with a bare
6220 /// `EndStreamAction::SendDefault(502)` — the pre-fix shape.
6221 #[test]
6222 fn a_stale_pooled_upstream_that_never_answered_is_replayed_not_502() {
6223 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6224 let (mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6225 assert_eq!(
6226 shared::end_stream_decision(&mux.context.streams[0]),
6227 shared::EndStreamAction::ReplayOnFreshBackend,
6228 );
6229 }
6230
6231 /// The method veto. nginx keeps `non_idempotent` out of the
6232 /// `proxy_next_upstream` default so `POST, LOCK, PATCH` are "not passed
6233 /// to the next server if a request has been sent to an upstream server";
6234 /// pingora's default `error_while_proxy` calls `set_retry(false)` on
6235 /// `!method.is_idempotent()`. Replaying a POST can duplicate a write the
6236 /// origin may already have committed.
6237 ///
6238 /// To SEE THIS RED: drop the `Method::is_idempotent` conjunct from
6239 /// `Stream::can_replay_on_fresh_upstream`.
6240 #[test]
6241 fn a_non_idempotent_request_is_never_replayed_on_a_stale_upstream() {
6242 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6243 let (mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Post);
6244 assert_eq!(
6245 shared::end_stream_decision(&mux.context.streams[0]),
6246 shared::EndStreamAction::SendDefault(502),
6247 );
6248 }
6249
6250 /// An extension method sozu does not know the semantics of is treated
6251 /// like POST. nginx's list is a closed `POST, LOCK, PATCH`; sozu parses
6252 /// `PATCH` as `Method::Custom`, so refusing every `Custom` is what makes
6253 /// PATCH non-replayable here without maintaining a second list.
6254 #[test]
6255 fn an_unknown_method_is_never_replayed_on_a_stale_upstream() {
6256 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6257 let (mux, _peer) = stale_pooled_upstream_stream(&pool, Method::new(b"PATCH"));
6258 assert!(matches!(
6259 mux.context.streams[0].context.method,
6260 Some(Method::Custom(_))
6261 ));
6262 assert_eq!(
6263 shared::end_stream_decision(&mux.context.streams[0]),
6264 shared::EndStreamAction::SendDefault(502),
6265 );
6266 }
6267
6268 /// pingora's `RetryType::ReusedOnly`. The same EOF means "the pool handed
6269 /// out a socket the peer had already closed" on a reused connection and
6270 /// "the origin accepted the request and then died" on a fresh dial; only
6271 /// the first is replayable. A fresh dial never arms the capture, so the
6272 /// absent buffer IS the provenance check.
6273 ///
6274 /// To SEE THIS RED: drop the `retry_buffer.is_some()` conjunct from
6275 /// `Stream::can_replay_on_fresh_upstream` — the provenance check itself.
6276 /// NOT `ConnectionH1::writable`'s `reused_from_pool` guard: this test
6277 /// never calls `writable`. It stages the capture through
6278 /// `stale_pooled_upstream_stream` and clears it with
6279 /// `Stream::forget_upstream_replay`, so a recipe naming the write path
6280 /// cannot redden it however plausible it reads.
6281 #[test]
6282 fn a_request_written_to_a_freshly_dialled_upstream_is_not_replayed() {
6283 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6284 let (mut mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6285 mux.context.streams[0].forget_upstream_replay();
6286 assert_eq!(
6287 shared::end_stream_decision(&mux.context.streams[0]),
6288 shared::EndStreamAction::SendDefault(502),
6289 );
6290 }
6291
6292 /// The boundary itself: once a response byte exists the request is no
6293 /// longer replayable. nginx — "passing a request to the next server is
6294 /// only possible if nothing has been sent to a client yet". A byte merely
6295 /// BUFFERED is already refused here, which is stricter: a partial status
6296 /// line is HAProxy's `junk-response`, absent from every default.
6297 ///
6298 /// To SEE THIS RED: drop the `back.storage.is_empty()` conjunct from
6299 /// `Stream::can_replay_on_fresh_upstream`.
6300 #[test]
6301 fn a_buffered_response_byte_forecloses_the_replay() {
6302 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6303 let (mut mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6304 let back = &mut mux.context.streams[0].back;
6305 back.storage.space()[..4].copy_from_slice(b"HTTP");
6306 back.storage.fill(4);
6307 assert!(!back.is_main_phase(), "a bare `HTTP` is not yet a response");
6308 assert_eq!(
6309 shared::end_stream_decision(&mux.context.streams[0]),
6310 shared::EndStreamAction::SendDefault(502),
6311 );
6312 }
6313
6314 /// A response byte already FORWARDED to the client is the same refusal,
6315 /// reached through the other conjunct.
6316 ///
6317 /// To SEE THIS RED: drop the `!back.consumed` conjunct from
6318 /// `Stream::can_replay_on_fresh_upstream`.
6319 #[test]
6320 fn a_forwarded_response_byte_forecloses_the_replay() {
6321 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6322 let (mut mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6323 mux.context.streams[0].back.consumed = true;
6324 assert_eq!(
6325 shared::end_stream_decision(&mux.context.streams[0]),
6326 shared::EndStreamAction::SendDefault(502),
6327 );
6328 }
6329
6330 /// The replay re-serializes the captured bytes and nothing else: they go
6331 /// back into `front.out` as an owned store, `front.blocks` stays empty so
6332 /// `kawa::Kawa::prepare` contributes nothing, and the buffer is taken so
6333 /// one capture cannot be replayed twice. That is not a per-REQUEST bound:
6334 /// `ConnectionH1::start_stream` arms a fresh capture on the next pooled
6335 /// attempt, and `max_connection_attempts` is what bounds the request as a
6336 /// whole.
6337 ///
6338 /// To SEE THIS RED: make `Stream::queue_upstream_replay` clone the buffer
6339 /// instead of taking it.
6340 #[test]
6341 fn queueing_a_replay_moves_the_captured_bytes_into_the_front_buffer() {
6342 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6343 let (mut mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6344 let stream = &mut mux.context.streams[0];
6345 let captured = stream
6346 .retry_buffer
6347 .as_ref()
6348 .expect("the staged stream carries a capture")
6349 .to_vec();
6350
6351 assert_eq!(stream.queue_upstream_replay(), Some(captured.len()));
6352
6353 assert!(
6354 stream.retry_buffer.is_none(),
6355 "the capture must be taken, not cloned: it cannot outlive its replay"
6356 );
6357 assert!(
6358 stream.front.blocks.is_empty(),
6359 "prepare must have nothing to convert, or it would prepend a \
6360 second copy of the request line"
6361 );
6362 let buffer = stream.front.storage.buffer();
6363 let queued: Vec<u8> = stream
6364 .front
6365 .out
6366 .iter()
6367 .map(|block| match block {
6368 kawa::OutBlock::Store(store) => store.data(buffer).to_vec(),
6369 kawa::OutBlock::Delimiter => Vec::new(),
6370 })
6371 .collect::<Vec<_>>()
6372 .concat();
6373 assert_eq!(
6374 queued, captured,
6375 "the replayed request must be byte-identical to the first attempt"
6376 );
6377 assert_eq!(
6378 shared::end_stream_decision(stream),
6379 shared::EndStreamAction::SendDefault(502),
6380 "with the capture spent and nothing re-arming it, a second stale \
6381 upstream falls back to 502"
6382 );
6383 }
6384
6385 /// Park the frontend core's next deadline at an exact instant, then push it
6386 /// onto the wheel the way a real pass would.
6387 ///
6388 /// Writes `ConnectionH1::timeout_deadline` directly rather than going
6389 /// through `arm_timeout`, so a test can stage a deadline in the past ("the
6390 /// wheel is late") as easily as one in the future, without disturbing the
6391 /// configured duration the re-arm will use.
6392 fn arm_frontend_at(
6393 mux: &mut Mux<mio::net::TcpStream, test_support::TestListener>,
6394 deadline: Option<Instant>,
6395 ) {
6396 let Connection::H1(h1) = &mut mux.frontend else {
6397 unreachable!("the helper builds an H1 frontend")
6398 };
6399 h1.timeout_deadline = deadline;
6400 mux.reschedule();
6401 }
6402
6403 /// Same, for a backend connection that has not been inserted yet.
6404 fn park_backend_deadline(connection: &mut Connection<SessionTcpStream>, deadline: Instant) {
6405 let Connection::H1(h1) = connection else {
6406 unreachable!("the helper builds an H1 backend")
6407 };
6408 h1.timeout_deadline = Some(deadline);
6409 }
6410
6411 /// Fire the frontend timeout of `mux` now, on an H1 frontend whose
6412 /// lingering close may take `linger_timeout`, after `partial` request
6413 /// bytes arrived. Returns the result, the instants bracketing the call,
6414 /// and the frontend's linger state and deadline afterwards.
6415 fn fire_frontend_timeout_after(
6416 partial: &[u8],
6417 linger_timeout: Duration,
6418 ) -> (
6419 StateResult,
6420 Instant,
6421 Instant,
6422 h1::Linger,
6423 Option<Instant>,
6424 std::net::TcpStream,
6425 ) {
6426 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6427 let (mut mux, peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6428 let mut metrics = SessionMetrics::new(None);
6429 {
6430 let Connection::H1(h1) = &mut mux.frontend else {
6431 unreachable!("the helper builds an H1 frontend")
6432 };
6433 h1.linger_timeout = linger_timeout;
6434 }
6435 let storage = &mut mux.context.streams[0].front.storage;
6436 storage.space()[..partial.len()].copy_from_slice(partial);
6437 storage.fill(partial.len());
6438 arm_frontend_at(&mut mux, Some(Instant::now() - Duration::from_millis(1)));
6439
6440 let before = Instant::now();
6441 let result = mux.timeout(Token(0), &mut metrics);
6442 let after = Instant::now();
6443 let Connection::H1(h1) = &mux.frontend else {
6444 unreachable!("the helper builds an H1 frontend")
6445 };
6446 (
6447 result,
6448 before,
6449 after,
6450 h1.linger,
6451 mux.frontend.poll_timeout(),
6452 peer,
6453 )
6454 }
6455
6456 /// A frontend timeout answers a partly received request (408), and the
6457 /// write pass that sends the answer starts the lingering close. The
6458 /// re-arm `Mux::timeout_inner` runs after that pass must not replace the
6459 /// linger deadline (`request_timeout`, 5 s here) with the frontend
6460 /// timeout (60 s).
6461 ///
6462 /// TO SEE THIS RED: remove the `is_lingering()` guard of
6463 /// `ConnectionH1::arm_timeout`. The deadline is then about 60 s out.
6464 #[test]
6465 fn a_linger_started_by_a_timeout_write_keeps_its_own_deadline() {
6466 let linger = Duration::from_secs(5);
6467 let (result, before, after, state, deadline, _peer) =
6468 fire_frontend_timeout_after(b"POST /upl", linger);
6469
6470 assert_eq!(result, StateResult::Continue, "the session lingers");
6471 assert!(
6472 matches!(state, h1::Linger::Draining { .. }),
6473 "a partly received request is drained after its answer, got {state:?}"
6474 );
6475 let deadline = deadline.expect("a lingering frontend has a deadline");
6476 assert!(
6477 deadline >= before + linger && deadline <= after + linger,
6478 "the deadline is the linger's, {:?} from the call",
6479 deadline.saturating_duration_since(before)
6480 );
6481 }
6482
6483 /// A silent client — no byte of a request — gets its 408 and an
6484 /// immediate close: nothing is in flight to drain.
6485 ///
6486 /// TO SEE THIS RED: in `ConnectionH1::writable`, set the linger on
6487 /// `!request_complete` alone, without `request_started`.
6488 #[test]
6489 fn a_408_to_a_silent_client_closes_without_lingering() {
6490 let (result, _, _, state, _, mut peer) =
6491 fire_frontend_timeout_after(b"", Duration::from_secs(5));
6492
6493 assert_eq!(
6494 result,
6495 StateResult::CloseSession,
6496 "nothing was received: the 408 closes at once"
6497 );
6498 assert_eq!(state, h1::Linger::Off, "no linger for a silent client");
6499 peer.set_nonblocking(false)
6500 .expect("the peer reads blocking");
6501 peer.set_read_timeout(Some(Duration::from_secs(1)))
6502 .expect("the peer read is bounded");
6503 let mut head = [0u8; 12];
6504 std::io::Read::read_exact(&mut peer, &mut head).expect("the 408 reaches the client");
6505 assert_eq!(&head, b"HTTP/1.1 408", "the client gets its 408");
6506 }
6507
6508 /// A frontend socket standing for TLS: its output is buffered behind the
6509 /// socket (`socket_wants_write`) until the next write flushes it, and a
6510 /// close queues a `close_notify` the same way.
6511 #[derive(Debug)]
6512 struct BufferedTlsSocket {
6513 stream: mio::net::TcpStream,
6514 buffered: bool,
6515 }
6516
6517 impl SocketHandler for BufferedTlsSocket {
6518 fn socket_read(&mut self, buf: &mut [u8]) -> (usize, SocketResult) {
6519 self.stream.socket_read(buf)
6520 }
6521
6522 fn socket_write(&mut self, buf: &[u8]) -> (usize, SocketResult) {
6523 self.stream.socket_write(buf)
6524 }
6525
6526 fn socket_write_vectored(&mut self, bufs: &[std::io::IoSlice]) -> (usize, SocketResult) {
6527 // What was buffered leaves with this write.
6528 self.buffered = false;
6529 self.stream.socket_write_vectored(bufs)
6530 }
6531
6532 fn socket_wants_write(&self) -> bool {
6533 self.buffered
6534 }
6535
6536 fn socket_close(&mut self) {
6537 self.buffered = true;
6538 }
6539
6540 fn socket_ref(&self) -> &mio::net::TcpStream {
6541 &self.stream
6542 }
6543
6544 fn socket_mut(&mut self) -> &mut mio::net::TcpStream {
6545 &mut self.stream
6546 }
6547
6548 fn peer_addr(&self) -> Option<std::net::SocketAddr> {
6549 self.stream.peer_addr().ok()
6550 }
6551
6552 fn protocol(&self) -> crate::socket::TransportProtocol {
6553 crate::socket::TransportProtocol::Tcp
6554 }
6555
6556 fn read_error(&self) {}
6557
6558 fn write_error(&self) {}
6559 }
6560
6561 /// A TLS client half-closes while the answer to its partly received
6562 /// request is still buffered (a 408 from `Mux::timeout_inner`): the
6563 /// answer is flushed, then the `close_notify`, and the lingering close
6564 /// starts inside the loop (RFC 9112 §9.6, `ConnectionH1::start_linger`).
6565 /// The in-loop HUP check must leave that frontend to its drain: closing it
6566 /// with the client's last bytes unread resets the connection.
6567 ///
6568 /// What is asserted is the socket, not the reader: whichever path
6569 /// consumed the client's last bytes, the request reader or the linger
6570 /// drain, none may be left unread at the close.
6571 ///
6572 /// TO SEE THIS RED: drop `c.readiness.filter_interest().is_readable() ||`
6573 /// from `Connection::has_unread_input`, which is what keeps a lingering
6574 /// frontend (READABLE raised by `ConnectionH1::start_linger`) from the
6575 /// in-loop close: 4 bytes are left unread. Before sozu-proxy/sozu#1779 the
6576 /// `!self.frontend.is_lingering()` guard of that check did this.
6577 #[test]
6578 fn a_lingering_frontend_reads_the_clients_last_bytes_before_closing() {
6579 use std::io::{Read, Write};
6580
6581 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6582 let (socket, mut peer) = connected_socket();
6583 let frontend_timeout = Duration::from_secs(60);
6584 let mut frontend = Connection::new_h1_server(
6585 Ulid::generate(),
6586 BufferedTlsSocket {
6587 stream: socket,
6588 buffered: true,
6589 },
6590 frontend_timeout,
6591 );
6592 let mut context = test_context(&pool);
6593 context
6594 .create_stream(Ulid::generate(), 1 << 16)
6595 .expect("test context must create a stream");
6596 let partial = b"POST /upload HTTP/1.1\r\nHost: localhost\r\nContent-Length: 64\r\n\r\npart";
6597 {
6598 let Connection::H1(h1) = &mut frontend else {
6599 unreachable!("new_h1_server builds an H1 connection")
6600 };
6601 h1.stream = Some(0);
6602 let stream = &mut context.streams[0];
6603 let storage = &mut stream.front.storage;
6604 storage.space()[..partial.len()].copy_from_slice(partial);
6605 storage.fill(partial.len());
6606 let answers_rc = stream.answers.clone();
6607 let answers = answers_rc.borrow();
6608 set_default_answer(stream, &mut h1.readiness, 408, &answers);
6609 }
6610 let mut mux = Mux {
6611 configured_frontend_timeout: frontend_timeout,
6612 frontend_token: Token(0),
6613 frontend,
6614 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
6615 context,
6616 session_ulid: Ulid::generate(),
6617 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(frontend_timeout)),
6618 backend_registry: BackendRegistry::default(),
6619 backends: Rc::default(),
6620 };
6621
6622 // The client sends more of its body, then hangs up: one edge
6623 // carries both, as `EPOLLIN|EPOLLRDHUP`.
6624 peer.write_all(b"more").expect("the client peer writes");
6625 peer.shutdown(std::net::Shutdown::Write)
6626 .expect("the client peer sends its FIN");
6627 std::thread::sleep(Duration::from_millis(20));
6628 mux.frontend.readiness_mut().event = Ready::READABLE | Ready::HUP;
6629
6630 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
6631 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
6632 let mut metrics = SessionMetrics::new(None);
6633 let result = mux.ready(session, proxy, &mut metrics);
6634 let Connection::H1(h1) = &mux.frontend else {
6635 unreachable!("the frontend is H1")
6636 };
6637 // What the close must not leave behind: a byte the client sent and
6638 // sozu never read, which turns the close into a reset. Whichever
6639 // reader consumed them, the request reader or the linger drain, a
6640 // peek on the socket must meet the client's EOF, not data.
6641 let mut probe = [0u8; 16];
6642 let unread = match h1.socket.stream.peek(&mut probe) {
6643 Ok(n) => n,
6644 Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => 0,
6645 Err(e) => panic!("the frontend socket peek failed: {e}"),
6646 };
6647 assert!(
6648 matches!(result, SessionResult::Close),
6649 "the drain meets the client's EOF and closes, got {result:?}"
6650 );
6651 assert_eq!(
6652 unread, 0,
6653 "the session must read the rest of the request before it closes, {unread} bytes \
6654 left unread, {:?}",
6655 h1.linger
6656 );
6657 drop(mux);
6658
6659 peer.set_nonblocking(false)
6660 .expect("the peer reads blocking");
6661 peer.set_read_timeout(Some(Duration::from_secs(1)))
6662 .expect("the peer read is bounded");
6663 let mut received = Vec::new();
6664 let outcome = peer.read_to_end(&mut received);
6665 assert!(
6666 received.starts_with(b"HTTP/1.1 408"),
6667 "the client gets its answer, got {:?}",
6668 String::from_utf8_lossy(&received)
6669 );
6670 assert!(
6671 outcome.is_ok(),
6672 "the response ends with the FIN, not a reset: {outcome:?}"
6673 );
6674 }
6675
6676 /// A request answered before its head was whole (a 408 from
6677 /// `Mux::timeout_inner` on an `Idle` stream) is never linked to a backend
6678 /// when the rest of its head arrives afterwards: the answer is final, and
6679 /// the bytes behind it are drained after it, never forwarded.
6680 ///
6681 /// TO SEE THIS RED: drop `&& !answered` from the first-seen-request branch
6682 /// of `ConnectionH1::readable`.
6683 #[test]
6684 fn an_answered_request_is_never_linked() {
6685 use std::io::Write;
6686
6687 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6688 let (socket, mut peer) = connected_socket();
6689 let frontend_timeout = Duration::from_secs(60);
6690 let mut frontend = Connection::new_h1_server(Ulid::generate(), socket, frontend_timeout);
6691 let mut context = test_context(&pool);
6692 context
6693 .create_stream(Ulid::generate(), 1 << 16)
6694 .expect("test context must create a stream");
6695 let head_start = b"GET /slow HTTP/1.1\r\nHost: loc";
6696 {
6697 let Connection::H1(h1) = &mut frontend else {
6698 unreachable!("new_h1_server builds an H1 connection")
6699 };
6700 h1.stream = Some(0);
6701 let stream = &mut context.streams[0];
6702 let storage = &mut stream.front.storage;
6703 storage.space()[..head_start.len()].copy_from_slice(head_start);
6704 storage.fill(head_start.len());
6705 let answers_rc = stream.answers.clone();
6706 let answers = answers_rc.borrow();
6707 set_default_answer(stream, &mut h1.readiness, 408, &answers);
6708 }
6709 assert_eq!(
6710 context.streams[0].state,
6711 StreamState::Unlinked,
6712 "premise: the 408 leaves the stream answered"
6713 );
6714
6715 peer.write_all(b"alhost\r\n\r\n")
6716 .expect("the client peer writes the rest of its head");
6717 std::thread::sleep(Duration::from_millis(20));
6718 frontend.readiness_mut().event.insert(Ready::READABLE);
6719 let mut router = Router::new(Duration::from_secs(30), Duration::from_secs(30));
6720 let _ = frontend.readable(&mut context, EndpointClient(&mut router));
6721
6722 assert!(
6723 context.streams[0].front.is_main_phase(),
6724 "premise: the rest of the head was read and parsed"
6725 );
6726 assert_eq!(
6727 context.streams[0].state,
6728 StreamState::Unlinked,
6729 "an answered request must stay answered"
6730 );
6731 assert!(
6732 context.pending_links.is_empty(),
6733 "an answered request must not be queued for a backend"
6734 );
6735 }
6736
6737 /// An H1 frontend whose request was received whole and whose response
6738 /// has not started, as one waiting for its backend, with nothing queued
6739 /// for the client. `event` is the readiness the next pass sees.
6740 fn waiting_request_pass(event: Ready) -> SessionResult {
6741 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6742 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6743 {
6744 let stream = &mut mux.context.streams[0];
6745 let request = b"GET / HTTP/1.1\r\nHost: localhost\r\n\r\n";
6746 let storage = &mut stream.front.storage;
6747 storage.space()[..request.len()].copy_from_slice(request);
6748 storage.fill(request.len());
6749 kawa::h1::parse(&mut stream.front, &mut stream.context);
6750 assert!(
6751 stream.front.is_terminated(),
6752 "premise: the request is whole"
6753 );
6754 stream.state = StreamState::Unlinked;
6755 }
6756 let readiness = mux.frontend.readiness_mut();
6757 readiness.interest.remove(Ready::READABLE);
6758 readiness.event = event;
6759 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
6760 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
6761 let mut metrics = SessionMetrics::new(None);
6762 mux.ready(session, proxy, &mut metrics)
6763 }
6764
6765 /// A full hang-up whose reset a `read` or `write` already consumed comes
6766 /// as `EPOLLHUP` without `EPOLLERR`: HUP and WRITE_CLOSED, no ERROR. It
6767 /// closes the session at once, even with a request in flight and no write
6768 /// pending to fail; a half-close (HUP alone) keeps it for the response.
6769 ///
6770 /// TO SEE THIS RED: drop the `event.is_write_closed()` clause from
6771 /// `Mux::frontend_exchange_in_flight`.
6772 #[test]
6773 fn a_hang_up_without_error_closes_a_waiting_request() {
6774 assert!(
6775 matches!(
6776 waiting_request_pass(Ready::READABLE | Ready::HUP | Ready::WRITE_CLOSED),
6777 SessionResult::Close
6778 ),
6779 "a hang-up must close the session at once"
6780 );
6781 assert!(
6782 matches!(
6783 waiting_request_pass(Ready::READABLE | Ready::HUP),
6784 SessionResult::Continue
6785 ),
6786 "a half-close keeps the session for the response"
6787 );
6788 }
6789
6790 /// A write that meets the client's reset before `epoll` reports it fails
6791 /// (`ECONNRESET`/`EPIPE`): the session closes at once, instead of
6792 /// waiting for an event that only says what the write already learned.
6793 /// Any failed write closes it, since nothing more can be flushed.
6794 ///
6795 /// TO SEE THIS RED: make `ConnectionH1::client_write_failed` return
6796 /// `None`.
6797 #[test]
6798 #[cfg(target_os = "linux")]
6799 fn a_failed_write_to_the_client_closes_the_session() {
6800 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6801 let (mut mux, peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6802 {
6803 let Connection::H1(h1) = &mut mux.frontend else {
6804 unreachable!("the helper builds an H1 frontend")
6805 };
6806 let stream = &mut mux.context.streams[0];
6807 let request = b"GET / HTTP/1.1\r\nHost: localhost\r\n\r\n";
6808 let storage = &mut stream.front.storage;
6809 storage.space()[..request.len()].copy_from_slice(request);
6810 storage.fill(request.len());
6811 kawa::h1::parse(&mut stream.front, &mut stream.context);
6812 let answers_rc = stream.answers.clone();
6813 let answers = answers_rc.borrow();
6814 set_default_answer(stream, &mut h1.readiness, 503, &answers);
6815 }
6816 // The client resets: `SO_LINGER` zero, then close.
6817 let linger = libc::linger {
6818 l_onoff: 1,
6819 l_linger: 0,
6820 };
6821 // SAFETY: `linger` is a valid `libc::linger` and `peer` owns the
6822 // descriptor for the duration of the call.
6823 let rc = unsafe {
6824 libc::setsockopt(
6825 std::os::fd::AsRawFd::as_raw_fd(&peer),
6826 libc::SOL_SOCKET,
6827 libc::SO_LINGER,
6828 &linger as *const _ as *const libc::c_void,
6829 std::mem::size_of::<libc::linger>() as libc::socklen_t,
6830 )
6831 };
6832 assert_eq!(rc, 0, "SO_LINGER zero must set");
6833 drop(peer);
6834 std::thread::sleep(Duration::from_millis(20));
6835 // The pass runs on a WRITABLE edge only: the reset is learned by the
6836 // write, not from the event.
6837 mux.frontend.readiness_mut().event = Ready::WRITABLE;
6838
6839 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
6840 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
6841 let mut metrics = SessionMetrics::new(None);
6842 let result = mux.ready(session, proxy, &mut metrics);
6843 assert!(
6844 matches!(result, SessionResult::Close),
6845 "a failed write to the client must close the session, got {result:?}"
6846 );
6847 }
6848
6849 /// A client that never sends its EOF is closed at the linger deadline.
6850 #[test]
6851 fn a_silent_client_is_closed_at_the_linger_deadline() {
6852 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6853 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6854 let mut metrics = SessionMetrics::new(None);
6855 let deadline = Instant::now() - Duration::from_millis(1);
6856 {
6857 let Connection::H1(h1) = &mut mux.frontend else {
6858 unreachable!("the helper builds an H1 frontend")
6859 };
6860 h1.linger = h1::Linger::Draining {
6861 deadline,
6862 remaining: h1::LINGER_MAX_BYTES,
6863 };
6864 }
6865 arm_frontend_at(&mut mux, Some(deadline));
6866
6867 assert_eq!(
6868 mux.timeout(Token(0), &mut metrics),
6869 StateResult::CloseSession,
6870 "the linger deadline closes the session"
6871 );
6872 assert_eq!(
6873 mux.context.streams[0].context.access_log_message, None,
6874 "no per-stream timeout answer for a lingering frontend"
6875 );
6876 }
6877
6878 /// The timer wheel rounds a delay to the NEAREST tick and `Timer::poll`
6879 /// fires everything whose tick has come, so an entry armed for deadline `D`
6880 /// is handed over from `tick * round(D / tick) - tick/2` onwards — up to
6881 /// 99 ms early with the default 100 ms tick. An early delivery is not an
6882 /// expiry: `Mux::timeout` must leave the session alone.
6883 ///
6884 /// To SEE THIS RED: make `Mux::consume_timer_entry` return `true`
6885 /// unconditionally (drop its `core_deadline.is_some_and(...)` early
6886 /// return). The body then runs on an early delivery and this fails with
6887 /// `access_log_message == Some("client_timeout")` — a session closed up to
6888 /// 99 ms before its configured `front_timeout`.
6889 #[test]
6890 fn an_early_wheel_delivery_does_not_run_the_timeout_body() {
6891 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6892 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6893 let mut metrics = SessionMetrics::new(None);
6894
6895 // A minute out, then delivered immediately: early, exaggerated so the
6896 // test cannot be flaky.
6897 arm_frontend_at(&mut mux, Some(Instant::now() + Duration::from_secs(60)));
6898
6899 let result = mux.timeout(Token(0), &mut metrics);
6900
6901 assert_eq!(
6902 result,
6903 StateResult::Continue,
6904 "an early delivery must leave the session running"
6905 );
6906 assert_eq!(
6907 mux.context.streams[0].context.access_log_message, None,
6908 "the timeout body must not run for a delivery before the deadline"
6909 );
6910 }
6911
6912 /// **The ordering test.** This is the one that proves why the
6913 /// `poll_timeout()` split (stored deadline first, memoized reschedule
6914 /// second) must land in that order and not the other way round.
6915 ///
6916 /// `Mux::sync_timeout` memoizes: it touches the wheel only when what the
6917 /// handle holds differs from what the core wants. On an early delivery the
6918 /// core's deadline has NOT moved — it still wants `D` — so a naive memo
6919 /// says "nothing to do" and emits nothing. But the wheel entry is already
6920 /// gone: the delivery consumed it. The session is then left with no timer
6921 /// at all and nothing to arm one. That is the LOST WAKEUP, and it is
6922 /// exactly the bug fixed for the UDP shell in `UdpManager::handle_timeout`,
6923 /// where `reschedule` emitted `ArmTimer` only when the minimum deadline had
6924 /// MOVED.
6925 ///
6926 /// What prevents it here is consume-then-reschedule:
6927 /// `Mux::consume_timer_entry` calls `TimeoutContainer::triggered`, which
6928 /// clears the handle's deadline, so the memo correctly reads "the wheel
6929 /// holds nothing" and re-arms at the same instant.
6930 ///
6931 /// To SEE THIS RED — this is the C2-without-C1 tree:
6932 /// 1. delete `self.deadline = None;` from `TimeoutContainer::triggered`
6933 /// (`timer.rs`), and
6934 /// 2. delete the `&& container.is_armed() == next.is_some()` half of the
6935 /// memo in `Mux::sync_timeout`.
6936 ///
6937 /// `sync_timeout` then sees `deadline() == Some(D) == next` and emits
6938 /// nothing, leaving the session alive with no wheel entry. In a DEBUG build
6939 /// `Mux::debug_assert_timer_coherence` catches it first, inside the
6940 /// `reschedule` on the way out of `timeout`, with `timer coherence for
6941 /// Token(0): armed=false but the core wants Some(..)`; the `is_armed()`
6942 /// assertion below is what catches it with debug assertions off. Either
6943 /// mutation alone is enough; both together are the shape the refactor would
6944 /// have had if the memoized reschedule had landed before the stored
6945 /// deadline.
6946 #[test]
6947 fn an_early_wheel_delivery_leaves_a_live_entry_at_the_same_deadline() {
6948 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6949 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6950 let mut metrics = SessionMetrics::new(None);
6951
6952 let deadline = Instant::now() + Duration::from_secs(60);
6953 arm_frontend_at(&mut mux, Some(deadline));
6954
6955 let result = mux.timeout(Token(0), &mut metrics);
6956 assert_eq!(result, StateResult::Continue);
6957
6958 let container = &mux.timeouts.frontend;
6959 assert!(
6960 container.is_armed(),
6961 "the wheel entry was consumed by the delivery; if the memoized \
6962 reschedule does not put one back the session has no timer at all \
6963 — a lost wakeup"
6964 );
6965 assert_eq!(
6966 container.deadline(),
6967 Some(deadline),
6968 "the entry must go back at the SAME instant: a later one is a \
6969 silently extended timeout, a different one is a moved deadline"
6970 );
6971 assert_eq!(
6972 mux.frontend.poll_timeout(),
6973 Some(deadline),
6974 "an early delivery must not disturb what the core asked for"
6975 );
6976 }
6977
6978 /// The mirror: a delivery at or past the deadline is a real expiry and must
6979 /// run the timeout body. Without this, "treat everything as early" would
6980 /// pass the two tests above and never time anything out.
6981 ///
6982 /// To SEE THIS RED: make `Mux::consume_timer_entry` return `false`
6983 /// unconditionally.
6984 #[test]
6985 fn a_delivery_past_the_deadline_runs_the_timeout_body() {
6986 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6987 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6988 let mut metrics = SessionMetrics::new(None);
6989
6990 arm_frontend_at(&mut mux, Some(Instant::now() - Duration::from_secs(1)));
6991
6992 let _ = mux.timeout(Token(0), &mut metrics);
6993
6994 assert_eq!(
6995 mux.context.streams[0].context.access_log_message,
6996 Some("client_timeout"),
6997 "a delivery at or past the deadline is a real expiry and must run \
6998 the timeout body"
6999 );
7000 }
7001
7002 /// A firing for a core that wants no timer at all must fail OPEN — be
7003 /// reported due — rather than being re-validated away. Failing closed there
7004 /// would make such a session immortal, which is strictly worse than the
7005 /// up-to-99 ms-early close the re-validation exists to prevent.
7006 ///
7007 /// To SEE THIS RED: change `Mux::consume_timer_entry` to return `false`
7008 /// when `core_deadline` is `None`.
7009 #[test]
7010 fn a_firing_with_no_deadline_is_treated_as_due() {
7011 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7012 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7013 let mut metrics = SessionMetrics::new(None);
7014
7015 arm_frontend_at(&mut mux, None);
7016 assert_eq!(mux.frontend.poll_timeout(), None, "precondition: no timer");
7017
7018 let _ = mux.timeout(Token(0), &mut metrics);
7019
7020 assert_eq!(
7021 mux.context.streams[0].context.access_log_message,
7022 Some("client_timeout"),
7023 "a firing with no deadline must fail open and run the body"
7024 );
7025 }
7026
7027 /// A real expiry must CLEAR the core's elapsed deadline rather than leave
7028 /// it in place for `reschedule` to re-arm.
7029 ///
7030 /// The core asked to be called back at `D` and has been. Leaving `D` there
7031 /// makes the memoized reschedule arm an already-elapsed instant, which the
7032 /// wheel re-delivers on the very next tick, and the session spins — the
7033 /// canonical sans-io busy loop that `UdpManager::handle_timeout`'s
7034 /// strict-advance `debug_assert!` exists to catch. Every branch of
7035 /// `Mux::timeout` that keeps the session alive re-arms explicitly with
7036 /// `arm_timeout`, which is what the old `triggered()` + `set(token)` pair
7037 /// did.
7038 ///
7039 /// Staged on a BACKEND token deliberately. A frontend firing runs
7040 /// `ConnectionH1::writable`, whose first act is `arm_timeout`, so the
7041 /// frontend re-arms as a side effect and cannot witness the difference. The
7042 /// backend `should_close` path re-arms nothing — it is the one place the
7043 /// elapsed deadline would survive.
7044 ///
7045 /// To SEE THIS RED: delete the `clear_timeout()` block from
7046 /// `Mux::consume_timer_entry`. The strict-advance `debug_assert!` in
7047 /// `SessionState::timeout` fires first (which is the point of it); with
7048 /// debug assertions off, these assertions catch it.
7049 #[test]
7050 fn a_real_expiry_clears_the_elapsed_deadline_instead_of_re_arming_it() {
7051 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7052 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7053 let mut metrics = SessionMetrics::new(None);
7054 let backend_token = Token(1);
7055
7056 let (mut connection, _backend_peer) =
7057 test_backend_connection(&mut mux, Duration::from_secs(30));
7058 let Connection::H1(h1) = &mut connection else {
7059 unreachable!("new_h1_client builds an H1 connection")
7060 };
7061 h1.stream = Some(0);
7062 let fired_at = Instant::now();
7063 park_backend_deadline(&mut connection, fired_at - Duration::from_secs(1));
7064 mux.router.backends.insert(backend_token, connection);
7065 mux.reschedule();
7066
7067 // One linked stream whose response is already terminated and fully
7068 // proxied: the arm that leaves `should_close` true, so nothing re-arms
7069 // the backend.
7070 mux.context.link_stream(0, backend_token);
7071 mux.context.streams[0].back.consumed = true;
7072 mux.context.streams[0].back.parsing_phase = kawa::ParsingPhase::Terminated;
7073
7074 let _ = mux.timeout(backend_token, &mut metrics);
7075
7076 let core = mux
7077 .router
7078 .backends
7079 .get(&backend_token)
7080 .and_then(Connection::poll_timeout);
7081 assert!(
7082 core.is_none_or(|next| next > fired_at),
7083 "after a real expiry the core must want no timer or a strictly \
7084 later one, never the instant that just fired ({core:?})"
7085 );
7086 assert!(
7087 mux.timeouts
7088 .backends
7089 .get(&backend_token)
7090 .and_then(TimeoutContainer::deadline)
7091 .is_none_or(|next| next > fired_at),
7092 "and the wheel must not be holding the elapsed instant either"
7093 );
7094 }
7095
7096 /// A backend that leaves `router.backends` must take its wheel handle with
7097 /// it. Nothing cancels it explicitly any more — `reschedule`'s `retain`
7098 /// drops the handle and `TimeoutContainer::drop` cancels the entry — so the
7099 /// eviction has to be pinned somewhere.
7100 ///
7101 /// To SEE THIS RED: delete the `self.timeouts.backends.retain(...)` call from
7102 /// `Mux::reschedule`. The handle then outlives its connection and
7103 /// `debug_assert_timer_coherence` panics with "timer handle for Token(1)
7104 /// outlived its connection".
7105 #[test]
7106 fn a_departed_backend_releases_its_wheel_handle() {
7107 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7108 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7109 let backend_token = Token(1);
7110 let (connection, _backend_peer) =
7111 test_backend_connection(&mut mux, Duration::from_secs(30));
7112 mux.router.backends.insert(backend_token, connection);
7113
7114 mux.reschedule();
7115 assert!(
7116 mux.timeouts.backends.contains_key(&backend_token),
7117 "a live backend must hold a wheel handle"
7118 );
7119
7120 mux.router.backends.remove(&backend_token);
7121 mux.reschedule();
7122
7123 assert!(
7124 !mux.timeouts.backends.contains_key(&backend_token),
7125 "a departed backend must not leave a wheel handle behind"
7126 );
7127 }
7128
7129 // ── Endpoint::peer_rtt is keyed by token ────────────────────────────
7130 //
7131 // `EndpointClient` keys backends by token; `EndpointServer` has a single
7132 // frontend and ignores the token. The pair matters because the two sides
7133 // populate DIFFERENT access-log cells — `snapshot_rtts` reads the local
7134 // socket for `client_rtt` and this trait method for `server_rtt` — so a
7135 // lookup that returned the wrong connection's RTT would mislabel the
7136 // value rather than lose it, which no downstream assertion would catch.
7137
7138 /// An unknown token yields `None`, not some other backend's RTT.
7139 ///
7140 /// The premise is asserted first: on a live loopback socket
7141 /// `getsockopt(TCP_INFO)` succeeds, so a KNOWN token really does return
7142 /// `Some`. Without that half, this test would pass against a
7143 /// `peer_rtt` that always answered `None`.
7144 ///
7145 /// TO SEE THIS RED: in `EndpointClient::peer_rtt` (`connection.rs`),
7146 /// replace `.get(&token)` with `.values().next()`. The unknown token then
7147 /// resolves to the only backend in the map and the final assertion fails
7148 /// with `an unknown token must not resolve to another backend's RTT`.
7149 #[test]
7150 fn endpoint_client_peer_rtt_is_keyed_by_token() {
7151 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7152 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(30));
7153 let backend_token = Token(1);
7154 let (connection, _backend_peer) =
7155 test_backend_connection(&mut mux, Duration::from_secs(30));
7156 mux.router.backends.insert(backend_token, connection);
7157
7158 let endpoint = EndpointClient(&mut mux.router);
7159
7160 assert!(
7161 endpoint.peer_rtt(backend_token).is_some(),
7162 "premise: TCP_INFO answers on a live loopback backend, so a known \
7163 token returns Some — otherwise the assertion below proves nothing"
7164 );
7165 assert!(
7166 endpoint.peer_rtt(Token(99)).is_none(),
7167 "an unknown token must not resolve to another backend's RTT"
7168 );
7169 }
7170
7171 /// `EndpointServer` ignores the token: it holds one frontend, and every
7172 /// token must report that frontend's RTT rather than `None`.
7173 ///
7174 /// TO SEE THIS RED: in `EndpointServer::peer_rtt` (`connection.rs`),
7175 /// return `None` instead of `self.0.rtt()`. It fails on
7176 /// the first loop iteration with `EndpointServer must report its single
7177 /// frontend's RTT for any token` — the second never runs, which is why
7178 /// the loop is two tokens rather than an assertion about "any".
7179 #[test]
7180 fn endpoint_server_peer_rtt_ignores_the_token() {
7181 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7182 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(30));
7183
7184 let endpoint = EndpointServer(&mut mux.frontend);
7185
7186 for token in [Token(0), Token(7)] {
7187 assert!(
7188 endpoint.peer_rtt(token).is_some(),
7189 "EndpointServer must report its single frontend's RTT for any token"
7190 );
7191 }
7192 }
7193
7194 /// A `Mux` pass keeps the connection's frontend-RTT sample, and takes
7195 /// none when nothing in the pass is logged.
7196 ///
7197 /// Until the per-connection sample, every `Mux` entry point expired the
7198 /// frontend's sample so the next pass read a fresh one: twenty keep-alive
7199 /// requests cost twenty `getsockopt(TCP_INFO)` on the frontend alone. The
7200 /// sample is now the connection's (`memoized_rtt`), so a pass must leave
7201 /// it exactly where the previous one put it.
7202 ///
7203 /// Entered through [`SessionState::timeout`] rather than
7204 /// [`SessionState::ready`]: `ready` takes `Rc<RefCell<dyn ProxySession>>`
7205 /// and `Rc<RefCell<dyn L7Proxy>>` and this crate has no test
7206 /// implementation of either — `L7Proxy::sessions` alone would require a
7207 /// live `SessionManager`. `timeout` needs no mock and is not a lesser
7208 /// path: it is where `cancel_timed_out_streams` reaps a silent peer's
7209 /// streams and emits their access logs, and it was one of the three entry
7210 /// points that used to expire the sample.
7211 ///
7212 /// TO SEE THIS RED: at the top of [`Mux::timeout_inner`], below the
7213 /// early-delivery gate, clear the frontend's `local_rtt` again
7214 /// (`if let Connection::H2(h2) = &self.frontend { h2.local_rtt.set(None) }`),
7215 /// which is what `Mux::expire_client_rtt` did. The first assertion then
7216 /// fails with `a Mux pass must keep the connection's frontend RTT`,
7217 /// `left: None`.
7218 #[test]
7219 fn a_mux_pass_keeps_the_connection_sample_and_reads_none_without_a_log() {
7220 /// A value no loopback SRTT can take, standing for the sample an
7221 /// earlier pass of the connection took.
7222 const CONNECTION_SAMPLE: Duration = Duration::from_secs(4321);
7223
7224 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7225 let (socket, _peer) = connected_socket();
7226 let h2 = h2::H2Shell::new(
7227 Ulid::generate(),
7228 socket,
7229 Position::Server,
7230 &mut PoolBufferSource::new(Rc::downgrade(&pool)),
7231 H2FloodConfig::default(),
7232 H2ConnectionConfig::default(),
7233 Duration::from_secs(30),
7234 None,
7235 Duration::from_secs(30),
7236 None,
7237 Ready::READABLE | Ready::HUP | Ready::ERROR,
7238 )
7239 .expect("a pool with free buffers must yield an H2 connection");
7240
7241 let mut frontend = Connection::H2(h2);
7242 let Connection::H2(h2) = &mut frontend else {
7243 unreachable!("frontend was built as H2")
7244 };
7245 h2.local_rtt.set(Some(Some(CONNECTION_SAMPLE)));
7246 // A deadline already in the past, so `Mux::consume_timer_entry`
7247 // accepts the firing as a real expiry and the body below it runs.
7248 // Without this the connection carries the 30 s deadline it armed at
7249 // construction, the firing re-validates as an early wheel delivery,
7250 // and `timeout_inner` returns before reaching anything this test is
7251 // about.
7252 h2.core.timeout_deadline = Some(Instant::now() - Duration::from_secs(1));
7253
7254 let mut mux = Mux {
7255 configured_frontend_timeout: Duration::from_secs(30),
7256 frontend_token: Token(0),
7257 frontend,
7258 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
7259 context: test_context(&pool),
7260 session_ulid: Ulid::generate(),
7261 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
7262 backend_registry: BackendRegistry::default(),
7263 backends: Rc::default(),
7264 };
7265
7266 let mut metrics = SessionMetrics::new(None);
7267 let before = tcp_info_reads();
7268 let _ = mux.timeout(Token(0), &mut metrics);
7269
7270 let Connection::H2(h2) = &mux.frontend else {
7271 unreachable!("frontend was built as H2")
7272 };
7273 assert_eq!(
7274 h2.local_rtt.get(),
7275 Some(Some(CONNECTION_SAMPLE)),
7276 "a Mux pass must keep the connection's frontend RTT"
7277 );
7278 assert_eq!(
7279 tcp_info_reads() - before,
7280 0,
7281 "a Mux pass that logs no stream must not read TCP_INFO"
7282 );
7283 }
7284
7285 /// An `L7Proxy` for [`Mux::close`] on a session that holds no backend:
7286 /// the teardown only borrows the proxy per backend, so no method runs.
7287 struct NoBackendProxy;
7288
7289 impl L7Proxy for NoBackendProxy {
7290 fn kind(&self) -> sozu_command::proto::command::ListenerType {
7291 unreachable!("Mux::close without a backend never asks the proxy")
7292 }
7293 fn register_socket(
7294 &self,
7295 _socket: &mut TcpStream,
7296 _token: Token,
7297 _interest: Interest,
7298 ) -> Result<(), std::io::Error> {
7299 unreachable!("Mux::close without a backend never asks the proxy")
7300 }
7301 fn add_session(&self, _session: Rc<RefCell<dyn ProxySession>>) -> Token {
7302 unreachable!("Mux::close without a backend never asks the proxy")
7303 }
7304 fn remove_session(&self, _token: Token) -> bool {
7305 unreachable!("Mux::close without a backend never asks the proxy")
7306 }
7307 fn clusters(
7308 &self,
7309 ) -> &HashMap<sozu_command::state::ClusterId, sozu_command::proto::command::Cluster>
7310 {
7311 unreachable!("Mux::close without a backend never asks the proxy")
7312 }
7313 fn sessions(&self) -> Rc<RefCell<crate::server::SessionManager>> {
7314 unreachable!("Mux::close without a backend never asks the proxy")
7315 }
7316 }
7317
7318 /// Close a mux whose only stream is in `state`, still owing its access log
7319 /// or not, and return the `getsockopt(TCP_INFO)` reads the close made.
7320 ///
7321 /// `SessionMetrics::reset` is what every access-log site runs right after
7322 /// the line goes out, so a reset stream is one whose log was written.
7323 fn tcp_info_reads_of_a_close(state: StreamState, owes_a_log: bool) -> usize {
7324 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7325 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(30));
7326 mux.context.streams[0].state = state;
7327 if !owes_a_log {
7328 mux.context.streams[0].metrics.reset();
7329 }
7330 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(NoBackendProxy));
7331 let mut metrics = SessionMetrics::new(None);
7332 let before = tcp_info_reads();
7333 mux.close(proxy, &mut metrics);
7334 tcp_info_reads() - before
7335 }
7336
7337 /// `Mux::close` reads the frontend RTT only when a stream still owes an
7338 /// access log — the ordinary close, where every stream was logged when it
7339 /// finished, reads nothing (issue #1590).
7340 ///
7341 /// TO SEE THIS RED: in `Mux::close`, hoist the sample above the loop
7342 /// (`let client_rtt = self.frontend.rtt();`). The first assertion then
7343 /// fails with `left: 1`, `right: 0`.
7344 #[test]
7345 fn mux_close_reads_the_frontend_rtt_only_for_a_stream_it_logs() {
7346 assert_eq!(
7347 tcp_info_reads_of_a_close(StreamState::Link, false),
7348 0,
7349 "a close with every stream already logged must not read TCP_INFO"
7350 );
7351 assert_eq!(
7352 tcp_info_reads_of_a_close(StreamState::Recycle, true),
7353 0,
7354 "a recycled stream is not logged at close, so it costs no read"
7355 );
7356 assert_eq!(
7357 tcp_info_reads_of_a_close(StreamState::Link, true),
7358 1,
7359 "an open stream logged at close costs exactly the frontend read: \
7360 it has no backend, so no server_rtt read"
7361 );
7362 }
7363
7364 /// #1564: naming an already-interned backend for another dial allocates
7365 /// nothing. `BackendRegistry::id_for` runs once per backend connection a
7366 /// session opens; the first dial of a backend pays for its slot, every
7367 /// later one must only bump reference counts. Two backends, so a cache
7368 /// that held a single id instead of one per slot fails too.
7369 #[test]
7370 fn a_redial_of_an_interned_backend_allocates_nothing() {
7371 use std::hint::black_box;
7372
7373 use crate::test_allocations::allocations;
7374
7375 const DIALS: usize = 64;
7376 let new_backend = |id: &str, port: u16| {
7377 Rc::new(RefCell::new(Backend::new(
7378 id,
7379 SocketAddr::from(([127, 0, 0, 1], port)),
7380 None,
7381 None,
7382 None,
7383 )))
7384 };
7385 let first = new_backend("redial-first", 7001);
7386 let second = new_backend("redial-second", 7002);
7387
7388 let mut registry = BackendRegistry::default();
7389 // Warm-up: the first dial of each backend interns it.
7390 let warm = [registry.id_for(&first), registry.id_for(&second)];
7391
7392 let before = allocations();
7393 for _ in 0..DIALS {
7394 black_box(registry.id_for(black_box(&first)));
7395 black_box(registry.id_for(black_box(&second)));
7396 }
7397 let allocated = allocations() - before;
7398 assert_eq!(
7399 allocated, 0,
7400 "{DIALS} redials of two interned backends made {allocated} heap \
7401 allocations, expected none"
7402 );
7403
7404 // The redial still names the right slot, id and address.
7405 let again = registry.id_for(&second);
7406 assert_eq!(again, warm[1]);
7407 assert_eq!(&*again.backend_id, "redial-second");
7408 assert_eq!(again.address, SocketAddr::from(([127, 0, 0, 1], 7002)));
7409 assert_ne!(warm[0].slot(), warm[1].slot());
7410 }
7411
7412 /// #1610: a session's first interned backend sizes the registry for one
7413 /// entry, not the four a first `Vec::push` reserves.
7414 ///
7415 /// The interning pays for the entry and for the `Rc<str>` copy of its id,
7416 /// and for nothing else; the id copy is measured on its own, so the
7417 /// assertion carries no size literal.
7418 ///
7419 /// TO SEE THIS RED: drop the `reserve_exact(1)` in
7420 /// `BackendRegistry::intern`. The first entry then allocates four
7421 /// `RegistryEntry` slots.
7422 #[test]
7423 fn the_first_interned_backend_sizes_the_registry_for_one() {
7424 use crate::test_allocations::bytes;
7425
7426 let backend = Rc::new(RefCell::new(Backend::new(
7427 "interned-first",
7428 SocketAddr::from(([127, 0, 0, 1], 7003)),
7429 None,
7430 None,
7431 None,
7432 )));
7433 let before = bytes();
7434 let id_copy: Rc<str> = Rc::from(backend.borrow().backend_id.as_str());
7435 let id_bytes = bytes() - before;
7436 drop(id_copy);
7437
7438 let mut registry = BackendRegistry::default();
7439 let before = bytes();
7440 let id = registry.id_for(&backend);
7441 let interned = bytes() - before;
7442
7443 assert_eq!(
7444 interned,
7445 std::mem::size_of::<RegistryEntry>() + id_bytes,
7446 "the first interned backend must size the registry for one entry"
7447 );
7448 assert_eq!(&*id.backend_id, "interned-first");
7449 }
7450
7451 /// #1613: the first backend entry of a session's reverse index holds its
7452 /// one stream inline, so linking it allocates nothing. The `HashMap` of
7453 /// `Vec`s it replaces allocated its table and the entry's vector here —
7454 /// two allocations, 156 bytes — once per session, so once per request
7455 /// on a one-request connection. (#1610 had already cut that vector from
7456 /// four slots to one.)
7457 ///
7458 /// TO SEE THIS RED: in `LinkedStreams::push`, spill into the `Vec` on
7459 /// the first link as well (1 allocation); or in
7460 /// `InlineTokenMap::get_or_insert_with`, send every token to `rest`
7461 /// (the `BTreeMap` leaf is allocated).
7462 #[test]
7463 fn a_new_backend_entry_in_the_reverse_index_is_sized_for_one_stream() {
7464 use crate::test_allocations::{allocations, bytes};
7465
7466 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7467 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7468
7469 let (before, before_bytes) = (allocations(), bytes());
7470 mux.context.link_stream(0, Token(1));
7471 let linked = (allocations() - before, bytes() - before_bytes);
7472
7473 assert_eq!(
7474 linked,
7475 (0, 0),
7476 "the first reverse-index entry must hold its one stream without allocating"
7477 );
7478 assert_eq!(
7479 mux.context
7480 .backend_streams
7481 .get(&Token(1))
7482 .map(|ids| ids.to_vec()),
7483 Some(vec![0])
7484 );
7485 }
7486
7487 /// The other side of the inline slot: a second stream linked to the same
7488 /// backend (an H2 backend multiplexing) moves both into one `Vec` sized
7489 /// for four, in link order — the single 32-byte allocation the `Vec` the
7490 /// slot replaced made at the same point — and a later link on that
7491 /// connection reuses it.
7492 #[test]
7493 fn a_second_stream_on_one_backend_spills_the_reverse_index_entry_once() {
7494 use crate::test_allocations::{allocations, bytes};
7495
7496 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7497 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7498 let second = mux
7499 .context
7500 .create_stream(Ulid::generate(), 1 << 16)
7501 .expect("the test pool must hand out a second stream");
7502 mux.context.link_stream(0, Token(1));
7503
7504 let (before, before_bytes) = (allocations(), bytes());
7505 mux.context.link_stream(second, Token(1));
7506 let spilled = (allocations() - before, bytes() - before_bytes);
7507 assert_eq!(
7508 spilled,
7509 (1, 4 * std::mem::size_of::<GlobalStreamId>()),
7510 "the second stream must spill both into one four-slot vector"
7511 );
7512 assert_eq!(
7513 mux.context
7514 .backend_streams
7515 .get(&Token(1))
7516 .map(|ids| ids.to_vec()),
7517 Some(vec![0, second]),
7518 "the spill must keep the link order"
7519 );
7520
7521 mux.context.unlink_stream(0);
7522 let before = allocations();
7523 mux.context.link_stream(0, Token(1));
7524 assert_eq!(
7525 allocations() - before,
7526 0,
7527 "a spilled entry keeps its capacity for the next link (#1583)"
7528 );
7529 }
7530
7531 /// #1613: the wheel handles of the usual session — the frontend's and one
7532 /// backend's — cost no allocation, because the frontend's is a field of
7533 /// `MuxTimeouts` and the first backend's is held inline. The
7534 /// `HashMap<Token, TimeoutContainer>` they replace allocated its table
7535 /// (one allocation, 340 bytes) once per session.
7536 ///
7537 /// Neither core wants a deadline here, so `reschedule` never reaches the
7538 /// timer wheel, and what is measured is the handles' storage alone.
7539 ///
7540 /// TO SEE THIS RED: in `InlineTokenMap::get_or_insert_with`, send every
7541 /// token to `rest` (the `BTreeMap` leaf is allocated).
7542 #[test]
7543 fn a_session_holds_its_frontend_and_first_backend_wheel_handles_without_allocating() {
7544 use crate::test_allocations::{allocations, bytes};
7545
7546 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7547 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7548 let backend_token = Token(1);
7549 let (mut connection, _backend_peer) =
7550 test_backend_connection(&mut mux, Duration::from_secs(30));
7551 connection.clear_timeout();
7552 mux.frontend.clear_timeout();
7553
7554 let (before, before_bytes) = (allocations(), bytes());
7555 mux.router.backends.insert(backend_token, connection);
7556 mux.reschedule();
7557 let held = (allocations() - before, bytes() - before_bytes);
7558
7559 assert_eq!(
7560 held,
7561 (0, 0),
7562 "the frontend and first backend handles must not allocate"
7563 );
7564 assert!(
7565 mux.timeouts.backends.contains_key(&backend_token),
7566 "the live backend must hold a wheel handle"
7567 );
7568 }
7569
7570 /// Move a client connection built `Connecting` to `Connected`, as the
7571 /// WRITABLE edge of an established dial does in `Mux::ready_inner`.
7572 fn mark_connected(h1: &mut ConnectionH1<SessionTcpStream>) {
7573 let Position::Client(_, _, status) = &mut h1.position else {
7574 unreachable!("a backend connection is in client position")
7575 };
7576 *status = BackendStatus::Connected;
7577 }
7578
7579 /// A backend timer that fires while the dial is still `Connecting` is a
7580 /// connect timeout (sozu-proxy/sozu#1800): no byte reached the backend,
7581 /// so the request is neither answered 504 nor unlinked here. The
7582 /// connection is flagged dead as a refused dial is, its streams stay
7583 /// linked for the `ready` pass that counts the failure and re-links them,
7584 /// and the session asks for that pass.
7585 ///
7586 /// To SEE THIS RED: delete the `Connecting` early return from the
7587 /// backend-token branch of `Mux::timeout_inner`.
7588 #[test]
7589 fn a_connect_timeout_flags_the_dial_dead_instead_of_answering_504() {
7590 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7591 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7592 let mut metrics = SessionMetrics::new(None);
7593 let backend_token = Token(1);
7594
7595 let (mut connection, _backend_peer) =
7596 test_backend_connection(&mut mux, Duration::from_secs(3));
7597 let Connection::H1(h1) = &mut connection else {
7598 unreachable!("new_h1_client builds an H1 connection")
7599 };
7600 h1.stream = Some(0);
7601 assert!(
7602 matches!(
7603 h1.position,
7604 Position::Client(_, _, BackendStatus::Connecting(_))
7605 ),
7606 "precondition: the dial has not completed"
7607 );
7608 park_backend_deadline(&mut connection, Instant::now() - Duration::from_secs(1));
7609 mux.router.backends.insert(backend_token, connection);
7610 mux.reschedule();
7611 mux.context.link_stream(0, backend_token);
7612 assert!(!mux.has_pending_backend_failure());
7613
7614 let result = mux.timeout(backend_token, &mut metrics);
7615
7616 assert_eq!(result, StateResult::Continue, "the session stays open");
7617 assert!(
7618 mux.has_pending_backend_failure(),
7619 "the dial is flagged dead for the ready pass"
7620 );
7621 assert_eq!(
7622 mux.context.streams[0].state,
7623 StreamState::Linked(backend_token),
7624 "the stream stays linked until the ready pass re-links it"
7625 );
7626 assert_eq!(
7627 mux.context.streams[0].context.access_log_message, None,
7628 "no backend_timeout answer"
7629 );
7630 assert!(
7631 mux.context.streams[0].back.blocks.is_empty()
7632 && mux.context.streams[0].back.out.is_empty(),
7633 "no default answer was written"
7634 );
7635 }
7636
7637 /// An H1 backend connection on a live loopback socket, plus the peer the
7638 /// caller must keep alive.
7639 fn test_backend_connection(
7640 mux: &mut Mux<mio::net::TcpStream, test_support::TestListener>,
7641 duration: Duration,
7642 ) -> (Connection<SessionTcpStream>, std::net::TcpStream) {
7643 let (backend_socket, backend_peer) = connected_socket();
7644 let backend_address = "127.0.0.1:2".parse().expect("backend address must parse");
7645 let backend = Rc::new(RefCell::new(Backend::new(
7646 "test-backend",
7647 backend_address,
7648 None,
7649 None,
7650 None,
7651 )));
7652 // Through the embedder's table, exactly as a real dial does: the
7653 // handle stops here and the connection carries the opaque id.
7654 let backend = mux.backend_registry.id_for(&backend);
7655 let connection = Connection::new_h1_client(
7656 Ulid::generate(),
7657 SessionTcpStream::new(backend_socket, mux.session_ulid, Some(backend_address)),
7658 "test-cluster".into(),
7659 backend,
7660 duration,
7661 );
7662 (connection, backend_peer)
7663 }
7664
7665 /// The loose end flagged as UNVERIFIED in the `poll_timeout` series: in the
7666 /// backend-token branch of `Mux::timeout`, the backend's timer is consumed
7667 /// unconditionally but re-armed only when `!should_close`, while the shared
7668 /// tail can still return `Continue`. If a backend could reach that tail
7669 /// still holding linked streams, those streams would be left with no
7670 /// backend timer at all — a lost wakeup.
7671 ///
7672 /// It cannot, and this pins the reason: the branch calls
7673 /// `Connection::end_stream` for every id in `context.backend_streams`.
7674 /// `ConnectionH2::end_stream` starts with `Context::unlink_stream`;
7675 /// `ConnectionH1::end_stream` guards `self.stream != Some(stream)` and
7676 /// returns early first, which is safe only because an H1 connection carries
7677 /// at most one linked stream and it equals `self.stream` (the comment at
7678 /// the re-arm site has the full chain). This test sets
7679 /// `h1.stream = Some(0)` by hand, so it exercises the MATCHED path only —
7680 /// the mismatch arm is unreached here. It is covered by that invariant plus
7681 /// the `debug_assert_h1_owns_stream` tripwire that runs at every
7682 /// `end_stream` call in `Mux::timeout`, not by a test that drives the arm
7683 /// itself. It drives the exact shape the issue asked for — a backend
7684 /// stalled after a PARTIAL response (`back.consumed`, not terminated),
7685 /// which is the arm that sets `should_write` while leaving `should_close`
7686 /// true, i.e. the arm that skips the re-arm.
7687 ///
7688 /// To SEE THIS RED: delete the `self.context.unlink_stream(stream_id);`
7689 /// from the "forcefully terminate it" arm AND the
7690 /// `backend.end_stream(stream_id, &mut self.context);` at the bottom of
7691 /// that loop — this arm evicts through both, so either alone still clears
7692 /// the index. The companion test
7693 /// `a_completed_backend_response_timeout_unlinks_through_end_stream_alone`
7694 /// covers the arm where `end_stream` IS the sole eviction.
7695 #[test]
7696 fn a_backend_timeout_leaves_no_linked_stream_behind_on_the_close_path() {
7697 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7698 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7699 let mut metrics = SessionMetrics::new(None);
7700 let backend_token = Token(1);
7701
7702 let (mut connection, _backend_peer) =
7703 test_backend_connection(&mut mux, Duration::from_secs(30));
7704 let Connection::H1(h1) = &mut connection else {
7705 unreachable!("new_h1_client builds an H1 connection")
7706 };
7707 h1.stream = Some(0);
7708 // A backend that has sent part of a response is connected. Left
7709 // `Connecting`, as `new_h1_client` builds it, the timer would be a
7710 // connect timeout, which fails the request over instead
7711 // (sozu-proxy/sozu#1800).
7712 mark_connected(h1);
7713 // Due, not early: the entry really elapsed.
7714 park_backend_deadline(&mut connection, Instant::now() - Duration::from_secs(1));
7715 mux.router.backends.insert(backend_token, connection);
7716 mux.reschedule();
7717
7718 mux.context.link_stream(0, backend_token);
7719 // A response that started but never finished.
7720 mux.context.streams[0].back.consumed = true;
7721 assert_eq!(
7722 mux.context
7723 .backend_streams
7724 .get(&backend_token)
7725 .map(|ids| ids.len()),
7726 Some(1),
7727 "precondition: the backend carries exactly one linked stream"
7728 );
7729
7730 let _ = mux.timeout(backend_token, &mut metrics);
7731
7732 assert!(
7733 mux.context
7734 .backend_streams
7735 .get(&backend_token)
7736 .is_none_or(|ids| ids.is_empty()),
7737 "every stream linked to a timed-out backend must be unlinked before \
7738 the shared tail runs, because the tail may return Continue with \
7739 only the frontend timer re-armed"
7740 );
7741 assert_eq!(
7742 mux.context.streams[0].context.access_log_message,
7743 Some("backend_response_timeout"),
7744 "precondition: this is the partial-response arm, the one that \
7745 leaves should_close true and therefore skips the backend re-arm"
7746 );
7747 }
7748
7749 /// A frontend whose stream 0 is linked to a connected H1 backend, with a
7750 /// response in progress (`back.consumed`, not terminated). `queued`
7751 /// leaves response bytes waiting for the client; the two deadlines are
7752 /// staged exactly. Shared by the sozu-proxy/sozu#1906 timeout tests.
7753 fn mux_with_response_in_progress(
7754 pool: &Rc<RefCell<Pool>>,
7755 queued: bool,
7756 frontend_deadline: Instant,
7757 backend_deadline: Instant,
7758 ) -> (
7759 Mux<mio::net::TcpStream, test_support::TestListener>,
7760 Token,
7761 std::net::TcpStream,
7762 std::net::TcpStream,
7763 ) {
7764 let (mut mux, peer) = h1_mux_with_idle_stream(pool, Duration::from_secs(60));
7765 let backend_token = Token(1);
7766 let (mut connection, backend_peer) =
7767 test_backend_connection(&mut mux, Duration::from_secs(30));
7768 let Connection::H1(h1) = &mut connection else {
7769 unreachable!("new_h1_client builds an H1 connection")
7770 };
7771 h1.stream = Some(0);
7772 mark_connected(h1);
7773 park_backend_deadline(&mut connection, backend_deadline);
7774 mux.router.backends.insert(backend_token, connection);
7775 mux.context.link_stream(0, backend_token);
7776 mux.context.streams[0].back.consumed = true;
7777 if queued {
7778 mux.context.streams[0]
7779 .back
7780 .out
7781 .push_back(kawa::OutBlock::Store(kawa::Store::Static(b"pending")));
7782 }
7783 arm_frontend_at(&mut mux, Some(frontend_deadline));
7784 (mux, backend_token, peer, backend_peer)
7785 }
7786
7787 /// A backend timeout on a response whose bytes wait for a slow client
7788 /// does not cut it: Sōzu stopped reading that backend itself, so the
7789 /// backend is not inactive. The stream stays linked, the backend is
7790 /// re-armed, and the FRONTEND deadline is left exactly where it was, so
7791 /// a client that stopped reading still meets `front_timeout`
7792 /// (sozu-proxy/sozu#1906).
7793 ///
7794 /// To SEE THIS RED: drop the `response_waits_on_client` arm of the
7795 /// backend-timeout branch of `Mux::timeout_inner` — the stream is cut
7796 /// with `backend_response_timeout` — or drop its early
7797 /// `return StateResult::Continue`, and the frontend deadline moves.
7798 #[test]
7799 fn a_backend_timeout_defers_to_the_client_while_response_bytes_wait_for_it() {
7800 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7801 let frontend_deadline = Instant::now() + Duration::from_secs(45);
7802 let (mut mux, backend_token, _peer, _backend_peer) = mux_with_response_in_progress(
7803 &pool,
7804 true,
7805 frontend_deadline,
7806 Instant::now() - Duration::from_secs(1),
7807 );
7808 mux.reschedule();
7809 let mut metrics = SessionMetrics::new(None);
7810
7811 let before = Instant::now();
7812 let result = mux.timeout(backend_token, &mut metrics);
7813
7814 assert_eq!(result, StateResult::Continue);
7815 assert_eq!(
7816 mux.context.streams[0].state,
7817 StreamState::Linked(backend_token),
7818 "a response waiting on the client must stay linked to its backend"
7819 );
7820 assert_eq!(mux.context.streams[0].context.access_log_message, None);
7821 assert!(
7822 mux.router
7823 .backends
7824 .get(&backend_token)
7825 .and_then(|backend| backend.poll_timeout())
7826 .is_some_and(|deadline| deadline > before),
7827 "the deferred backend timer must be re-armed"
7828 );
7829 assert_eq!(
7830 mux.frontend.poll_timeout(),
7831 Some(frontend_deadline),
7832 "a deferral must not push the frontend deadline out"
7833 );
7834 }
7835
7836 /// A frontend timeout on a response the client has fully taken does not
7837 /// cut it: the response waits on its backend, and `back_timeout` bounds
7838 /// that wait. The stream stays linked to its backend and the frontend is
7839 /// re-armed (sozu-proxy/sozu#1906).
7840 ///
7841 /// To SEE THIS RED: drop the `!response_waits_on_client` arm of the
7842 /// frontend-timeout branch of `Mux::timeout_inner`.
7843 #[test]
7844 fn a_frontend_timeout_defers_to_the_backend_while_the_client_waits_on_it() {
7845 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7846 let (mut mux, backend_token, _peer, _backend_peer) = mux_with_response_in_progress(
7847 &pool,
7848 false,
7849 Instant::now() - Duration::from_secs(1),
7850 Instant::now() + Duration::from_secs(20),
7851 );
7852 let mut metrics = SessionMetrics::new(None);
7853
7854 let before = Instant::now();
7855 let result = mux.timeout(Token(0), &mut metrics);
7856
7857 assert_eq!(result, StateResult::Continue);
7858 assert_eq!(
7859 mux.context.streams[0].state,
7860 StreamState::Linked(backend_token),
7861 "a response waiting on its backend must stay linked"
7862 );
7863 assert_eq!(mux.context.streams[0].context.access_log_message, None);
7864 assert!(
7865 mux.frontend
7866 .poll_timeout()
7867 .is_some_and(|deadline| deadline > before),
7868 "the deferred frontend timer must be re-armed"
7869 );
7870 }
7871
7872 /// The complement: a frontend timeout on a response whose bytes wait for
7873 /// the client still cuts it, so a client that stopped reading cannot
7874 /// hold the session through the backend-side deferral
7875 /// (sozu-proxy/sozu#1906).
7876 #[test]
7877 fn a_frontend_timeout_still_cuts_a_response_waiting_on_the_client() {
7878 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7879 let (mut mux, backend_token, _peer, _backend_peer) = mux_with_response_in_progress(
7880 &pool,
7881 true,
7882 Instant::now() - Duration::from_secs(1),
7883 Instant::now() + Duration::from_secs(20),
7884 );
7885 let mut metrics = SessionMetrics::new(None);
7886
7887 let _ = mux.timeout(Token(0), &mut metrics);
7888
7889 assert_eq!(
7890 mux.context.streams[0].context.access_log_message,
7891 Some("client_timeout_during_response")
7892 );
7893 assert_ne!(
7894 mux.context.streams[0].state,
7895 StreamState::Linked(backend_token),
7896 "the cut stream must be unlinked"
7897 );
7898 }
7899
7900 /// The other half of the loose-end proof. In the "response terminated and
7901 /// fully proxied" arm of the backend-timeout branch there is no
7902 /// `unlink_stream` call of its own: the only thing that evicts the stream
7903 /// from `context.backend_streams` on this arm is the unconditional
7904 /// `backend.end_stream(...)` at the bottom of the loop. As above, this test
7905 /// sets `h1.stream = Some(0)` so `ConnectionH1::end_stream` takes its
7906 /// matched path; the mismatch arm is unreached, and guarded by
7907 /// `debug_assert_h1_owns_stream` rather than by a test. That arm also
7908 /// leaves `should_close` true and `should_write` false, which is exactly
7909 /// the `delay_close_for_frontend_flush("timeout")` path the issue named as
7910 /// the one that can return `Continue` with only the frontend re-armed.
7911 ///
7912 /// To SEE THIS RED: delete `backend.end_stream(stream_id, &mut
7913 /// self.context);` from the bottom of that loop.
7914 #[test]
7915 fn a_completed_backend_response_timeout_unlinks_through_end_stream_alone() {
7916 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7917 let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7918 let mut metrics = SessionMetrics::new(None);
7919 let backend_token = Token(1);
7920
7921 let (mut connection, _backend_peer) =
7922 test_backend_connection(&mut mux, Duration::from_secs(30));
7923 let Connection::H1(h1) = &mut connection else {
7924 unreachable!("new_h1_client builds an H1 connection")
7925 };
7926 h1.stream = Some(0);
7927 // A backend whose response was fully proxied is connected; see
7928 // `a_backend_timeout_leaves_no_linked_stream_behind_on_the_close_path`.
7929 mark_connected(h1);
7930 park_backend_deadline(&mut connection, Instant::now() - Duration::from_secs(1));
7931 mux.router.backends.insert(backend_token, connection);
7932 mux.reschedule();
7933
7934 mux.context.link_stream(0, backend_token);
7935 // Terminated AND fully proxied: the arm that does nothing but fall
7936 // through to `end_stream`.
7937 mux.context.streams[0].back.consumed = true;
7938 mux.context.streams[0].back.parsing_phase = kawa::ParsingPhase::Terminated;
7939 assert!(
7940 mux.context.streams[0].back.is_terminated()
7941 && mux.context.streams[0].back.is_completed(),
7942 "precondition: this test needs the terminated-and-completed arm"
7943 );
7944
7945 let _ = mux.timeout(backend_token, &mut metrics);
7946
7947 assert!(
7948 mux.context
7949 .backend_streams
7950 .get(&backend_token)
7951 .is_none_or(|ids| ids.is_empty()),
7952 "end_stream is the sole eviction point on this arm; without it the \
7953 backend reaches the shared tail still holding a linked stream and \
7954 with no timer of its own"
7955 );
7956 assert_eq!(
7957 mux.context.streams[0].context.access_log_message, None,
7958 "precondition: a fully proxied response is not a timeout outcome, \
7959 so this arm leaves should_write false and reaches \
7960 delay_close_for_frontend_flush"
7961 );
7962 }
7963
7964 /// Q7 of sozu-proxy/sozu#1340: a stream runs on the listener
7965 /// configuration that was in force when its request arrived, and the next
7966 /// request on the same connection picks up a reload that landed in
7967 /// between. The staleness window is therefore exactly one request, and
7968 /// this test is where that window is pinned — see `LIFECYCLE.md` §2.5.
7969 ///
7970 /// Both halves matter and they fail for opposite reasons:
7971 ///
7972 /// - freezing the listener at [`Context::new`] would satisfy the first
7973 /// half and break the second — the reload would never reach the
7974 /// connection at all, which is the construction-time snapshot the
7975 /// maintainer rejected because hot reconfiguration is the point of this
7976 /// proxy;
7977 /// - borrowing the listener live on the datapath satisfies the second and
7978 /// breaks the first — a request already in flight would answer under a
7979 /// cookie name, an `X-Real-IP` policy or an error page installed after
7980 /// it started.
7981 ///
7982 /// To SEE THIS RED: give `Context` back an `elide_x_real_ip: bool` field
7983 /// set in [`Context::new`] from `listener.borrow().get_elide_x_real_ip()`,
7984 /// and make [`Context::create_stream`] pass `self.elide_x_real_ip` to
7985 /// `HttpContext::new` instead of `listener.get_elide_x_real_ip()`. The
7986 /// second stream then still carries the pre-reload value.
7987 #[test]
7988 fn a_listener_reload_between_two_streams_moves_only_the_second() {
7989 setup_test_logger!();
7990 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 10, 16_384)));
7991 let mut context = test_context(&pool);
7992
7993 let before = context.listener.borrow().get_answers().clone();
7994 let first = context
7995 .create_stream(Ulid::generate(), 1 << 16)
7996 .expect("the first stream must check out its buffers");
7997
7998 // The operator reloads the listener while the first request is still
7999 // in flight. `reload` publishes a new answer registry under a new
8000 // `Rc`, exactly as `HttpListener::update_config` does.
8001 let after = context.listener.borrow_mut().reload("RELOADEDID", true);
8002 assert!(
8003 !Rc::ptr_eq(&before, &after),
8004 "precondition: the reload must PUBLISH a registry, not rewrite the \
8005 captured one — otherwise this test cannot tell a snapshot from a \
8006 shared handle"
8007 );
8008
8009 let second = context
8010 .create_stream(Ulid::generate(), 1 << 16)
8011 .expect("the second stream must check out its buffers");
8012 assert_ne!(
8013 first, second,
8014 "precondition: the second request must land in its own slot, not \
8015 recycle the first one's"
8016 );
8017
8018 // The next request picks the reload up.
8019 assert_eq!(
8020 context.streams[second].context.sticky_name, "RELOADEDID",
8021 "a request arriving after the reload must use the new sticky name"
8022 );
8023 assert!(
8024 context.streams[second].context.elide_x_real_ip,
8025 "a request arriving after the reload must use the new X-Real-IP \
8026 policy: a listener frozen at Context::new never sees a reload on \
8027 a connection that is already open"
8028 );
8029 assert!(
8030 Rc::ptr_eq(&context.streams[second].answers, &after),
8031 "a request arriving after the reload must render from the newly \
8032 published answer registry"
8033 );
8034
8035 // The request already in flight does not.
8036 assert_eq!(
8037 context.streams[first].context.sticky_name, TEST_STICKY_NAME,
8038 "a request in flight keeps the sticky name it started under"
8039 );
8040 assert!(
8041 !context.streams[first].context.elide_x_real_ip,
8042 "a request in flight keeps the X-Real-IP policy it started under"
8043 );
8044 assert!(
8045 Rc::ptr_eq(&context.streams[first].answers, &before),
8046 "a request in flight keeps the answer registry it captured: a live \
8047 listener borrow on the write, reset or access-log path would hand \
8048 it a page installed after it started"
8049 );
8050 }
8051
8052 // ── peer-address snapshot in the MUX log envelope ────────────────────
8053 //
8054 // `log_context!` / `log_context_lite!` render `peer=` from
8055 // `Connection::peer_address` (`lib/src/protocol/mux/connection.rs`), the
8056 // snapshot the connection took at construction — not a live
8057 // `getpeername(2)`. Both tests below make the two answers differ on
8058 // purpose: the socket is genuinely connected to a loopback listener, so a
8059 // live lookup succeeds and would print `127.0.0.1:<port>`; only a macro
8060 // reading the snapshot can print `10.0.0.42:12345`.
8061 //
8062 // That gap is the PROXY-protocol symptom in miniature — the frontend's real
8063 // peer is the load balancer while the snapshot holds the advertised client
8064 // — and the same read is what keeps the slot populated after the peer's
8065 // RST, when `getpeername(2)` answers ENOTCONN and a live lookup collapses
8066 // to `None` on exactly the error lines an operator is reading. Until this
8067 // change the `MUX` envelope printed one peer while the `MUX-H1`, `MUX-H2`,
8068 // `SOCKET` and `HTTPS` lines of the same ULID printed another.
8069 //
8070 // Mirrors `h2::tests::log_context_renders_the_cached_peer_not_a_live_lookup`
8071 // and `h1::tests::log_context_renders_the_proxy_advertised_peer_not_the_load_balancer`,
8072 // which pin the same slot one layer down in the per-protocol envelopes.
8073 //
8074 // A counting test would not discriminate here, unlike
8075 // `h1::tests::log_context_reads_the_peer_address_once_per_connection`:
8076 // before this change the macros never reached `SocketHandler::peer_addr` at
8077 // all — they went through `Connection::socket()` to mio's inherent method —
8078 // so a trait-call counter reads zero on both sides. The assertions are
8079 // therefore on the rendered VALUE.
8080
8081 /// A live, established loopback connection plus the address
8082 /// `getpeername(2)` reports for it. The listener is returned so the
8083 /// connection stays up for the whole test: these tests assert the snapshot
8084 /// wins even while the live lookup is perfectly healthy, so a half-dead
8085 /// socket would weaken them rather than strengthen them.
8086 ///
8087 /// Connects with a blocking `std` socket and converts afterwards rather
8088 /// than reusing `connected_socket`, because `mio::net::TcpStream::connect`
8089 /// returns before the handshake completes and the live lookup below is a
8090 /// load-bearing premise, not a convenience.
8091 fn connected_loopback_stream() -> (std::net::TcpListener, TcpStream, SocketAddr) {
8092 let listener = std::net::TcpListener::bind("127.0.0.1:0")
8093 .expect("test listener must bind to a loopback port");
8094 let live_peer = listener
8095 .local_addr()
8096 .expect("test listener must report its local address");
8097 let stream =
8098 std::net::TcpStream::connect(live_peer).expect("loopback connect must complete");
8099 stream
8100 .set_nonblocking(true)
8101 .expect("mio requires a nonblocking stream");
8102 (listener, TcpStream::from_std(stream), live_peer)
8103 }
8104
8105 /// Address the frontend handler snapshots. Deliberately non-loopback so it
8106 /// cannot collide with whatever ephemeral port the live lookup reports.
8107 const SNAPSHOT_PEER: &str = "10.0.0.42:12345";
8108
8109 /// An H1 frontend `Mux` whose connection snapshotted [`SNAPSHOT_PEER`]
8110 /// while its socket is really connected to `live_peer`.
8111 /// `Connection::new_h1_server` takes that snapshot itself, from the
8112 /// handler's `SocketHandler::peer_addr`, so nothing here writes the field
8113 /// behind the production path's back.
8114 ///
8115 /// An H1 frontend only because it is the cheaper of the two to build: the
8116 /// accessor under test resolves both `Connection` arms to the same
8117 /// `peer_address` field through `forward!`, and the two macros are defined
8118 /// once for every frontend.
8119 fn mux_with_snapshotted_peer(
8120 pool: &Rc<RefCell<Pool>>,
8121 stream: TcpStream,
8122 ) -> Mux<SessionTcpStream, test_support::TestListener> {
8123 let session_ulid = Ulid::generate();
8124 let socket = SessionTcpStream::new(
8125 stream,
8126 session_ulid,
8127 Some(
8128 SNAPSHOT_PEER
8129 .parse()
8130 .expect("the snapshotted peer literal must parse"),
8131 ),
8132 );
8133 Mux {
8134 configured_frontend_timeout: Duration::from_secs(60),
8135 frontend_token: Token(0),
8136 frontend: Connection::new_h1_server(session_ulid, socket, Duration::from_secs(60)),
8137 router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
8138 context: test_context(pool),
8139 session_ulid,
8140 timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(60))),
8141 backend_registry: BackendRegistry::default(),
8142 backends: Rc::default(),
8143 }
8144 }
8145
8146 /// To SEE THIS RED: in `log_context!` (mod.rs), put
8147 /// `peer = $self.frontend.socket().peer_addr().ok(),` back in place of
8148 /// `peer = $self.frontend.peer_address(),`. `Connection::socket()` hands
8149 /// back a `&mio::net::TcpStream`, so the slot resolves to mio's *inherent*
8150 /// `peer_addr` — a live `getpeername(2)` — and the rendered line carries
8151 /// the loopback address the socket is really connected to.
8152 #[test]
8153 fn log_context_renders_the_snapshotted_peer_not_a_live_lookup() {
8154 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
8155 let (_listener, stream, live_peer) = connected_loopback_stream();
8156 let mux = mux_with_snapshotted_peer(&pool, stream);
8157
8158 // Premise of the test: the live lookup is healthy and disagrees with the
8159 // snapshot. Without it the assertions below could pass for the wrong
8160 // reason (both answers happening to be the same address).
8161 assert_eq!(
8162 mux.frontend.socket().peer_addr().ok(),
8163 Some(live_peer),
8164 "the test socket must be genuinely connected, so a live lookup succeeds"
8165 );
8166 assert_ne!(
8167 SNAPSHOT_PEER,
8168 live_peer.to_string(),
8169 "the snapshotted and live addresses must differ for this test to discriminate"
8170 );
8171
8172 let rendered = log_context!(mux);
8173
8174 assert!(
8175 rendered.contains(&format!("peer=Some({SNAPSHOT_PEER})")),
8176 "the MUX peer= slot must render the snapshotted address; rendered: {rendered}"
8177 );
8178 assert!(
8179 !rendered.contains(&live_peer.to_string()),
8180 "the MUX peer= slot must not fall back to a live getpeername(2); rendered: {rendered}"
8181 );
8182 }
8183
8184 /// The lighter envelope reads the same slot. Separate from its full
8185 /// sibling because the two macros carry their own copy of the `peer` line:
8186 /// fixing one and not the other would render two different peers for one
8187 /// session depending only on which borrow the callsite happened to hold.
8188 ///
8189 /// To SEE THIS RED: in `log_context_lite!` (mod.rs), put
8190 /// `peer = $self.frontend.socket().peer_addr().ok(),` back in place of
8191 /// `peer = $self.frontend.peer_address(),`.
8192 #[test]
8193 fn log_context_lite_renders_the_snapshotted_peer_not_a_live_lookup() {
8194 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
8195 let (_listener, stream, live_peer) = connected_loopback_stream();
8196 let mux = mux_with_snapshotted_peer(&pool, stream);
8197
8198 assert_eq!(
8199 mux.frontend.socket().peer_addr().ok(),
8200 Some(live_peer),
8201 "the test socket must be genuinely connected, so a live lookup succeeds"
8202 );
8203
8204 let rendered = log_context_lite!(mux);
8205
8206 assert!(
8207 rendered.contains(&format!("peer=Some({SNAPSHOT_PEER})")),
8208 "the lite MUX peer= slot must render the snapshotted address; rendered: {rendered}"
8209 );
8210 assert!(
8211 !rendered.contains(&live_peer.to_string()),
8212 "the lite MUX peer= slot must not fall back to a live getpeername(2); \
8213 rendered: {rendered}"
8214 );
8215 }
8216
8217 /// `Mux::close` issues no `EPOLL_CTL_DEL` for its backend sockets: each
8218 /// stays in the epoll set until the `Mux` drops, and its last close is what
8219 /// removes it. The second assertion is the half that makes skipping the
8220 /// deregister safe — the socket is not duplicated anywhere, so dropping the
8221 /// `Mux` really does take it out of the epoll set.
8222 ///
8223 /// To SEE THIS RED: make `Mux::close`'s backend loop issue an
8224 /// `EPOLL_CTL_DEL` after `let socket = client.socket_mut();`. `L7Proxy`
8225 /// no longer carries `deregister_socket` (sozu#1615), so restore it for
8226 /// the experiment — its `HttpProxy` body was
8227 /// `self.registry.deregister(tcp_stream)` — and call
8228 /// `let _ = proxy_borrow.deregister_socket(socket);` there. The first
8229 /// assertion then fails.
8230 #[cfg(target_os = "linux")]
8231 #[test]
8232 fn close_leaves_backend_sockets_to_their_last_close() {
8233 use std::os::fd::AsRawFd;
8234
8235 use mio::Interest;
8236
8237 use super::test_support::epoll_watches;
8238 use crate::{backends::Backend, http::HttpProxy, socket::SessionTcpStream};
8239
8240 let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
8241 let (mut mux, _front_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(30));
8242 let parts =
8243 crate::testing::prebuild_server(4, 16_384, false).expect("test server must build");
8244 let registry = parts
8245 .registry
8246 .try_clone()
8247 .expect("the test registry must clone");
8248 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(HttpProxy::new(
8249 parts.registry,
8250 parts.sessions,
8251 parts.pool,
8252 parts.backends,
8253 )));
8254
8255 let (socket, _back_peer) = connected_socket();
8256 let backend = Rc::new(RefCell::new(Backend::new(
8257 "test-backend",
8258 "127.0.0.1:2".parse().expect("backend address must parse"),
8259 None,
8260 None,
8261 None,
8262 )));
8263 let backend_id = mux.backend_registry.id_for(&backend);
8264 let session_ulid = mux.session_ulid;
8265 let mut connection = Connection::new_h1_client(
8266 session_ulid,
8267 SessionTcpStream::new(socket, session_ulid, None),
8268 "test-cluster".into(),
8269 backend_id,
8270 Duration::from_secs(30),
8271 );
8272 let back_fd = connection.socket().as_raw_fd();
8273 registry
8274 .register(
8275 connection.socket_mut(),
8276 Token(7),
8277 Interest::READABLE | Interest::WRITABLE,
8278 )
8279 .expect("the backend socket must register");
8280 mux.router.backends.insert(Token(7), connection);
8281 assert!(
8282 epoll_watches(®istry, back_fd),
8283 "precondition: the backend socket is in the epoll set"
8284 );
8285
8286 mux.close(proxy, &mut SessionMetrics::new(None));
8287 assert!(
8288 epoll_watches(®istry, back_fd),
8289 "Mux::close must not deregister a backend socket it is about to \
8290 close: the close removes it, the EPOLL_CTL_DEL is a wasted syscall"
8291 );
8292
8293 drop(mux);
8294 assert!(
8295 !epoll_watches(®istry, back_fd),
8296 "dropping the Mux closes the backend socket's only descriptor, \
8297 which must take it out of the epoll set"
8298 );
8299 drop(parts.event_loop);
8300 }
8301
8302 // ── a read that meets the EOF records HUP ──────────────────────────
8303
8304 /// An H1 backend read that meets the EOF records HUP; the same read on
8305 /// the frontend leaves HUP to the event loop.
8306 ///
8307 /// Both peers write bytes and their FIN before the read. Since
8308 /// sozu-proxy/sozu#1606 a read stops on a short read, as HAProxy does
8309 /// (`src/raw_sock.c`, `fd_cant_recv` when `ret < try`): the first
8310 /// `readable` delivers the bytes and stops before the EOF, and the EOF is
8311 /// read by the next `readable`, which the poller's `EPOLLRDHUP` edge (or a
8312 /// HUP already in the same event) triggers. That second read is the one
8313 /// that meets the EOF and must record HUP.
8314 ///
8315 /// TO SEE THIS RED: delete the HUP insertion from `ConnectionH1::readable`
8316 /// (`h1.rs`). The backend assertion fails with `a backend read that met the
8317 /// EOF must record HUP`.
8318 #[test]
8319 fn an_h1_backend_eof_read_records_hup_and_a_frontend_one_does_not() {
8320 use std::io::Write;
8321
8322 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8323 let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8324 let (mut backend, mut backend_peer) =
8325 test_backend_connection(&mut mux, Duration::from_secs(30));
8326 let Connection::H1(h1) = &mut backend else {
8327 unreachable!("new_h1_client builds an H1 connection")
8328 };
8329 h1.stream = Some(0);
8330 mux.context.link_stream(0, Token(1));
8331
8332 backend_peer
8333 .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello")
8334 .expect("the backend peer must write its response");
8335 backend_peer
8336 .shutdown(std::net::Shutdown::Write)
8337 .expect("the backend peer must send its FIN");
8338 std::thread::sleep(Duration::from_millis(20));
8339 backend.readiness_mut().event = Ready::READABLE;
8340 let _ = backend.readable(&mut mux.context, EndpointServer(&mut mux.frontend));
8341 assert!(
8342 mux.context.streams[0].back.is_terminated(),
8343 "the bytes read before the EOF must be parsed"
8344 );
8345 assert!(
8346 !backend.readiness().event.is_hup(),
8347 "the short read stops before the EOF, so it has not met it (#1606)"
8348 );
8349 let _ = backend.readable(&mut mux.context, EndpointServer(&mut mux.frontend));
8350 assert!(
8351 backend.readiness().event.is_hup(),
8352 "a backend read that met the EOF must record HUP"
8353 );
8354
8355 // The same stream carries the frontend's request.
8356 mux.context.streams[0].state = StreamState::Idle;
8357 frontend_peer
8358 .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\n\r\n")
8359 .expect("the client must write its request");
8360 frontend_peer
8361 .shutdown(std::net::Shutdown::Write)
8362 .expect("the client must send its FIN");
8363 std::thread::sleep(Duration::from_millis(20));
8364 mux.frontend.readiness_mut().event = Ready::READABLE;
8365 // The first read takes the request and stops short (#1606); the
8366 // second meets the EOF.
8367 let _ = mux
8368 .frontend
8369 .readable(&mut mux.context, EndpointClient(&mut mux.router));
8370 let _ = mux
8371 .frontend
8372 .readable(&mut mux.context, EndpointClient(&mut mux.router));
8373 assert!(
8374 !mux.frontend.readiness().event.is_readable(),
8375 "precondition: the frontend read reached the EOF"
8376 );
8377 assert!(
8378 !mux.frontend.readiness().event.is_hup(),
8379 "a frontend read must leave HUP to the event loop"
8380 );
8381 }
8382
8383 /// A `ProxySession` for a `Mux::ready` pass that links no stream: only
8384 /// `Mux::dial_backend` reads the session, and nothing in these tests
8385 /// dials.
8386 struct NoDialSession;
8387
8388 impl ProxySession for NoDialSession {
8389 fn protocol(&self) -> Protocol {
8390 unreachable!("a Mux pass that dials nothing never asks the session")
8391 }
8392 fn ready(&mut self, _session: Rc<RefCell<dyn ProxySession>>) -> SessionIsToBeClosed {
8393 unreachable!("a Mux pass that dials nothing never asks the session")
8394 }
8395 fn update_readiness(&mut self, _token: Token, _events: Ready) {
8396 unreachable!("a Mux pass that dials nothing never asks the session")
8397 }
8398 fn close(&mut self) {
8399 unreachable!("a Mux pass that dials nothing never asks the session")
8400 }
8401 fn timeout(&mut self, _t: Token) -> SessionIsToBeClosed {
8402 unreachable!("a Mux pass that dials nothing never asks the session")
8403 }
8404 fn last_event(&self) -> Instant {
8405 unreachable!("a Mux pass that dials nothing never asks the session")
8406 }
8407 fn print_session(&self) {
8408 unreachable!("a Mux pass that dials nothing never asks the session")
8409 }
8410 fn frontend_token(&self) -> Token {
8411 unreachable!("a Mux pass that dials nothing never asks the session")
8412 }
8413 fn shutting_down(&mut self) -> SessionIsToBeClosed {
8414 unreachable!("a Mux pass that dials nothing never asks the session")
8415 }
8416 }
8417
8418 /// An `L7Proxy` for a session that only ever removes backend sessions:
8419 /// the dead-backend sweep of `Mux::ready_inner` and `Mux::close` call
8420 /// `remove_session` once per backend and nothing else.
8421 struct RemoveOnlyProxy;
8422
8423 impl L7Proxy for RemoveOnlyProxy {
8424 fn kind(&self) -> sozu_command::proto::command::ListenerType {
8425 unreachable!("closing a backend only removes its session")
8426 }
8427 fn register_socket(
8428 &self,
8429 _socket: &mut TcpStream,
8430 _token: Token,
8431 _interest: Interest,
8432 ) -> Result<(), std::io::Error> {
8433 unreachable!("closing a backend only removes its session")
8434 }
8435 fn add_session(&self, _session: Rc<RefCell<dyn ProxySession>>) -> Token {
8436 unreachable!("closing a backend only removes its session")
8437 }
8438 fn remove_session(&self, _token: Token) -> bool {
8439 true
8440 }
8441 fn clusters(
8442 &self,
8443 ) -> &HashMap<sozu_command::state::ClusterId, sozu_command::proto::command::Cluster>
8444 {
8445 unreachable!("closing a backend only removes its session")
8446 }
8447 fn sessions(&self) -> Rc<RefCell<crate::server::SessionManager>> {
8448 unreachable!("closing a backend only removes its session")
8449 }
8450 }
8451
8452 /// An `L7Proxy` for a pass that must have released every stream before
8453 /// the `pending_links` pass of `Mux::ready_inner` runs: removing a
8454 /// backend session is allowed, and reaching the routing view or
8455 /// registering a backend names the regression.
8456 struct NoRoutingProxy;
8457
8458 impl L7Proxy for NoRoutingProxy {
8459 fn kind(&self) -> sozu_command::proto::command::ListenerType {
8460 unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8461 }
8462 fn register_socket(
8463 &self,
8464 _socket: &mut TcpStream,
8465 _token: Token,
8466 _interest: Interest,
8467 ) -> Result<(), std::io::Error> {
8468 unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8469 }
8470 fn add_session(&self, _session: Rc<RefCell<dyn ProxySession>>) -> Token {
8471 unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8472 }
8473 fn remove_session(&self, _token: Token) -> bool {
8474 true
8475 }
8476 fn clusters(
8477 &self,
8478 ) -> &HashMap<sozu_command::state::ClusterId, sozu_command::proto::command::Cluster>
8479 {
8480 unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8481 }
8482 fn sessions(&self) -> Rc<RefCell<crate::server::SessionManager>> {
8483 unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8484 }
8485 }
8486
8487 /// Read what the peer has been sent until it sees the FIN, or until the
8488 /// bytes read end with a non-empty `until`, and report whether the FIN was
8489 /// seen. Bounded: a peer that sees neither within the deadline reports
8490 /// what it has.
8491 fn drain_peer(peer: &mut std::net::TcpStream, until: &[u8]) -> (Vec<u8>, bool) {
8492 use std::io::Read;
8493 let deadline = Instant::now() + Duration::from_secs(2);
8494 let mut bytes = Vec::new();
8495 let mut chunk = [0u8; 4096];
8496 loop {
8497 match peer.read(&mut chunk) {
8498 Ok(0) => return (bytes, true),
8499 Ok(n) => {
8500 bytes.extend_from_slice(&chunk[..n]);
8501 if !until.is_empty() && bytes.ends_with(until) {
8502 return (bytes, false);
8503 }
8504 }
8505 Err(e) if e.kind() == ErrorKind::WouldBlock => {
8506 if Instant::now() >= deadline {
8507 return (bytes, false);
8508 }
8509 std::thread::sleep(Duration::from_millis(5));
8510 }
8511 Err(e) => panic!("the test peer failed to read: {e}"),
8512 }
8513 }
8514 }
8515
8516 /// #1610: a session whose backend connections die one pass after another
8517 /// sizes its dead-backend list once, then reuses it.
8518 ///
8519 /// Each pass closes one idle keep-alive backend whose event carries HUP,
8520 /// the shape a backend that closes after every response leaves on a
8521 /// keep-alive frontend. Past the first such pass, a pass allocates
8522 /// nothing at all, so the list must not either.
8523 ///
8524 /// TO SEE THIS RED: in `Mux::ready_inner`, collect the dead tokens in a
8525 /// local `Vec::new()` again instead of `self.router.dead_backends`. Every
8526 /// pass that closes a backend then allocates the list.
8527 #[test]
8528 fn successive_dead_backends_reuse_one_dead_backend_list() {
8529 use crate::test_allocations::allocations;
8530
8531 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8532 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8533 mux.frontend.readiness_mut().event = Ready::EMPTY;
8534 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
8535 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
8536 let mut metrics = SessionMetrics::new(None);
8537
8538 let mut close_one = |mux: &mut Mux<_, _>, token: Token| {
8539 let (mut connection, backend_peer) =
8540 test_backend_connection(mux, Duration::from_secs(30));
8541 let Connection::H1(h1) = &mut connection else {
8542 unreachable!("new_h1_client builds an H1 connection")
8543 };
8544 if let Position::Client(_, _, status) = &mut h1.position {
8545 *status = BackendStatus::KeepAlive;
8546 }
8547 h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8548 h1.readiness.event = Ready::HUP;
8549 mux.router.backends.insert(token, connection);
8550 drop(backend_peer);
8551
8552 let before = allocations();
8553 let _ = mux.ready(session.clone(), proxy.clone(), &mut metrics);
8554 let allocated = allocations() - before;
8555 assert!(
8556 !mux.router.backends.contains_key(&token),
8557 "premise: the HUP must close the backend in this pass"
8558 );
8559 allocated
8560 };
8561
8562 // Warm-up: the session's first dead backend sizes the list, and the
8563 // thread's first pass initialises state that outlives the session.
8564 close_one(&mut mux, Token(1));
8565 for token in [Token(2), Token(3)] {
8566 assert_eq!(
8567 close_one(&mut mux, token),
8568 0,
8569 "a later pass that closes a backend must not allocate"
8570 );
8571 }
8572 }
8573
8574 /// Issues #1603 and #1606: a backend whose last bytes and FIN reached the
8575 /// kernel before `epoll_wait` returned is closed in that same pass, after
8576 /// its bytes were parsed and forwarded.
8577 ///
8578 /// The event carries READABLE and HUP together, as
8579 /// `Ready::from(&mio::event::Event)` builds it for an `EPOLLIN|EPOLLRDHUP`
8580 /// edge (`data_and_fin_in_one_edge_keep_readable_until_the_eof_is_read`
8581 /// shows a real socket reporting both in one edge). The first read stops
8582 /// short before the EOF (#1606, as HAProxy does in `src/raw_sock.c`), and
8583 /// the dead-backend check of `Mux::ready_inner`, which reads the HUP the
8584 /// edge already carried, closes the backend on the pass's next inner
8585 /// iteration. The bytes read before the EOF are forwarded first. (When the
8586 /// stream still needs the EOF, as a close-delimited body does,
8587 /// `update_readiness_after_read` keeps READABLE so it is read first:
8588 /// `data_and_fin_in_one_edge_keep_readable_until_the_eof_is_read`.)
8589 ///
8590 /// TO SEE THIS RED: drop `readiness.event.is_hup() ||` from `dead` in
8591 /// `Mux::ready_inner`. The backend then survives the pass: `the backend
8592 /// that read its EOF must be closed in the same pass`.
8593 #[test]
8594 fn a_backend_eof_read_with_its_last_bytes_closes_the_backend_in_the_same_pass() {
8595 use std::io::Write;
8596
8597 const RESPONSE: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello";
8598
8599 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8600 let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8601 let backend_token = Token(1);
8602 let (mut connection, mut backend_peer) =
8603 test_backend_connection(&mut mux, Duration::from_secs(30));
8604 let Connection::H1(h1) = &mut connection else {
8605 unreachable!("new_h1_client builds an H1 connection")
8606 };
8607 h1.stream = Some(0);
8608 if let Position::Client(_, _, status) = &mut h1.position {
8609 *status = BackendStatus::Connected;
8610 }
8611 // The request is already written; the backend waits to read. Data and
8612 // FIN share one edge.
8613 h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8614 h1.readiness.event = Ready::READABLE | Ready::HUP;
8615 mux.router.backends.insert(backend_token, connection);
8616 mux.context.link_stream(0, backend_token);
8617 mux.frontend.readiness_mut().event = Ready::EMPTY;
8618
8619 backend_peer
8620 .write_all(RESPONSE)
8621 .expect("the backend peer must write its response");
8622 backend_peer
8623 .shutdown(std::net::Shutdown::Write)
8624 .expect("the backend peer must send its FIN");
8625 // Loopback delivers both before `write`/`shutdown` return; the margin
8626 // only guards a scheduler that defers the softirq.
8627 std::thread::sleep(Duration::from_millis(20));
8628
8629 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
8630 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
8631 let mut metrics = SessionMetrics::new(None);
8632 let _ = mux.ready(session, proxy, &mut metrics);
8633
8634 assert!(
8635 !mux.router.backends.contains_key(&backend_token),
8636 "the backend that read its EOF must be closed in the same pass"
8637 );
8638 let (forwarded, _) = drain_peer(&mut frontend_peer, b"hello");
8639 assert!(
8640 forwarded.ends_with(b"hello"),
8641 "the bytes read together with the EOF must still reach the client, \
8642 got {:?}",
8643 String::from_utf8_lossy(&forwarded)
8644 );
8645 }
8646
8647 /// Issues #1603 and #1606: the FIN lands AFTER the `epoll_wait` that
8648 /// woke the pass, so the event carries READABLE without HUP. The pass
8649 /// reads the response in one short read and stops before the EOF (#1606):
8650 /// it does not issue the `recv` that would find the FIN, exactly as
8651 /// HAProxy stops on a short read and lets the poller report the EOF
8652 /// (`src/raw_sock.c`, `fd_cant_recv` when `ret < try`). The backend is
8653 /// therefore closed one `epoll_wait` round later, on the `EPOLLRDHUP` edge
8654 /// the FIN queued — and not later: that edge is already pending when the
8655 /// pass ends, so a zero-timeout poll returns it.
8656 ///
8657 /// That round is the price of not issuing a `recv` after every short
8658 /// read, which answers EAGAIN whenever no FIN follows (the common case).
8659 /// Real loopback socket, registered edge-triggered with mio.
8660 #[test]
8661 fn a_fin_after_epoll_wait_closes_the_backend_on_the_next_round_at_the_latest() {
8662 use std::io::Write;
8663
8664 const RESPONSE: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello";
8665
8666 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8667 let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8668 let backend_token = Token(1);
8669 let (mut connection, mut backend_peer) =
8670 test_backend_connection(&mut mux, Duration::from_secs(30));
8671 let mut poll = mio::Poll::new().expect("a poll instance must open");
8672 let mut events = mio::Events::with_capacity(8);
8673 let Connection::H1(h1) = &mut connection else {
8674 unreachable!("new_h1_client builds an H1 connection")
8675 };
8676 poll.registry()
8677 .register(
8678 &mut h1.socket.stream,
8679 backend_token,
8680 mio::Interest::READABLE,
8681 )
8682 .expect("the backend socket must register");
8683 h1.stream = Some(0);
8684 if let Position::Client(_, _, status) = &mut h1.position {
8685 *status = BackendStatus::Connected;
8686 }
8687 h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8688
8689 // The response arrives; `epoll_wait` reports it, without a FIN.
8690 backend_peer
8691 .write_all(RESPONSE)
8692 .expect("the backend peer must write its response");
8693 poll.poll(&mut events, Some(Duration::from_secs(5)))
8694 .expect("poll must succeed");
8695 let event = events
8696 .iter()
8697 .find(|event| event.token() == backend_token)
8698 .expect("the response must raise an edge");
8699 assert!(
8700 event.is_readable() && !event.is_read_closed(),
8701 "premise: no FIN yet in the edge that wakes the pass"
8702 );
8703 h1.readiness.event = Ready::from(event);
8704 mux.router.backends.insert(backend_token, connection);
8705 mux.context.link_stream(0, backend_token);
8706 // A request received whole, from a client that asked to close: the
8707 // response closes the frontend without a lingering drain (which
8708 // would end this backend's stream at once).
8709 mux.context.streams[0].front.parsing_phase = kawa::ParsingPhase::Terminated;
8710 mux.context.streams[0].context.keep_alive_frontend = false;
8711 mux.frontend.readiness_mut().event = Ready::EMPTY;
8712
8713 // The FIN lands after `epoll_wait` returned, before the pass reads.
8714 backend_peer
8715 .shutdown(std::net::Shutdown::Write)
8716 .expect("the backend peer must send its FIN");
8717 std::thread::sleep(Duration::from_millis(20));
8718
8719 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
8720 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
8721 let mut metrics = SessionMetrics::new(None);
8722 let _ = mux.ready(session.clone(), proxy.clone(), &mut metrics);
8723 let (forwarded, _) = drain_peer(&mut frontend_peer, b"hello");
8724 assert!(
8725 forwarded.ends_with(b"hello"),
8726 "the response must reach the client in the first pass, got {:?}",
8727 String::from_utf8_lossy(&forwarded)
8728 );
8729 assert!(
8730 mux.router.backends.contains_key(&backend_token),
8731 "the short read stopped before the EOF: the close waits one round (#1606)"
8732 );
8733
8734 // One round: the RDHUP edge is already queued, a zero-timeout poll
8735 // returns it.
8736 poll.poll(&mut events, Some(Duration::ZERO))
8737 .expect("poll must succeed");
8738 let event = events
8739 .iter()
8740 .find(|event| event.token() == backend_token)
8741 .expect("the FIN's edge must already be pending when the pass ends");
8742 assert!(event.is_read_closed(), "the pending edge is the FIN");
8743 mux.update_readiness(backend_token, Ready::from(event));
8744 let _ = mux.ready(session, proxy, &mut metrics);
8745 assert!(
8746 !mux.router.backends.contains_key(&backend_token),
8747 "the backend must be closed on that round, not later"
8748 );
8749 }
8750
8751 // ── a backend that closed with its response still buffered ─────────
8752
8753 /// Drive one H1 exchange whose backend wrote its whole response and
8754 /// closed before a slow client read it, and return the number of body
8755 /// bytes the client received, the body length, the `SessionResult` of
8756 /// the first `Mux::ready` pass, and how many times the exchange reached
8757 /// the loop limit (`LOOP_LIMIT_HITS`).
8758 ///
8759 /// The backend's last bytes and its FIN wait in the kernel while the
8760 /// stream's response buffer is full and the client's socket would block:
8761 /// the dead-backend check keeps the backend for its buffered bytes, and
8762 /// nothing in the pass can progress until the client reads. With
8763 /// `client_half_closed`, the client also shut its write side after its
8764 /// request, as `test_plain_client_half_close_after_request_receives_large_response`
8765 /// does.
8766 fn slow_client_after_backend_close(
8767 client_half_closed: bool,
8768 ) -> (usize, usize, SessionResult, u32) {
8769 use std::io::{Read, Write};
8770
8771 const BODY: usize = 160 * 1024;
8772 let head = format!("HTTP/1.1 200 OK\r\nContent-Length: {BODY}\r\n\r\n");
8773 let length = head.len() + BODY;
8774
8775 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8776 let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8777 // A slow client: small buffers on both ends of the frontend, so its
8778 // socket would block long before the response is out.
8779 socket2::SockRef::from(&frontend_peer)
8780 .set_recv_buffer_size(16 * 1024)
8781 .expect("the client receive buffer must be settable");
8782 socket2::SockRef::from(mux.frontend.socket())
8783 .set_send_buffer_size(4096)
8784 .expect("the frontend send buffer must be settable");
8785
8786 let backend_token = Token(1);
8787 let (mut connection, mut backend_peer) =
8788 test_backend_connection(&mut mux, Duration::from_secs(30));
8789 // Room in the kernel for the whole response, so the backend can
8790 // write all of it and close before sozu reads it.
8791 socket2::SockRef::from(connection.socket())
8792 .set_recv_buffer_size(1 << 20)
8793 .expect("the backend receive buffer must be settable");
8794 socket2::SockRef::from(&backend_peer)
8795 .set_send_buffer_size(1 << 20)
8796 .expect("the backend peer send buffer must be settable");
8797 let Connection::H1(h1) = &mut connection else {
8798 unreachable!("new_h1_client builds an H1 connection")
8799 };
8800 h1.stream = Some(0);
8801 if let Position::Client(_, _, status) = &mut h1.position {
8802 *status = BackendStatus::Connected;
8803 }
8804 h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8805 h1.readiness.event = Ready::READABLE | Ready::HUP;
8806 mux.router.backends.insert(backend_token, connection);
8807 mux.context.link_stream(0, backend_token);
8808 // The request was received whole and forwarded.
8809 mux.context.streams[0].front.parsing_phase = kawa::ParsingPhase::Terminated;
8810 mux.frontend.readiness_mut().event = if client_half_closed {
8811 Ready::HUP
8812 } else {
8813 Ready::EMPTY
8814 };
8815
8816 let mut response = head.into_bytes();
8817 response.resize(length, b'b');
8818 let mut written = 0;
8819 let deadline = Instant::now() + Duration::from_secs(5);
8820 while written < response.len() {
8821 match backend_peer.write(&response[written..]) {
8822 Ok(size) => written += size,
8823 Err(e) if e.kind() == ErrorKind::WouldBlock => {
8824 assert!(
8825 Instant::now() < deadline,
8826 "premise: the kernel must hold the whole response, wrote {written}"
8827 );
8828 std::thread::sleep(Duration::from_millis(1));
8829 }
8830 Err(e) => panic!("the backend peer failed to write: {e}"),
8831 }
8832 }
8833 backend_peer
8834 .shutdown(std::net::Shutdown::Write)
8835 .expect("the backend peer must send its FIN");
8836 std::thread::sleep(Duration::from_millis(20));
8837
8838 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
8839 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
8840 let mut metrics = SessionMetrics::new(None);
8841
8842 LOOP_LIMIT_HITS.with(|hits| hits.set(0));
8843 let first = mux.ready(session.clone(), proxy.clone(), &mut metrics);
8844 let mut result = first;
8845 let mut received = Vec::new();
8846 let mut chunk = [0u8; 4096];
8847 let deadline = Instant::now() + Duration::from_secs(10);
8848 // sozu adds its own response headers: the body is what follows the
8849 // head the client received.
8850 let body_received = |received: &[u8]| {
8851 received
8852 .windows(4)
8853 .position(|window| window == b"\r\n\r\n")
8854 .map_or(0, |end| received.len() - end - 4)
8855 };
8856 loop {
8857 // The client reads what it can, then the poller reports the
8858 // frontend writable again, as epoll does once the client drains.
8859 loop {
8860 match frontend_peer.read(&mut chunk) {
8861 Ok(0) => break,
8862 Ok(size) => received.extend_from_slice(&chunk[..size]),
8863 Err(e) if e.kind() == ErrorKind::WouldBlock => break,
8864 Err(e) => panic!("the client failed to read: {e}"),
8865 }
8866 }
8867 if body_received(&received) >= BODY
8868 || result == SessionResult::Close
8869 || Instant::now() >= deadline
8870 {
8871 break;
8872 }
8873 mux.update_readiness(Token(0), Ready::WRITABLE);
8874 result = mux.ready(session.clone(), proxy.clone(), &mut metrics);
8875 }
8876 if body_received(&received) < BODY {
8877 // Whatever the last pass queued in the kernel reaches the client.
8878 let (rest, _) = drain_peer(&mut frontend_peer, &[]);
8879 received.extend_from_slice(&rest);
8880 }
8881 let hits = LOOP_LIMIT_HITS.with(|hits| hits.get());
8882 assert!(
8883 mux.context.client_bytes_out >= received.len(),
8884 "every byte the client received was counted as written to it: {} < {}",
8885 mux.context.client_bytes_out,
8886 received.len()
8887 );
8888 (body_received(&received), BODY, first, hits)
8889 }
8890
8891 /// A slow client downloading a response whose backend has already
8892 /// written all of it and closed must receive the whole response.
8893 ///
8894 /// The kept backend's HUP stays in its `filter_interest()`, which is
8895 /// never empty while HUP is in its interest: the inner loop of
8896 /// `Mux::ready_inner` used to count that HUP as work, spin to
8897 /// `MAX_LOOP_ITERATIONS` with nothing to progress, and close the session
8898 /// with the end of the body still in the stream's buffer.
8899 ///
8900 /// TO SEE THIS RED: make `Connection::has_loop_work` return
8901 /// `!readiness.filter_interest().is_empty()` again. The first pass then
8902 /// runs to `MAX_LOOP_ITERATIONS`. With `Mux::wait_for_client_at_loop_limit`
8903 /// also back to checking `frontend.has_pending_write()` alone, that pass
8904 /// returns `SessionResult::Close` and the client is short of bytes.
8905 #[test]
8906 fn a_slow_client_receives_the_whole_response_of_a_backend_that_closed() {
8907 let (received, expected, first, hits) = slow_client_after_backend_close(false);
8908 assert_eq!(
8909 hits, 0,
8910 "every pass must stop once nothing can progress, not at the loop limit"
8911 );
8912 assert_ne!(
8913 first,
8914 SessionResult::Close,
8915 "the first pass must wait for the client, not close"
8916 );
8917 assert_eq!(received, expected, "the client must receive the whole body");
8918 }
8919
8920 /// The loop budget yields only on real progress: at least one stream byte
8921 /// per budgeted iteration. A spin that trickled a few bytes, which an
8922 /// unconditional "any byte moved" rule would let yield forever, is left
8923 /// to the spin path (sozu-proxy/sozu#1906 hardening).
8924 #[test]
8925 fn only_a_byte_per_budgeted_iteration_counts_as_progress_at_the_loop_budget() {
8926 let budget = MAX_LOOP_ITERATIONS as usize;
8927 assert!(!progressed_through_loop_budget(100, 100), "nothing moved");
8928 assert!(
8929 !progressed_through_loop_budget(100, 101),
8930 "one byte is a trickle"
8931 );
8932 assert!(!progressed_through_loop_budget(0, budget - 1));
8933 assert!(progressed_through_loop_budget(0, budget));
8934 assert!(progressed_through_loop_budget(0, 512 * budget));
8935 // `Context::stream_bytes_out` wraps.
8936 assert!(progressed_through_loop_budget(usize::MAX - 10, budget));
8937 assert!(!progressed_through_loop_budget(usize::MAX, 3));
8938 }
8939
8940 /// A transfer that forwards bytes on every iteration of the inner loop of
8941 /// `Mux::ready_inner` is not closed at `MAX_LOOP_ITERATIONS`: the pass
8942 /// yields to the event loop with its readiness intact, and the next pass,
8943 /// run with no new event, carries on (sozu-proxy/sozu#1906).
8944 ///
8945 /// Both peers keep up: the kernel already holds the backend's whole
8946 /// response, and the client reads everything it is sent. Each iteration
8947 /// reads one 512-byte buffer from the backend and writes it to the
8948 /// client, so the response needs more iterations than the budget before
8949 /// any socket answers `WouldBlock`. In production the same pass needs
8950 /// both peers to keep up for about 160 MiB with 16 KiB buffers: a fast
8951 /// backend and a fast client were cut at a random offset, with a clean
8952 /// close and, on 2.2.1, no log at the default level.
8953 ///
8954 /// The passes after the first are driven exactly as
8955 /// `Server::resume_yielded_sessions` drives them: `ready` again, with no
8956 /// `update_readiness` call. Re-arming the readiness by hand would hide a
8957 /// pass that yields with an event consumed, which edge-triggered epoll
8958 /// would never deliver again.
8959 ///
8960 /// TO SEE THIS RED: make the yield arm of the loop budget in
8961 /// `Mux::ready_inner` `return SessionResult::Close` instead. The first
8962 /// pass then closes the session with the body unsent.
8963 #[test]
8964 fn a_transfer_progressing_at_the_loop_limit_yields_and_resumes_without_a_new_event() {
8965 use std::io::{Read, Write};
8966
8967 const BUFFER: usize = 512;
8968 const BODY: usize = (MAX_LOOP_ITERATIONS as usize + 1000) * BUFFER;
8969 let head = format!("HTTP/1.1 200 OK\r\nContent-Length: {BODY}\r\n\r\n");
8970 let length = head.len() + BODY;
8971
8972 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, BUFFER)));
8973 let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8974 // Room for the whole response between sozu and the client.
8975 socket2::SockRef::from(&frontend_peer)
8976 .set_recv_buffer_size(4 << 20)
8977 .expect("the client receive buffer must be settable");
8978 socket2::SockRef::from(mux.frontend.socket())
8979 .set_send_buffer_size(4 << 20)
8980 .expect("the frontend send buffer must be settable");
8981
8982 let backend_token = Token(1);
8983 let (mut connection, mut backend_peer) =
8984 test_backend_connection(&mut mux, Duration::from_secs(30));
8985 socket2::SockRef::from(connection.socket())
8986 .set_recv_buffer_size(4 << 20)
8987 .expect("the backend receive buffer must be settable");
8988 socket2::SockRef::from(&backend_peer)
8989 .set_send_buffer_size(4 << 20)
8990 .expect("the backend peer send buffer must be settable");
8991 let Connection::H1(h1) = &mut connection else {
8992 unreachable!("new_h1_client builds an H1 connection")
8993 };
8994 h1.stream = Some(0);
8995 if let Position::Client(_, _, status) = &mut h1.position {
8996 *status = BackendStatus::Connected;
8997 }
8998 h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8999 h1.readiness.event = Ready::READABLE;
9000 mux.router.backends.insert(backend_token, connection);
9001 mux.context.link_stream(0, backend_token);
9002 mux.context.streams[0].front.parsing_phase = kawa::ParsingPhase::Terminated;
9003
9004 // The backend writes its whole response and keeps its connection
9005 // open, as a keep-alive backend does.
9006 let mut response = head.into_bytes();
9007 response.resize(length, b'b');
9008 let mut written = 0;
9009 let deadline = Instant::now() + Duration::from_secs(5);
9010 while written < response.len() {
9011 match backend_peer.write(&response[written..]) {
9012 Ok(size) => written += size,
9013 Err(e) if e.kind() == ErrorKind::WouldBlock => {
9014 assert!(
9015 Instant::now() < deadline,
9016 "premise: the kernel must hold the whole response, wrote {written}"
9017 );
9018 std::thread::sleep(Duration::from_millis(1));
9019 }
9020 Err(e) => panic!("the backend peer failed to write: {e}"),
9021 }
9022 }
9023 std::thread::sleep(Duration::from_millis(20));
9024
9025 let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
9026 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
9027 let mut metrics = SessionMetrics::new(None);
9028
9029 LOOP_LIMIT_HITS.with(|hits| hits.set(0));
9030 let first = mux.ready(session.clone(), proxy.clone(), &mut metrics);
9031 assert_ne!(
9032 first,
9033 SessionResult::Close,
9034 "the transfer must not be closed while it progresses"
9035 );
9036 assert!(
9037 mux.take_loop_budget_yield(),
9038 "the first pass must yield at the budget: it forwarded {} of {length} bytes",
9039 mux.context.stream_bytes_out
9040 );
9041 assert!(
9042 mux.context.stream_bytes_out < length,
9043 "premise: the budget must stop the first pass before the end of the response"
9044 );
9045
9046 let mut result = first;
9047 let mut passes = 1;
9048 let mut received = Vec::new();
9049 let mut chunk = vec![0u8; 1 << 16];
9050 let body_received = |received: &[u8]| {
9051 received
9052 .windows(4)
9053 .position(|window| window == b"\r\n\r\n")
9054 .map_or(0, |end| received.len() - end - 4)
9055 };
9056 let mut yielded = true;
9057 loop {
9058 loop {
9059 match frontend_peer.read(&mut chunk) {
9060 Ok(0) => break,
9061 Ok(size) => received.extend_from_slice(&chunk[..size]),
9062 Err(e) if e.kind() == ErrorKind::WouldBlock => break,
9063 Err(e) => panic!("the client failed to read: {e}"),
9064 }
9065 }
9066 if body_received(&received) >= BODY || result == SessionResult::Close || !yielded {
9067 break;
9068 }
9069 assert!(passes < 10, "the transfer must end in a few passes");
9070 // What `Server::resume_yielded_sessions` does: no new event.
9071 result = mux.ready(session.clone(), proxy.clone(), &mut metrics);
9072 yielded = mux.take_loop_budget_yield();
9073 passes += 1;
9074 }
9075 assert!(passes >= 2, "the body must have needed a resumed pass");
9076 assert_eq!(
9077 LOOP_LIMIT_HITS.with(|hits| hits.get()),
9078 0,
9079 "a progressing pass must yield, never take the infinite-loop path"
9080 );
9081 assert_ne!(
9082 result,
9083 SessionResult::Close,
9084 "no pass may close the transfer"
9085 );
9086 assert_eq!(
9087 body_received(&received),
9088 BODY,
9089 "the resumed passes must deliver the whole body with no new event"
9090 );
9091 assert!(
9092 mux.context.stream_bytes_out >= received.len(),
9093 "every byte the client received was counted as forwarded: {} < {}",
9094 mux.context.stream_bytes_out,
9095 received.len()
9096 );
9097 }
9098
9099 /// The same exchange from a client that shut its write side after its
9100 /// request.
9101 #[test]
9102 fn a_half_closed_slow_client_receives_the_whole_response_of_a_backend_that_closed() {
9103 let (received, expected, first, hits) = slow_client_after_backend_close(true);
9104 assert_eq!(
9105 hits, 0,
9106 "every pass must stop once nothing can progress, not at the loop limit"
9107 );
9108 assert_ne!(
9109 first,
9110 SessionResult::Close,
9111 "the first pass must wait for the client, not close"
9112 );
9113 assert_eq!(received, expected, "the client must receive the whole body");
9114 }
9115
9116 /// At the loop limit, a response still queued in an H1 frontend's stream
9117 /// buffer is not dropped: the session waits for the client's next
9118 /// writable event instead of closing. A plain-TCP H1 frontend's
9119 /// connection-level `has_pending_write()` is false whatever its stream
9120 /// holds, which is what the limit used to check (sozu-proxy/sozu#1819).
9121 ///
9122 /// TO SEE THIS RED: make `Mux::wait_for_client_at_loop_limit` compute
9123 /// `output_queued` from `self.frontend.has_pending_write()` alone.
9124 #[test]
9125 fn the_loop_limit_waits_for_the_client_while_a_stream_holds_its_response() {
9126 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9127 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9128 mux.context.streams[0].state = StreamState::Linked(Token(1));
9129 mux.context.streams[0]
9130 .back
9131 .push_out(kawa::Store::Static(b"the end of the body"));
9132 mux.frontend.readiness_mut().event = Ready::WRITABLE;
9133 assert!(
9134 !mux.frontend.has_pending_write(),
9135 "premise: the connection itself has nothing pending"
9136 );
9137
9138 assert!(
9139 mux.wait_for_client_at_loop_limit(),
9140 "a queued response must make the session wait, not close"
9141 );
9142 let readiness = mux.frontend.readiness();
9143 assert!(
9144 readiness.interest.is_writable() && !readiness.event.is_writable(),
9145 "the session waits for the next writable event, got {readiness:?}"
9146 );
9147 }
9148
9149 /// Waiting at the loop limit is bounded: reaching it again with no byte
9150 /// written to the client since closes the session, while a client that
9151 /// drained some of the response in between is waited for again.
9152 ///
9153 /// TO SEE THIS RED: drop `progressed &&` from `output_queued` in
9154 /// `Mux::wait_for_client_at_loop_limit`. The second hit then waits again.
9155 #[test]
9156 fn a_second_loop_limit_without_progress_closes_the_session() {
9157 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9158 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9159 mux.context.streams[0].state = StreamState::Linked(Token(1));
9160 mux.context.streams[0]
9161 .back
9162 .push_out(kawa::Store::Static(b"the end of the body"));
9163
9164 assert!(
9165 mux.wait_for_client_at_loop_limit(),
9166 "the first hit with output queued waits for the client"
9167 );
9168 mux.context.client_bytes_out += 100;
9169 assert!(
9170 mux.wait_for_client_at_loop_limit(),
9171 "the client drained bytes since: the session waits again"
9172 );
9173 assert!(
9174 !mux.wait_for_client_at_loop_limit(),
9175 "no byte reached the client since the last hit: the session must close"
9176 );
9177 }
9178
9179 /// Progress at the loop limit is read from the session's own count of
9180 /// bytes written to the client, not from the per-stream metrics, which
9181 /// `SessionMetrics::reset` zeroes when a stream ends and its slot is
9182 /// reused.
9183 ///
9184 /// TO SEE THIS RED: compute `bytes_out` in
9185 /// `Mux::wait_for_client_at_loop_limit` as the sum of every stream's
9186 /// `metrics.bout` again. A slot reused between two hits then reads as
9187 /// progress when nothing was written, and as no progress when the new
9188 /// stream wrote exactly what the old one had.
9189 #[test]
9190 fn a_stream_slot_reused_between_two_loop_limits_does_not_fake_progress() {
9191 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9192 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9193 mux.context.streams[0].state = StreamState::Linked(Token(1));
9194 mux.context.streams[0]
9195 .back
9196 .push_out(kawa::Store::Static(b"the end of the body"));
9197 mux.context.streams[0].metrics.bout = 100;
9198 mux.context.client_bytes_out = 100;
9199 assert!(
9200 mux.wait_for_client_at_loop_limit(),
9201 "the first hit with output queued waits for the client"
9202 );
9203
9204 // The stream ends and its slot is reused before anything is written.
9205 mux.context.streams[0].metrics.reset();
9206 assert!(
9207 !mux.wait_for_client_at_loop_limit(),
9208 "a reset metric is not progress: the session must close"
9209 );
9210
9211 // A fresh session state: the new stream writes as much as the old one
9212 // had, which is progress.
9213 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9214 mux.context.streams[0].state = StreamState::Linked(Token(1));
9215 mux.context.streams[0]
9216 .back
9217 .push_out(kawa::Store::Static(b"the end of the body"));
9218 mux.context.streams[0].metrics.bout = 100;
9219 mux.context.client_bytes_out = 100;
9220 assert!(mux.wait_for_client_at_loop_limit(), "the first hit waits");
9221 mux.context.streams[0].metrics.reset();
9222 mux.context.streams[0].metrics.bout = 100;
9223 mux.context.client_bytes_out += 100;
9224 assert!(
9225 mux.wait_for_client_at_loop_limit(),
9226 "the reused slot wrote 100 bytes since: the session waits again"
9227 );
9228 }
9229
9230 /// At the loop limit, a session with nothing queued for the client is
9231 /// closed: the limit stays a safety bound. Bytes left in a stream slot
9232 /// that is no longer open are not owed to the client.
9233 #[test]
9234 fn the_loop_limit_closes_a_session_with_nothing_queued() {
9235 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9236 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9237 mux.context.streams[0].state = StreamState::Recycle;
9238 mux.context.streams[0]
9239 .back
9240 .push_out(kawa::Store::Static(b"a recycled slot's leftover"));
9241 assert!(
9242 !mux.wait_for_client_at_loop_limit(),
9243 "nothing queued: the session must close"
9244 );
9245 }
9246
9247 // ── no write-side shutdown once the peer has closed ────────────────
9248
9249 /// Close a mux holding one backend whose readiness is `event` and whose
9250 /// status is `status`, and return the `shutdown(SHUT_WR)` calls the close
9251 /// issued together with what the backend peer then observed.
9252 fn backend_shutdowns_of_a_close(
9253 event: Ready,
9254 status: BackendStatus,
9255 peer_sends_fin: bool,
9256 ) -> (usize, bool) {
9257 let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9258 let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9259 mux.context.streams[0].metrics.reset();
9260 let (mut connection, mut backend_peer) =
9261 test_backend_connection(&mut mux, Duration::from_secs(30));
9262 if peer_sends_fin {
9263 backend_peer
9264 .shutdown(std::net::Shutdown::Write)
9265 .expect("the backend peer must send its FIN");
9266 }
9267 *connection.position_mut() = match connection.position() {
9268 Position::Client(cluster_id, backend, _) => {
9269 Position::Client(cluster_id.clone(), backend.clone(), status)
9270 }
9271 Position::Server => unreachable!("new_h1_client builds a client"),
9272 };
9273 connection.readiness_mut().event = event;
9274 mux.router.backends.insert(Token(1), connection);
9275
9276 let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
9277 let mut metrics = SessionMetrics::new(None);
9278 let before = shutdown_writes();
9279 mux.close(proxy, &mut metrics);
9280 let shutdowns = shutdown_writes() - before;
9281 // The descriptor closes when the mux drops, as it does with its
9282 // session in production.
9283 drop(mux);
9284 let (_, saw_fin) = drain_peer(&mut backend_peer, &[]);
9285 (shutdowns, saw_fin)
9286 }
9287
9288 /// Issue #1603: `Mux::close` skips `shutdown(SHUT_WR)` on a backend that
9289 /// has already closed, and the backend still gets its FIN from the close.
9290 ///
9291 /// The peer in the first case half-closed and is still reading: that is
9292 /// the peer the skipped shutdown could have mattered to, and the FIN it
9293 /// observes comes from `close(2)` alone.
9294 ///
9295 /// TO SEE THIS RED: make `shutdown_write` ignore `peer_closed`. The first
9296 /// assertion then fails with `left: 1`, `right: 0`.
9297 #[test]
9298 fn mux_close_skips_the_backend_shutdown_once_the_backend_closed() {
9299 let (shutdowns, saw_fin) =
9300 backend_shutdowns_of_a_close(Ready::HUP, BackendStatus::Connected, true);
9301 assert_eq!(
9302 shutdowns, 0,
9303 "a backend whose FIN was seen must not be shut down before its close"
9304 );
9305 assert!(
9306 saw_fin,
9307 "the close alone must still deliver the FIN to a half-closed peer"
9308 );
9309 }
9310
9311 /// The other half of #1603: a backend that has NOT closed keeps its
9312 /// `shutdown(SHUT_WR)` and observes the FIN — including one sozu itself
9313 /// is dropping, whose readiness `force_disconnect` overwrote with HUP.
9314 #[test]
9315 fn mux_close_keeps_the_backend_shutdown_while_the_backend_is_there() {
9316 for (event, status, why) in [
9317 (
9318 Ready::READABLE | Ready::WRITABLE,
9319 BackendStatus::Connected,
9320 "a live backend",
9321 ),
9322 (
9323 Ready::HUP,
9324 BackendStatus::Disconnecting,
9325 "a backend force_disconnect marked with HUP",
9326 ),
9327 ] {
9328 let (shutdowns, saw_fin) = backend_shutdowns_of_a_close(event, status, false);
9329 assert_eq!(shutdowns, 1, "{why} must keep its shutdown(SHUT_WR)");
9330 assert!(saw_fin, "{why} must observe the FIN");
9331 }
9332 }
9333}