Skip to main content

sozu_lib/protocol/mux/
mod.rs

1//! HTTP/1.1 and HTTP/2 multiplexing layer.
2//!
3//! This module unifies HTTP/1.1 and HTTP/2 behind a single [`Mux`] session
4//! state machine that integrates with sozu's mio event loop. The key types:
5//!
6//! - [`Mux`]: The top-level session state, generic over socket (`TcpStream` or
7//!   `FrontRustls`) and listener. Implements `SessionState`.
8//! - [`Connection`]: Enum dispatching to [`ConnectionH1`] or [`ConnectionH2`]
9//!   for protocol-specific readable/writable logic.
10//! - [`Stream`]: Per-request state with front/back kawa buffers, metrics, and
11//!   lifecycle tracking. Shared between H1 and H2 paths.
12//! - [`Context`]: Per-session context (cluster, backends, routing, timeouts).
13//!
14//! The H2 implementation handles RFC 9113 framing, HPACK (RFC 7541), flow
15//! control, flood detection (CVE-2023-44487, CVE-2019-9512/9514/9515/9518,
16//! CVE-2024-27316), and graceful shutdown (double-GOAWAY per RFC 9113 §6.8).
17
18use std::{
19    cell::RefCell,
20    collections::{BTreeMap, VecDeque},
21    fmt::Debug,
22    io::ErrorKind,
23    net::{Shutdown, SocketAddr},
24    rc::{Rc, Weak},
25    sync::Arc,
26    time::{Duration, Instant, SystemTime, UNIX_EPOCH},
27};
28
29use mio::{Interest, Token, net::TcpStream};
30use rand::{SeedableRng, rngs::StdRng};
31use rusty_ulid::Ulid;
32use sozu_command::{
33    logging::ansi_palette,
34    proto::command::{Event, EventKind},
35    ready::Ready,
36};
37
38/// Protocol label + session descriptor used as a prefix on every [`Mux`] log
39/// line. Matches the RUSTLS log-context convention:
40/// `[<ulid> - - -]\tMUX\tSession(...)\t >>>`. When colored output is enabled
41/// (via [`ansi_palette`]) the label is wrapped in bold bright-white ANSI
42/// (uniform across every protocol) and the session detail block is rendered
43/// in light grey.
44///
45/// Fields included in the session block:
46/// - `frontend` — mio token of the frontend socket
47/// - `peer` — [`Connection::peer_address`]: a snapshot, not a live lookup
48/// - `streams` — number of streams currently held by the [`Context`]
49/// - `backends` — number of backend connections in the [`Router`]
50/// - `pending_links` — streams waiting to be linked to a backend
51/// - `readiness` — frontend mio readiness snapshot
52macro_rules! log_context {
53    ($self:expr) => {{
54        let (open, reset, grey, gray, white) = ansi_palette();
55        format!(
56            "[{ulid} - - -]\t{open}MUX{reset}\t{grey}Session{reset}({gray}frontend{reset}={white}{frontend}{reset}, {gray}peer{reset}={white}{peer:?}{reset}, {gray}streams{reset}={white}{streams}{reset}, {gray}backends{reset}={white}{backends}{reset}, {gray}pending_links{reset}={white}{pending_links}{reset}, {gray}readiness{reset}={white}{readiness}{reset})\t >>>",
57            open = open,
58            reset = reset,
59            grey = grey,
60            gray = gray,
61            white = white,
62            ulid = $self.session_ulid,
63            frontend = $self.frontend_token.0,
64            peer = $self.frontend.peer_address(),
65            streams = $self.context.streams.len(),
66            backends = $self.router.backends.len(),
67            pending_links = $self.context.pending_links.len(),
68            readiness = $self.frontend.readiness(),
69        )
70    }};
71}
72
73/// Lighter variant of [`log_context!`] that omits the
74/// `streams`/`backends`/`pending_links` counts. Used at sites where the
75/// borrow checker forbids reading `self.router.backends` or
76/// `self.context.streams` (e.g. inside a method that already holds a mutable
77/// borrow on one of them). The ULID and frontend snapshot still carry enough
78/// context to correlate the line back to the rest of the session.
79macro_rules! log_context_lite {
80    ($self:expr) => {{
81        let (open, reset, grey, gray, white) = ansi_palette();
82        format!(
83            "[{ulid} - - -]\t{open}MUX{reset}\t{grey}Session{reset}({gray}frontend{reset}={white}{frontend}{reset}, {gray}peer{reset}={white}{peer:?}{reset}, {gray}readiness{reset}={white}{readiness}{reset})\t >>>",
84            open = open,
85            reset = reset,
86            grey = grey,
87            gray = gray,
88            white = white,
89            ulid = $self.session_ulid,
90            frontend = $self.frontend_token.0,
91            peer = $self.frontend.peer_address(),
92            readiness = $self.frontend.readiness(),
93        )
94    }};
95}
96
97/// Module-level prefix for logs emitted from free functions or routing
98/// blocks where no [`Mux`] is in scope. Honours the colored flag.
99///
100/// Two arms:
101/// * `log_module_context!()` — zero-arg, legacy `MUX\t >>>` output. Kept
102///   for sites without an `HttpContext` in scope (e.g. the generic
103///   `trace!` that fires before the variant-specific match).
104/// * `log_module_context!($http_context)` — rich form. `$http_context`
105///   must be `&HttpContext`. Produces the same
106///   `[session req cluster backend]` bracket as RUSTLS/PIPE/TCP followed
107///   by a `Session(...)` block, so MUX lines emitted from variant match
108///   arms stay filterable by session ULID or request ULID. Mirrors
109///   `router.rs:log_module_context!($http_context)` (see there). Custom
110///   methods render only their byte length, and authority renders only its
111///   byte length.
112macro_rules! log_module_context {
113    () => {{
114        let (open, reset, _, _, _) = ansi_palette();
115        format!("{open}MUX{reset}\t >>>", open = open, reset = reset)
116    }};
117    ($http_context:expr) => {{
118        let (open, reset, grey, gray, white) = ansi_palette();
119        let http_ctx: &HttpContext = &$http_context;
120        let ctx = http_ctx.log_context();
121        format!(
122            "{gray}{ctx}{reset}\t{open}MUX{reset}\t{grey}Session{reset}({gray}frontend{reset}={white}{frontend:?}{reset}, {gray}method{reset}={white}{method:?}{reset}, {gray}authority_bytes{reset}={white}{authority_bytes:?}{reset})\t >>>",
123            open = open,
124            reset = reset,
125            grey = grey,
126            gray = gray,
127            white = white,
128            ctx = ctx,
129            frontend = http_ctx.session_address,
130            method = http_ctx.method,
131            authority_bytes = http_ctx.authority.as_ref().map(String::len),
132        )
133    }};
134}
135
136use self::h2::MetricEvent;
137
138pub mod answers;
139pub mod auth;
140pub mod buffer_source;
141pub mod connection;
142mod converter;
143pub mod debug;
144mod h1;
145mod h2;
146mod h2_close;
147mod h2_control_tx;
148mod h2_drain;
149mod h2_flood_detector;
150mod h2_flow_control;
151mod h2_header_reassembly;
152mod h2_output;
153mod h2_scheduler;
154mod h2_stream_table;
155pub mod h2_transmit;
156mod h2_write_pass;
157pub mod hpack;
158mod hpack_state;
159pub mod parser;
160mod pkawa;
161pub mod router;
162pub(crate) mod serializer;
163mod shared;
164pub mod stream;
165
166use crate::metrics::names;
167use crate::{
168    BackendConnectionError, FrontendFromRequestError, L7ListenerHandler, L7Proxy, ListenerHandler,
169    Protocol, ProxySession, Readiness, RetrieveClusterError, SessionIsToBeClosed, SessionMetrics,
170    SessionResult, StateResult,
171    backends::{Backend, BackendError, BackendMap},
172    http::HttpListener,
173    https::HttpsListener,
174    pool::{Checkout, Pool},
175    protocol::{SessionState, http::editor::HttpContext},
176    retry::RetryPolicy,
177    server::push_event,
178    socket::{FrontRustls, SessionTcpStream, SocketHandler, SocketResult, stats::socket_rtt},
179    timer::TimeoutContainer,
180};
181
182use crate::protocol::mux::answers::silent_close_answer;
183pub(crate) use crate::protocol::mux::answers::{
184    forcefully_terminate_answer, set_default_answer, set_default_answer_with_retry_after,
185};
186use crate::protocol::mux::connection::{EndpointClient, EndpointServer};
187pub use crate::protocol::mux::{
188    answers::terminate_default_answer,
189    buffer_source::{BufferSource, PoolBufferSource},
190    connection::Connection,
191    debug::{DebugEvent, DebugHistory},
192    h1::ConnectionH1,
193    h2::CLIENT_PREFACE_SIZE,
194    h2::ConnectionH2,
195    h2::H2ByteAccounting,
196    h2::H2ConnectionConfig,
197    h2::{
198        H2ControlFlushTarget, H2FinalizeTarget, H2ForceDisconnectTarget, H2ReadOutcome,
199        H2ReadTarget, H2Shell, H2StreamId, H2WritableStateTarget, H2WriteTarget, H2Written,
200    },
201    h2_flood_detector::H2FloodConfig,
202    h2_write_pass::H2WritePass,
203    parser::H2Error,
204    router::Router,
205    stream::{Stream, StreamParts, StreamState},
206};
207
208// ── Tuning Constants ─────────────────────────────────────────────────────────
209
210/// Maximum inner-loop iterations of one `Mux::ready_inner` call, a bound
211/// that keeps one session from holding the single-threaded worker. A call
212/// that forwarded stream bytes yields there, counted in
213/// `http.mux.loop_budget_yield`, and the event loop resumes it on its next
214/// turn (sozu-proxy/sozu#1906). Reaching it without forwarding a byte is a
215/// bug: it increments `http.infinite_loop.error` and logs a warning. A
216/// session with output still queued for the client then waits for its next
217/// writable event; any other session is closed.
218const MAX_LOOP_ITERATIONS: i32 = 10_000;
219
220/// Whether a response in progress waits on the CLIENT rather than on its
221/// backend: Sōzu still holds response bytes for this stream that the client
222/// has not taken. `front_pending_write` is the frontend connection's own
223/// pending output (`Connection::has_pending_write`, a TLS record for
224/// instance); it is stream-specific only on an H1 frontend, so an H2
225/// frontend passes `false` and only the stream's own queued bytes count.
226///
227/// `back_timeout` and `front_timeout` bound the inactivity of one socket
228/// each (`doc/configure.md`). The two timeout arms of `Mux::timeout_inner`
229/// read this predicate with opposite polarity, so exactly one of the two
230/// timers can cut a response at any moment: a backend whose bytes Sōzu has
231/// stopped reading because a slow client left them queued is not inactive,
232/// and neither is a client waiting on a slow backend (sozu-proxy/sozu#1906).
233fn response_waits_on_client(stream: &Stream, front_pending_write: bool) -> bool {
234    let queued = !stream.back.out.is_empty() || !stream.back.blocks.is_empty();
235    debug_assert!(
236        stream.back.consumed,
237        "only a response in progress can wait on either peer"
238    );
239    debug_assert_eq!(
240        queued,
241        !stream.back.is_completed(),
242        "kawa's `is_completed` is exactly \"nothing queued\""
243    );
244    queued || front_pending_write
245}
246
247/// Whether a `Mux::ready_inner` call that reached `MAX_LOOP_ITERATIONS`
248/// moved enough stream bytes to be a transfer whose peers keep up, which
249/// yields to the event loop, rather than a spin, which the loop budget
250/// exists to catch (sozu-proxy/sozu#1906). A real transfer forwards up to a
251/// buffer per iteration; at least one byte per budgeted iteration on average
252/// is required, so a spin that trickles a few bytes still reaches
253/// `Mux::wait_for_client_at_loop_limit` instead of yielding forever.
254/// `Context::stream_bytes_out` wraps, hence the wrapping difference.
255fn progressed_through_loop_budget(bytes_out_at_entry: usize, bytes_out_now: usize) -> bool {
256    let moved = bytes_out_now.wrapping_sub(bytes_out_at_entry);
257    let progressed = moved >= MAX_LOOP_ITERATIONS as usize;
258    debug_assert!(
259        !progressed || moved > 0,
260        "a call judged as progressing must have moved bytes"
261    );
262    debug_assert!(
263        progressed || moved < MAX_LOOP_ITERATIONS as usize,
264        "a call that moved a byte per iteration must be judged as progressing"
265    );
266    progressed
267}
268// ─────────────────────────────────────────────────────────────────────────────
269
270/// Debug tripwire for the one-active-stream invariant that
271/// `ConnectionH1::end_stream`'s early-return guard rests on.
272///
273/// `Mux::timeout` retires streams through `Connection::end_stream`, and its
274/// proof that no linked stream survives (LIFECYCLE invariant 22) depends on
275/// that call actually unlinking. `ConnectionH2::end_stream` always does.
276/// `ConnectionH1::end_stream` does NOT: it guards `self.stream != Some(stream)`
277/// and returns early, skipping the `Context::unlink_stream`. The guard is safe
278/// only because an H1 connection carries at most one linked stream and it
279/// equals `self.stream` — `self.stream = Some(..)` occurs only in
280/// `ConnectionH1::start_stream`, paired with `Context::link_stream`, and
281/// `self.stream = None` only inside `end_stream` itself, after the unlink.
282///
283/// This is the tripwire for that invariant: a drift between
284/// `context.backend_streams` (or a stream's `StreamState::Linked`) and
285/// `ConnectionH1::stream` fires here, in every debug / test / e2e build,
286/// instead of silently leaving a stream in the reverse index with no backend
287/// timer.
288///
289/// To SEE THIS FIRE: in
290/// `a_backend_timeout_leaves_no_linked_stream_behind_on_the_close_path`, set
291/// `h1.stream = Some(1)` instead of `Some(0)` while still linking stream 0 to
292/// the backend.
293///
294/// Deliberately at the CALL SITE rather than inside `ConnectionH1::end_stream`,
295/// and the reason is scope, not risk. That function has around a dozen callers
296/// reaching it through `Connection::end_stream` and the two `Endpoint`
297/// adaptors, and the H2-driven ones are precisely the ones this proof never
298/// traced: `H2Shell::close` and `ConnectionH2::end_stream` iterate their
299/// OWN wire map (`for global_stream_id in self.streams.values()`, `h2.rs`) and
300/// hand each `StreamState::Linked` gid to the peer, which on an H2-frontend /
301/// H1-backend session is a `ConnectionH1`. Add the reset and HUP paths and the
302/// set is wider than the claim.
303///
304/// Asserting inside the function would therefore assert something broader than
305/// what was traced. Note what does NOT justify the placement: running the full
306/// e2e suite with a `debug_assert_eq!` in that mismatch arm passes with zero
307/// hits, and green e2e shows absence of COVERAGE, not absence of the path. The
308/// claim is about the two `Mux::timeout` loops, so the assertion is too.
309#[cfg(debug_assertions)]
310fn debug_assert_h1_owns_stream<Front: SocketHandler>(
311    connection: &Connection<Front>,
312    stream_id: GlobalStreamId,
313) {
314    if let Connection::H1(h1) = connection {
315        debug_assert_eq!(
316            h1.stream,
317            Some(stream_id),
318            "ConnectionH1::end_stream would skip its unlink: the connection's \
319             active stream is {:?}, not the {stream_id} Mux::timeout is ending \
320             — the backend index and ConnectionH1::stream have drifted",
321            h1.stream,
322        );
323    }
324}
325
326/// Generic Http representation using the Kawa crate using the Checkout of Sozu as buffer
327type GenericHttpStream = kawa::Kawa<Checkout>;
328type StreamId = u32;
329type GlobalStreamId = usize;
330pub type MuxClear = Mux<SessionTcpStream, HttpListener>;
331pub type MuxTls = Mux<FrontRustls, HttpsListener>;
332
333/// An opaque, session-scoped name for one entry of the worker's backend
334/// registry.
335///
336/// It is an index into `Mux::backend_registry` and nothing else: the core
337/// can carry it, compare it and hand it back, and cannot turn it into a
338/// `Backend`. Only the embedder resolves it, which is what keeps
339/// `Rc<RefCell<Backend>>` out of the core (#1340, Question 12).
340///
341/// Session-scoped rather than derived from `(cluster_id, backend_id)` on
342/// purpose. A reload that replaces a registry entry mid-session builds a NEW
343/// `Rc`, which takes a NEW slot, so connections opened before it keep charging
344/// the handle they were dialled against — exactly what holding the `Rc`
345/// inside `Position::Client` used to do. Keying on the backend's name instead
346/// would split a balanced start/end pair across two `Backend` values: the
347/// superseded one would keep a charge forever and the fresh one would
348/// saturate at zero.
349#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
350pub struct BackendSlot(usize);
351
352/// A backend as the core sees it: the opaque slot, plus the two identity
353/// fields the datapath renders.
354///
355/// `backend_id` and `address` are copied once, when the session first interns
356/// the backend in its `BackendRegistry`, and every later dial of it clones
357/// them from there. Both are immutable for the life of a registry entry, so
358/// carrying them is not a cached view of mutable state — no load state
359/// crosses this boundary.
360/// The mutable half (`active_connections`, `active_requests`, `failures`,
361/// `connection_time`, `health`, `status`) stays behind the slot, and the core
362/// reaches it only by emitting a [`BackendDelta`].
363///
364/// `backend_id` is an `Rc<str>` so that no step past interning copies it:
365/// rebuilding `Position::Client` on `Connecting` -> `Connected` once per dial,
366/// minting the id for a redial (the registry keeps the `Rc<str>` it built at
367/// interning, #1564), and stamping a request with the backend it runs on,
368/// dialled or reused, into `HttpContext::backend_id` and
369/// `SessionMetrics::backend_id` (#1579) are each a reference-count increment.
370/// A `String` would make every one of them a heap copy.
371#[derive(Clone, Debug, PartialEq, Eq)]
372pub struct BackendId {
373    /// Private to this module: [`Mux::backend`] is the only way out.
374    slot: BackendSlot,
375    /// `Backend::backend_id`, for log lines and per-backend metric labels.
376    pub backend_id: Rc<str>,
377    /// `Backend::address`, for log lines and `push_event` payloads.
378    pub address: SocketAddr,
379    /// Metrics lifetime captured by the request that opened this connection.
380    /// It stays with the connection through draining so its close cannot
381    /// mutate a replacement cluster with the same labels.
382    metric_incarnation: crate::metrics::ClusterMetricsCapture,
383}
384
385impl BackendId {
386    /// An id for the backend at `address`, named `backend_id`, in `slot` of
387    /// whichever registry mints it (#1684).
388    ///
389    /// The worker's mux never calls this: its ids come from the session's
390    /// `BackendRegistry`, which interns the backend it names and is the only
391    /// thing that can resolve a slot back to it. This is for an embedder that
392    /// owns its own selector — a simulator implementing
393    /// [`router::BackendSelector`] — and numbers its backends itself. A slot
394    /// means something only to the registry that issued it.
395    pub fn new(slot: usize, backend_id: Rc<str>, address: SocketAddr) -> Self {
396        BackendId {
397            slot: BackendSlot(slot),
398            backend_id,
399            address,
400            metric_incarnation: Default::default(),
401        }
402    }
403
404    pub(crate) fn metric_incarnation(&self) -> crate::metrics::ClusterMetricsCapture {
405        self.metric_incarnation
406    }
407
408    fn set_metric_incarnation(&mut self, incarnation: crate::metrics::ClusterMetricsCapture) {
409        self.metric_incarnation = incarnation;
410    }
411
412    /// The slot this id names. Module-private on purpose — see [`Mux::backend`]
413    /// for the crate-visible resolution path.
414    fn slot(&self) -> BackendSlot {
415        self.slot
416    }
417}
418
419/// One accounting change the core decided and the embedder performs.
420///
421/// LIFECYCLE §9 invariant 14 is a balance, and this is the ledger it balances:
422/// every [`BackendChange::StreamsStarted`] unit the mux emits is covered by
423/// exactly one [`BackendChange::StreamsEnded`] unit against the same slot.
424/// Because every write to `Backend::active_requests` and
425/// `Backend::active_connections` on the mux path now goes through
426/// `Mux::apply_backend_deltas`, reconciling the emitted stream to zero is a
427/// direct test of the invariant rather than a proxy for it.
428#[derive(Clone, Copy, Debug, PartialEq, Eq)]
429pub struct BackendDelta {
430    /// The registry entry to charge.
431    pub slot: BackendSlot,
432    /// What to charge it.
433    pub change: BackendChange,
434}
435
436/// The accounting operations [`BackendDelta`] can carry.
437///
438/// The two counters invariant 14 is about, plus the release of a
439/// reservation whose dial never started. The other health and latency writes
440/// the mux makes (`failures` and `retry_policy` on an asynchronous connect
441/// failure, `connection_time`) are NOT here: they stay immediate at their
442/// existing embedder-side call sites rather than being deferred to a drain.
443#[derive(Clone, Copy, Debug, PartialEq, Eq)]
444pub enum BackendChange {
445    /// `Backend::active_requests += n` — `n` streams started on this backend.
446    StreamsStarted(usize),
447    /// `Backend::active_requests -= n`, saturating — `n` streams left it.
448    StreamsEnded(usize),
449    /// `Backend::dec_connections()` — the connection to this backend closed,
450    /// or a dial that connected was abandoned before its connection was
451    /// registered (`Mux::dial_backend`, #1713): either way the reservation
452    /// selection took is released, and no failure is recorded.
453    ConnectionClosed,
454    /// The `connect(2)` of a dial failed at this instant (#1684): release the
455    /// reservation selection took (`Backend::dec_connections()`), and record
456    /// the failure (`failures += 1`, `retry_policy.fail`), as
457    /// `Backend::release_failed_dial` does. The drain before the next
458    /// selection applies it, so that selection sees the backend exactly as a
459    /// failed `Backend::try_connect` left it.
460    DialFailed(Instant),
461}
462
463/// The embedder's table of registry handles, indexed by [`BackendSlot`].
464///
465/// A newtype rather than a bare `Vec` so the two operations that may cross
466/// Question 12's perimeter — minting a slot for a freshly dialled backend,
467/// and performing a [`BackendDelta`] against one — are the only ones there
468/// are. Nothing hands a `&Rc<RefCell<Backend>>` out of here except
469/// [`Mux::backend`], whose one caller is the WebSocket upgrade.
470#[derive(Debug, Default)]
471pub(crate) struct BackendRegistry(Vec<RegistryEntry>);
472
473/// One slot of the [`BackendRegistry`]: the handle, plus the [`BackendId`]
474/// identity fields read from it once, when it was interned.
475///
476/// The cached `backend_id` is what keeps `BackendRegistry::id_for`
477/// allocation-free on every dial after a backend's first (#1564). It cannot go
478/// stale: `Backend::backend_id` and `Backend::address` are never written after
479/// construction, and a reload that replaces the backend builds a new `Rc`,
480/// which takes a new slot with its own entry.
481#[derive(Debug)]
482struct RegistryEntry {
483    handle: Rc<RefCell<Backend>>,
484    backend_id: Rc<str>,
485    address: SocketAddr,
486}
487
488impl BackendRegistry {
489    /// Name `backend` with a slot, reusing the one it already has.
490    ///
491    /// Identity is `Rc::ptr_eq`, not the backend's id or address: two dials
492    /// separated by a reload that replaced the registry entry are two
493    /// different `Backend` values, and each must keep its own accounting.
494    /// The scan is linear over the backends ONE session has dialled, which
495    /// is the cluster's backend count at worst — the same order as the
496    /// `Router::backends` walk the caller has just done.
497    ///
498    /// A new slot copies the backend's id into an `Rc<str>` here, once per
499    /// session and backend; that copy and the table's own growth are the only
500    /// allocations this path makes.
501    fn intern(&mut self, backend: &Rc<RefCell<Backend>>) -> BackendSlot {
502        match self
503            .0
504            .iter()
505            .position(|known| Rc::ptr_eq(&known.handle, backend))
506        {
507            Some(slot) => BackendSlot(slot),
508            None => {
509                let (backend_id, address) = {
510                    let borrow = backend.borrow();
511                    (Rc::from(borrow.backend_id.as_str()), borrow.address)
512                };
513                // A session almost always dials one backend: its first entry
514                // sizes the table for exactly one instead of four (#1610),
515                // and a second distinct backend grows it as usual.
516                if self.0.capacity() == 0 {
517                    self.0.reserve_exact(1);
518                }
519                self.0.push(RegistryEntry {
520                    handle: backend.clone(),
521                    backend_id,
522                    address,
523                });
524                BackendSlot(self.0.len() - 1)
525            }
526        }
527    }
528
529    /// Build the core's view of a backend it is about to be handed.
530    ///
531    /// Runs once per backend dial. Past a backend's first dial in this
532    /// session it clones the id cached in its slot and allocates nothing.
533    fn id_for(&mut self, backend: &Rc<RefCell<Backend>>) -> BackendId {
534        let slot = self.intern(backend);
535        let entry = &self.0[slot.0];
536        BackendId {
537            slot,
538            backend_id: entry.backend_id.clone(),
539            address: entry.address,
540            metric_incarnation: Default::default(),
541        }
542    }
543
544    fn get(&self, slot: BackendSlot) -> Option<&Rc<RefCell<Backend>>> {
545        self.0.get(slot.0).map(|entry| &entry.handle)
546    }
547
548    /// Resolve one of the core's opaque [`BackendId`]s to the registry handle
549    /// it names.
550    ///
551    /// The single crate-visible way back from an id to an
552    /// `Rc<RefCell<Backend>>`, so the perimeter Question 12 draws has one
553    /// door rather than a `pub` slot every caller can walk through. Its only
554    /// callers are `upgrade_mux` in `lib/src/http.rs` and `lib/src/https.rs`,
555    /// which hand the live handle to the WebSocket `Pipe` that takes the
556    /// backend over.
557    ///
558    /// It sits here rather than on [`Mux`] because those callers have already
559    /// moved `Mux::frontend` out by the time they need it; a field access
560    /// borrows disjointly where a `&self` method on `Mux` could not.
561    ///
562    /// `None` cannot happen for an id this session minted — the table never
563    /// shrinks — so a caller treats it as the desync it would be.
564    pub(crate) fn handle(&self, backend: &BackendId) -> Option<&Rc<RefCell<Backend>>> {
565        self.get(backend.slot())
566    }
567
568    /// Whether the backend `backend` names has left the configuration.
569    ///
570    /// `BackendList::remove_backend` and `BackendMap::remove_cluster`
571    /// (`lib/src/backends.rs`) mark a backend `Closing` as they drop it, and
572    /// the registry still holds that very `Rc`. A backend added again later,
573    /// even at the same address, is a new `Rc` in a new slot, so a
574    /// connection dialled to the old one stays retired. Reads one `RefCell`
575    /// borrow and allocates nothing.
576    pub(crate) fn is_retired(&self, backend: &BackendId) -> bool {
577        self.get(backend.slot())
578            .is_some_and(|handle| handle.borrow().status != crate::backends::BackendStatus::Normal)
579    }
580
581    /// Perform every accounting change in `deltas`, in the order they were
582    /// pushed, and leave the ledger empty.
583    ///
584    /// The one drain both `Mux::apply_backend_deltas` and the pre-plan settle
585    /// in `Mux::ready_inner` go through.
586    ///
587    /// Drained in place rather than taken, so the ledger keeps its capacity
588    /// for the next change: the charge `Connection::start_stream` records on
589    /// every request would otherwise reallocate it each time (#1583). The
590    /// capacity is bounded by the most deltas a single pass has pushed.
591    ///
592    /// `backends` is the worker's backend map, borrowed only to draw the
593    /// backoff jitter of a [`BackendChange::DialFailed`] from its generator;
594    /// every other change leaves it untouched.
595    pub(crate) fn apply_all(&self, deltas: &mut Vec<BackendDelta>, backends: &RefCell<BackendMap>) {
596        for delta in deltas.drain(..) {
597            self.apply(delta, backends);
598        }
599    }
600
601    /// Perform one accounting change the core decided.
602    ///
603    /// The before/after pair-assertions that used to sit at each emitting
604    /// site live here now: this is where both halves are observable, and one
605    /// copy covers every emitter instead of each carrying its own.
606    fn apply(&self, delta: BackendDelta, backends: &RefCell<BackendMap>) {
607        let Some(backend) = self.get(delta.slot) else {
608            // Unreachable while the vector only grows, which it does. Report
609            // rather than panic: losing one charge is a drifting gauge, and
610            // dropping a session over it would be worse.
611            error!(
612                "{} backend accounting delta {:?} names a slot this session never minted",
613                log_module_context!(),
614                delta
615            );
616            return;
617        };
618        let mut backend = backend.borrow_mut();
619        match delta.change {
620            BackendChange::StreamsStarted(count) => {
621                let before = backend.active_requests;
622                backend.active_requests += count;
623                debug_assert_eq!(
624                    backend.active_requests,
625                    before + count,
626                    "a StreamsStarted delta must raise active_requests by exactly its count"
627                );
628            }
629            BackendChange::StreamsEnded(count) => {
630                // `saturating_sub` is the network-safe floor a desynced peer
631                // needs, so the post-relation is all that can be asserted —
632                // not that `before >= count`.
633                let before = backend.active_requests;
634                backend.active_requests = backend.active_requests.saturating_sub(count);
635                debug_assert_eq!(
636                    backend.active_requests,
637                    before.saturating_sub(count),
638                    "a StreamsEnded delta must lower active_requests by its count (saturating)"
639                );
640                debug_assert!(
641                    backend.active_requests <= before,
642                    "active_requests must not grow on a StreamsEnded delta"
643                );
644            }
645            BackendChange::ConnectionClosed => {
646                // `dec_connections` floors at 0 (a double close from a
647                // desynced peer must not panic), so the post-relation is
648                // "decreased by one, unless already at zero".
649                let before = backend.active_connections;
650                backend.dec_connections();
651                debug_assert_eq!(
652                    backend.active_connections,
653                    before.saturating_sub(1),
654                    "a ConnectionClosed delta must release exactly one backend connection \
655                     (saturating at 0)"
656                );
657            }
658            BackendChange::DialFailed(at) => {
659                // The one change that draws randomness, so the only one that
660                // borrows the map. `release_failed_dial` carries the
661                // before/after assertions for both halves.
662                backend.release_failed_dial(at, backends.borrow_mut().rng());
663            }
664        }
665        trace!(
666            "{} backend accounting {:?} applied: {:#?}",
667            log_module_context!(),
668            delta.change,
669            backend
670        );
671    }
672}
673
674pub enum Position {
675    Client(sozu_command::state::ClusterId, BackendId, BackendStatus),
676    Server,
677}
678
679impl Debug for Position {
680    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
681        match self {
682            Self::Client(cluster_id, _, status) => f
683                .debug_tuple("Client")
684                .field(cluster_id)
685                .field(status)
686                .finish(),
687            Self::Server => write!(f, "Server"),
688        }
689    }
690}
691
692impl Position {
693    fn is_server(&self) -> bool {
694        match self {
695            Position::Client(..) => false,
696            Position::Server => true,
697        }
698    }
699    fn is_client(&self) -> bool {
700        !self.is_server()
701    }
702
703    /// The bytes-read event for this side.
704    ///
705    /// Returns exactly one event — never zero, never two — because both arms
706    /// of the match below emit one, so the caller needs no container to carry
707    /// it. A caller that is its own shell records it at once; a caller inside
708    /// the H2 core queues it. `#[must_use]` is what makes that a compiler
709    /// obligation rather than a convention: this leaf is reached from a core
710    /// AND from shells, and an event silently dropped here is an upward
711    /// drift that never underflows, so nothing would log and nothing would
712    /// saturate.
713    #[must_use]
714    fn bytes_in_event(&self, size: usize) -> MetricEvent {
715        match self {
716            Position::Client(..) => MetricEvent::BackendBytesIn(size as i64),
717            Position::Server => MetricEvent::FrontendBytesIn(size as i64),
718        }
719    }
720
721    /// The bytes-written event for this side. Same one-event guarantee and
722    /// same reason for `#[must_use]` as [`Self::bytes_in_event`].
723    #[must_use]
724    fn bytes_out_event(&self, size: usize) -> MetricEvent {
725        match self {
726            Position::Client(..) => MetricEvent::BackendBytesOut(size as i64),
727            Position::Server => MetricEvent::FrontendBytesOut(size as i64),
728        }
729    }
730
731    /// Attribute `size` bytes read to the appropriate `SessionMetrics` field.
732    pub fn count_bytes_in(&self, metrics: &mut SessionMetrics, size: usize) {
733        match self {
734            Position::Client(..) => metrics.backend_bin += size,
735            Position::Server => metrics.bin += size,
736        }
737    }
738
739    /// Attribute `size` bytes written to the appropriate `SessionMetrics` field.
740    pub fn count_bytes_out(&self, metrics: &mut SessionMetrics, size: usize) {
741        match self {
742            Position::Client(..) => metrics.backend_bout += size,
743            Position::Server => metrics.bout += size,
744        }
745    }
746}
747
748#[derive(Debug)]
749pub enum BackendStatus {
750    Connecting(Instant),
751    Connected,
752    KeepAlive,
753    Disconnecting,
754}
755
756#[derive(Debug, Clone, Copy)]
757pub enum MuxResult {
758    Continue,
759    Upgrade,
760    CloseSession,
761}
762
763pub trait Endpoint: Debug {
764    fn readiness(&self, token: Token) -> &Readiness;
765    fn readiness_mut(&mut self, token: Token) -> &mut Readiness;
766    /// Smoothed round-trip time of the peer side of a stream, already
767    /// sampled.
768    ///
769    /// Used by access-log emission to report TCP_INFO RTT for the side the
770    /// caller does NOT own directly: a frontend connection (Position::Server)
771    /// reports the backend's RTT through this method, and a backend connection
772    /// (Position::Client) reports the frontend's the same way. `token` is
773    /// ignored by `connection::EndpointServer` (which has a single
774    /// frontend connection) and used as a key by
775    /// `connection::EndpointClient` (which keys backends by token).
776    /// Returns `None` when the token doesn't resolve, mirroring the existing
777    /// fallback paths in `readiness`/`readiness_mut`, and also when the
778    /// platform declines to answer. The value is the peer connection's one
779    /// sample, taken by `memoized_rtt` the first time it is asked for.
780    ///
781    /// This returns the VALUE, not the socket it came from. The predecessor,
782    /// `fn socket(&self, token) -> Option<&TcpStream>`, handed out a
783    /// `mio::net::TcpStream` — a concrete OS type — so any connection could
784    /// reach any other connection's socket for any purpose, and no in-memory
785    /// transport could ever satisfy the trait. RTT is intrinsically a live
786    /// socket property and stays on the embedder's side of the boundary; the
787    /// cores receive an `Option<Duration>` that was captured for them.
788    fn peer_rtt(&self, token: Token) -> Option<Duration>;
789    /// Smoothed round-trip time of the connection this endpoint was handed
790    /// TO — the local side, where [`Self::peer_rtt`] reports the other one.
791    ///
792    /// Asked by `ConnectionH2::snapshot_rtts` for the access log's
793    /// `client_rtt` cell, at the moment a stream is logged and at no other
794    /// time, so a pass that logs nothing costs no `getsockopt(TCP_INFO)`.
795    /// Neither `EndpointClient` nor `EndpointServer` owns the local socket,
796    /// so both answer `None`: the value comes from `h2::ShellEndpoint`, the
797    /// wrapper `H2Shell` puts around them, which samples its own socket once
798    /// per connection through `memoized_rtt`. The core still never reaches
799    /// a socket.
800    fn local_rtt(&self) -> Option<Duration>;
801    /// If end_stream is called on a client it means the stream has PROPERLY finished,
802    /// the server has completed serving the response and informs the endpoint that this stream won't be used anymore.
803    /// If end_stream is called on a server it means the stream was BROKEN, the client was most likely disconnected or encountered an error
804    /// it is for the server to decide if the stream can be retried or an error should be sent. It should be GUARANTEED that all bytes from
805    /// the backend were read. However it is almost certain that all bytes were not already sent to the client.
806    fn end_stream<L: ListenerHandler + L7ListenerHandler>(
807        &mut self,
808        token: Token,
809        stream: GlobalStreamId,
810        context: &mut Context<L>,
811    );
812    /// If start_stream is called on a client it means the stream should be attached to this endpoint,
813    /// the stream might be recovering from a disconnection, in any case at this point its response MUST be empty.
814    /// If the start_stream is called on a H2 server it means the stream is a server push and its request MUST be empty.
815    /// Returns false if the stream could not be started (e.g. max concurrent streams reached).
816    fn start_stream<L: ListenerHandler + L7ListenerHandler>(
817        &mut self,
818        token: Token,
819        stream: GlobalStreamId,
820        context: &mut Context<L>,
821    ) -> bool;
822}
823
824/// Every `getsockopt(TCP_INFO)` the mux issues for an access log goes through
825/// here, and only through [`memoized_rtt`]: once per connection, frontend and
826/// backend alike.
827///
828/// One function so the tests can count the reads: under `cfg(test)` each call
829/// bumps a thread-local counter, read back by [`tcp_info_reads`]. The
830/// production body is the bare `socket_rtt` call.
831pub(super) fn sample_rtt<A: std::os::fd::AsRawFd>(socket: &A) -> Option<Duration> {
832    #[cfg(test)]
833    TCP_INFO_READS.with(|reads| reads.set(reads.get() + 1));
834    socket_rtt(socket)
835}
836
837/// The smoothed RTT of `socket`, read at most once for the connection that
838/// owns `memo` and reused by every later access log of that connection.
839///
840/// `memo` is `None` until the connection's first access log asks for its RTT
841/// — the first request completed on a frontend, the first response logged
842/// through a backend — and holds that one sample, `Some(None)` included when
843/// the platform declined, for the rest of the connection's life. Nothing
844/// resets it: a keep-alive frontend serving twenty requests, or a backend
845/// connection taken back out of the keep-alive pool, costs one
846/// `getsockopt(TCP_INFO)`, not one per request.
847///
848/// The first access log and not `accept`/`connect`: Linux already has a
849/// handshake RTT at `accept` (on loopback, 27–34 µs straight out of
850/// `accept`), so an earlier sample would be valid, but a lazy one is free on
851/// a connection that never logs, and a backend dial is still in flight when
852/// its `connect(2)` returns. The price is a frozen value: the kernel's SRTT
853/// keeps moving over the connection's life and the access log reports the
854/// sample taken at its first request. A `Cell` because `Endpoint::peer_rtt`
855/// and `Endpoint::local_rtt` take `&self`.
856pub(super) fn memoized_rtt<A: std::os::fd::AsRawFd>(
857    memo: &std::cell::Cell<Option<Option<Duration>>>,
858    socket: &A,
859) -> Option<Duration> {
860    if let Some(sample) = memo.get() {
861        return sample;
862    }
863    let sample = sample_rtt(socket);
864    memo.set(Some(sample));
865    debug_assert_eq!(memo.get(), Some(sample), "the first sample must be kept");
866    sample
867}
868
869#[cfg(test)]
870thread_local! {
871    static TCP_INFO_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
872}
873
874/// `getsockopt(TCP_INFO)` reads the calling thread has made through
875/// [`sample_rtt`] so far.
876#[cfg(test)]
877pub(super) fn tcp_info_reads() -> usize {
878    TCP_INFO_READS.with(std::cell::Cell::get)
879}
880
881/// `shutdown(SHUT_WR)` on a socket whose descriptor is about to be closed,
882/// unless the peer has already closed its side.
883///
884/// Every write-side shutdown of the HTTP and HTTPS close paths goes through
885/// here: the frontend in `HttpSession::close` (`lib/src/http.rs`) and
886/// `HttpsSession::close` (`lib/src/https.rs`), the backends in `Mux::close`
887/// and in the dead-backend sweep of `Mux::ready_inner`. A frontend's
888/// lingering close (`ConnectionH1::start_linger`, `h1.rs`, and
889/// `H2Shell::linger_instead_of_closing`, `h2.rs`) calls it earlier, before
890/// it drains what the client still sends; the session close then repeats
891/// it: a no-op once the FIN is queued, or `ENOTCONN` once the connection is
892/// gone, which both callers already ignore. `peer_closed` comes
893/// from [`Connection::peer_closed`], which `HttpsSession::close` widens with
894/// `FrontRustls::peer_disconnected`.
895///
896/// Skipping it changes nothing on the wire. None of these descriptors is
897/// shared (no `dup`, `try_clone` or fork; see `doc/lifetime_of_a_session.md`
898/// §11), so the `close(2)` that follows is the last one, and Linux's
899/// `tcp_close` then sends the FIN after the queued bytes itself when the
900/// receive queue is empty, or a RST when it is not — whether or not a
901/// `shutdown(SHUT_WR)` came first. Once the peer has closed, the shutdown is
902/// therefore a syscall that only reports what the close would do, and on a
903/// peer that has also reset it fails with `ENOTCONN`. HAProxy draws the same
904/// line: `conn_sock_shutw` (`include/haproxy/connection.h`) skips the socket
905/// shutdown once `CO_FL_SOCK_RD_SH` records the read0. The TLS
906/// `close_notify` is not affected: it is written before this call, and RFC
907/// 8446 §6.1 requires it before the write side closes whether or not the
908/// peer sent its own.
909///
910/// Under `cfg(test)` each shutdown actually issued bumps a thread-local
911/// counter, read back by [`shutdown_writes`].
912pub(crate) fn shutdown_write(socket: &TcpStream, peer_closed: bool) -> std::io::Result<()> {
913    if peer_closed {
914        return Ok(());
915    }
916    #[cfg(test)]
917    SHUTDOWN_WRITES.with(|writes| writes.set(writes.get() + 1));
918    socket.shutdown(Shutdown::Write)
919}
920
921#[cfg(test)]
922thread_local! {
923    static SHUTDOWN_WRITES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
924}
925
926/// `shutdown(SHUT_WR)` calls the calling thread has issued through
927/// [`shutdown_write`] so far.
928#[cfg(test)]
929pub(crate) fn shutdown_writes() -> usize {
930    SHUTDOWN_WRITES.with(std::cell::Cell::get)
931}
932
933/// Shared logic for half-close accounting: clear `bit` from `readiness.event` on
934/// socket errors/would-block, and return `true` (yield) when no bytes were
935/// transferred so the caller can park the half. Rationale for clearing only
936/// `bit` (not both halves) on `Closed`: the opposite half may still need one
937/// last pass to flush queued frames or the TLS close_notify.
938fn update_readiness(
939    size: usize,
940    status: SocketResult,
941    readiness: &mut Readiness,
942    bit: Ready,
943) -> bool {
944    trace!(
945        "{}   size={}, status={:?}",
946        log_module_context!(),
947        size,
948        status
949    );
950    match status {
951        SocketResult::Continue => {}
952        SocketResult::Closed | SocketResult::Error | SocketResult::WouldBlock => {
953            readiness.event.remove(bit);
954        }
955    }
956    if size > 0 {
957        false
958    } else {
959        readiness.event.remove(bit);
960        true
961    }
962}
963
964/// [`update_readiness`] for a read, with one exception: a short read after HUP.
965///
966/// The socket layer stops on a short read and answers `(n, WouldBlock)`
967/// without reading on to EOF (`plain_socket_read`, `rustls_socket_read` in
968/// `lib/src/socket.rs`). A FIN that arrived with the data raised its edge
969/// already, and HUP is that edge (`Ready::from(&Event)` in
970/// `command/src/ready.rs`); no later event will announce the EOF, so dropping
971/// READABLE here would leave it unread. Keeping READABLE makes the next pass
972/// read it as `(0, Closed)`, which the H1 close-delimited response path
973/// needs. HAProxy reads `read0` on the same condition (`src/raw_sock.c`,
974/// `FD_POLL_HUP` after `ret < try`), and tokio keeps closed readiness through
975/// `clear_readiness` (`tokio/src/runtime/io/scheduled_io.rs`). The next read
976/// returns bytes or answers with `size == 0`, which drops READABLE, so the
977/// exception cannot spin.
978fn update_readiness_after_read(
979    size: usize,
980    status: SocketResult,
981    readiness: &mut Readiness,
982) -> bool {
983    if size > 0 && status == SocketResult::WouldBlock && readiness.event.is_hup() {
984        return false;
985    }
986    update_readiness(size, status, readiness, Ready::READABLE)
987}
988
989fn update_readiness_after_write(
990    size: usize,
991    status: SocketResult,
992    readiness: &mut Readiness,
993) -> bool {
994    update_readiness(size, status, readiness, Ready::WRITABLE)
995}
996pub struct Context<L: ListenerHandler + L7ListenerHandler> {
997    pub streams: Vec<Stream>,
998    /// Streams whose state is `StreamState::Link` and need backend connection.
999    /// Replaces the O(n) scan of `streams` in the ready loop.
1000    pub pending_links: VecDeque<GlobalStreamId>,
1001    /// Reverse index: backend token -> global stream IDs currently in
1002    /// `StreamState::Linked(token)`. Eliminates O(n) scans of `streams`
1003    /// when handling backend connect/disconnect/timeout/close events.
1004    ///
1005    /// An entry may be empty: `remove_backend_stream` keeps an emptied `Vec`
1006    /// for the next link on the same live connection (#1583), so "has linked
1007    /// streams" is the entry's emptiness, not the key's presence.
1008    pub backend_streams: InlineTokenMap<LinkedStreams>,
1009    /// Where every buffer this session's streams and connections need comes
1010    /// from, and the only thing allowed to refuse one.
1011    ///
1012    /// Boxed rather than concrete so the core names the contract rather than
1013    /// the worker's [`Pool`]: a refusal is the same event whatever stands
1014    /// behind it, and the `REFUSED_STREAM` answer to one is the same answer.
1015    /// [`Context::new`] installs the [`PoolBufferSource`] the proxy runs on.
1016    pub buffers: Box<dyn BufferSource>,
1017    pub listener: Rc<RefCell<L>>,
1018    /// Connection/session ULID — mirrors `Mux.session_ulid`. Stored here so
1019    /// per-stream `HttpContext` construction in [`Self::create_stream`] can
1020    /// stamp the session slot of the log-context bracket without reaching
1021    /// back into the parent [`Mux`].
1022    pub session_ulid: Ulid,
1023    pub session_address: Option<SocketAddr>,
1024    pub public_address: SocketAddr,
1025    pub debug: DebugHistory,
1026    /// Shrink threshold ratio for recycled stream slots.
1027    /// Vec is shrunk when total_slots > active_streams * ratio.
1028    pub h2_stream_shrink_ratio: usize,
1029    /// TLS SNI value negotiated at handshake, propagated to every
1030    /// per-stream [`HttpContext`] so the routing layer can enforce
1031    /// the SNI ↔ `:authority` binding on every H2 stream (and the
1032    /// single H1 request). `None` for plaintext listeners or when
1033    /// the client omitted the SNI extension. Stored pre-lowercased
1034    /// and without a port for cheap exact-match comparison.
1035    pub tls_server_name: Option<String>,
1036    /// Snapshot of the SAN set of the certificate Sōzu actually served at
1037    /// the TLS handshake. Captured once in `https.rs::upgrade_handshake`
1038    /// from the resolver and frozen for the connection lifetime so H2
1039    /// stream coalescing (RFC 7540 §9.1.1 / RFC 9113 §9.1.1) accepts any
1040    /// `:authority` covered by the certificate, with RFC 6125 §6.4.3
1041    /// wildcard handling. `None` for plaintext listeners or when SNI was
1042    /// absent. `Some(empty)` when the default cert was served — every
1043    /// `:authority` is rejected. `Arc` so the snapshot is shared across
1044    /// every per-stream `HttpContext` without re-allocation.
1045    pub tls_cert_names: Option<Arc<Vec<String>>>,
1046    /// Whether this session's listener speaks HTTPS or plaintext HTTP.
1047    ///
1048    /// The one listener value that is genuinely per-connection rather than
1049    /// per-request: a listener's protocol is fixed by its type and no
1050    /// `update_config` patch can change it, so capturing it once in
1051    /// [`Self::new`] frees both readers — [`Self::create_stream`] and the H2
1052    /// write pass's `:scheme` decision — from borrowing the listener at all.
1053    /// Contrast the per-request knobs, which [`Self::create_stream`] re-reads
1054    /// on every stream precisely because a reload *can* change them
1055    /// (`LIFECYCLE.md` §2.5).
1056    pub protocol: Protocol,
1057    /// Negotiated TLS protocol version short-form (e.g. `"TLSv1.3"`).
1058    /// Captured once at handshake completion in `https.rs` and propagated
1059    /// to every per-stream [`HttpContext`] so the access log can record it
1060    /// without reaching back into the rustls session per request. `None`
1061    /// for plaintext listeners.
1062    pub tls_version: Option<&'static str>,
1063    /// Negotiated TLS cipher suite short-form (e.g.
1064    /// `"TLS_AES_128_GCM_SHA256"`). Captured once at handshake completion
1065    /// and propagated to every per-stream [`HttpContext`]. `None` for
1066    /// plaintext listeners.
1067    pub tls_cipher: Option<&'static str>,
1068    /// Negotiated ALPN protocol short-form (e.g. `"h2"`, `"http/1.1"`).
1069    /// Captured once at handshake completion and propagated to every
1070    /// per-stream [`HttpContext`]. `None` for plaintext listeners or when
1071    /// no ALPN was negotiated.
1072    pub tls_alpn: Option<&'static str>,
1073    /// Clock snapshot for the pass currently executing.
1074    ///
1075    /// [`Mux`] is the only clock sampler in the mux: it refreshes this field
1076    /// once per outer [`Mux::ready`] pass and at the top of [`Mux::timeout`]
1077    /// and [`Mux::shutting_down`]. Every time-based decision in the H2 core
1078    /// reads this snapshot — mirrored into
1079    /// `h2::ConnectionH2::now` at each public entry point — instead of
1080    /// calling [`Instant::now`] itself, so one pass sees one consistent
1081    /// "now". Every deadline armed or evaluated inside a pass is therefore
1082    /// accurate to within that pass — **in either direction**. The error is
1083    /// not one-sided: an arm site runs at an arbitrary depth into its pass
1084    /// (a DATA or HEADERS refresh, an outbound-byte refresh) and stamps the
1085    /// snapshot taken at the START of that pass, so the stored instant is
1086    /// older than the event it records; an eval site runs near the top of a
1087    /// pass (`cancel_timed_out_streams` is the first thing
1088    /// `h2::ConnectionH2::poll_read_target` does). The measured age is therefore
1089    /// inflated by the arm site's depth, and a deadline can fire up to one
1090    /// pass EARLY as well as one pass late. At base both ends read the real
1091    /// clock and the comparison was exact; this is the cost of the snapshot.
1092    ///
1093    /// The flood and back-pressure windows are the one asymmetric case, and
1094    /// they fail closed: `now` is constant for the whole pass, so a window
1095    /// cannot decay part-way through one. A burst arriving during a pass is
1096    /// weighed in full against the window that was open when the pass
1097    /// started, where before the change a long pass could decay the counters
1098    /// under the burst. The window BOUNDARY still carries the same
1099    /// one-pass error in either direction as every other deadline.
1100    ///
1101    /// Note that [`crate::SessionMetrics`] and [`crate::timer::TimeoutContainer`]
1102    /// deliberately stay on the real clock: metrics want true elapsed time,
1103    /// and the timeout containers are the embedder's timer, held by the [`Mux`]
1104    /// adapter in [`Mux::timeouts`] and never touched by a core — which
1105    /// publishes a deadline stamped from this snapshot instead.
1106    pub now: Instant,
1107    /// Wall-clock sibling of [`Self::now`]: Unix milliseconds for the pass
1108    /// currently executing.
1109    ///
1110    /// [`Self::now`] is an [`Instant`] — an opaque monotonic point with no
1111    /// epoch and no defined conversion to civil time — so it cannot supply
1112    /// the 48-bit Unix-millisecond prefix a ULID carries in its high bits
1113    /// (see [`Self::next_request_id`]). This field is that value, and it is
1114    /// refreshed by [`Mux`] at exactly the three sites that refresh
1115    /// [`Self::now`], from the same pass. The two are siblings: a site that
1116    /// sets one and not the other lets a request id carry the timestamp of
1117    /// an older pass.
1118    ///
1119    /// It is a snapshot rather than a provider closure for the same reason
1120    /// [`Self::now`] is: the mux's discipline is one clock sample per pass,
1121    /// read by the core from a field, and a boxed provider would let the
1122    /// core reach the host at an arbitrary depth into a pass instead.
1123    pub now_wall_ms: u64,
1124    /// Entropy source for the 80-bit random suffix of a request ULID.
1125    ///
1126    /// `Ulid::generate()` is exactly
1127    /// `Ulid::from_timestamp_with_rng(unix_epoch_ms(), &mut rand::rng())` —
1128    /// two ambient reaches (the wall clock and a thread-local RNG) hidden
1129    /// inside a dependency, invisible to any grep over this module for
1130    /// `Instant::now` / `SystemTime::now` / `rand`. Holding the RNG here
1131    /// replaces the second with a field the embedder owns, exactly as
1132    /// [`Self::now_wall_ms`] replaces the first.
1133    ///
1134    /// Seeded once per session in [`Self::new`] from `rand::rng()` — the
1135    /// same OS-backed reseeding CSPRNG `Ulid::generate()` draws from, so the
1136    /// unpredictability of a request id is unchanged. A request id that
1137    /// repeats across sessions would be a regression, so nothing in the mux
1138    /// re-seeds it; a deterministic simulator assigns this field directly
1139    /// after construction instead.
1140    pub request_id_rng: StdRng,
1141    /// Backend accounting the core decided and the embedder has not performed
1142    /// yet (#1340, Question 12).
1143    ///
1144    /// The core's outbound channel for registry writes, in the same place
1145    /// `udp`'s `Output` queue sits: a site that used to reach a
1146    /// `Rc<RefCell<Backend>>` through `Position::Client` and mutate it pushes
1147    /// a [`BackendDelta`] here instead. `Mux::apply_backend_deltas` is the
1148    /// only drain, and the only writer of `Backend::active_requests` /
1149    /// `Backend::active_connections` on the mux path.
1150    ///
1151    /// Entries are applied in the order they were pushed, and the drain runs
1152    /// before anything reads backend load state — in practice, before each
1153    /// `Router::plan_connect`, whose load balancer is that sole reader. FIFO plus
1154    /// drain-before-read leaves the counters at every read point exactly
1155    /// where mutating in place left them, saturation included.
1156    pub backend_deltas: Vec<BackendDelta>,
1157    /// Response bytes this session's frontend has handed to its socket, over
1158    /// the whole session. Only ever grows (wrapping), unlike the per-stream
1159    /// `SessionMetrics::bout`, which `SessionMetrics::reset` zeroes when a
1160    /// stream ends and its slot is reused. `ConnectionH1::writable` and
1161    /// `H2Shell::write_streams` add every stream byte a server-position
1162    /// write accepted.
1163    pub(super) client_bytes_out: usize,
1164    /// [`Self::client_bytes_out`] when the inner loop of `Mux::ready_inner`
1165    /// last reached `MAX_LOOP_ITERATIONS` and waited for the client instead
1166    /// of closing. A second time with no byte written to the client since
1167    /// closes the session: waiting is only worth it while the client drains
1168    /// (`Mux::wait_for_client_at_loop_limit`).
1169    pub(super) loop_limit_bytes_out: Option<usize>,
1170    /// Stream bytes this session has handed to any socket, the client's or
1171    /// a backend's, over the whole session. Only ever grows (wrapping).
1172    /// `ConnectionH1::writable` and `H2Shell::write_streams` add every stream
1173    /// byte a write accepted, whatever the connection's position; H2 control
1174    /// frames are not stream bytes and do not count. When the inner loop of
1175    /// `Mux::ready_inner` reaches `MAX_LOOP_ITERATIONS` and this moved by at
1176    /// least one byte per budgeted iteration during the call
1177    /// (`progressed_through_loop_budget`), the session is progressing, not
1178    /// spinning: it yields to the event loop instead of closing
1179    /// (sozu-proxy/sozu#1906).
1180    pub(super) stream_bytes_out: usize,
1181    /// Set when the last `Mux::ready_inner` call yielded at
1182    /// `MAX_LOOP_ITERATIONS` with its readiness left as it was. The session
1183    /// reports it through `ProxySession::yielded`, which takes it, and
1184    /// `Server::resume_yielded_sessions` runs `ready` again on the next turn
1185    /// of the event loop: no socket edge would come back for work the loop
1186    /// left unread.
1187    pub(super) loop_budget_yielded: bool,
1188}
1189
1190/// Unix milliseconds for the wall clock, saturating to 0 before the epoch.
1191///
1192/// The single host wall-clock reach behind [`Context::now_wall_ms`]. Kept as
1193/// one function so the three [`Mux`] refresh sites and [`Context::new`] cannot
1194/// drift apart on the unit or on the pre-epoch case.
1195fn unix_epoch_ms() -> u64 {
1196    SystemTime::now()
1197        .duration_since(UNIX_EPOCH)
1198        .map(|d| d.as_millis() as u64)
1199        .unwrap_or(0)
1200}
1201
1202impl<L: ListenerHandler + L7ListenerHandler> Context<L> {
1203    pub fn new(
1204        session_ulid: Ulid,
1205        pool: Weak<RefCell<Pool>>,
1206        listener: Rc<RefCell<L>>,
1207        session_address: Option<SocketAddr>,
1208        public_address: SocketAddr,
1209    ) -> Self {
1210        let h2_stream_shrink_ratio = listener
1211            .borrow()
1212            .get_h2_connection_config()
1213            .stream_shrink_ratio as usize;
1214        let protocol = listener.borrow().protocol();
1215        Self {
1216            streams: Vec::new(),
1217            pending_links: VecDeque::new(),
1218            backend_streams: InlineTokenMap::new(),
1219            buffers: Box::new(PoolBufferSource::new(pool)),
1220            listener,
1221            session_ulid,
1222            session_address,
1223            public_address,
1224            debug: DebugHistory::new(),
1225            h2_stream_shrink_ratio,
1226            tls_server_name: None,
1227            tls_cert_names: None,
1228            protocol,
1229            tls_version: None,
1230            tls_cipher: None,
1231            tls_alpn: None,
1232            now: Instant::now(),
1233            now_wall_ms: unix_epoch_ms(),
1234            request_id_rng: StdRng::from_rng(&mut rand::rng()),
1235            backend_deltas: Vec::new(),
1236            client_bytes_out: 0,
1237            loop_limit_bytes_out: None,
1238            stream_bytes_out: 0,
1239            loop_budget_yielded: false,
1240        }
1241    }
1242
1243    /// Record one backend accounting change for the embedder to perform.
1244    ///
1245    /// The single push site, so every emitter reads the same way and a
1246    /// `grep` for `record_backend_delta` enumerates the ledger's inputs.
1247    pub(super) fn record_backend_delta(&mut self, backend: &BackendId, change: BackendChange) {
1248        self.backend_deltas.push(BackendDelta {
1249            slot: backend.slot(),
1250            change,
1251        });
1252    }
1253
1254    /// Mint the request ULID for a stream this session is about to open, or
1255    /// for the next request of an H1 keep-alive connection.
1256    ///
1257    /// The mux's two production ULID sites call it: `ConnectionH2::create_stream`
1258    /// (`lib/src/protocol/mux/h2.rs`) for each H2 stream, and the keep-alive
1259    /// branch of `ConnectionH1::writable` (`lib/src/protocol/mux/h1.rs`), which
1260    /// hands it to `HttpContext::reset`. It replaces `Ulid::generate()` there.
1261    /// `generate()` composes two ambient reaches — the wall clock and a
1262    /// thread-local RNG — inside `rusty_ulid`; this composes the same ULID
1263    /// from [`Self::now_wall_ms`] and [`Self::request_id_rng`], the two
1264    /// sources the embedder owns. Same bytes, same layout, no host reach
1265    /// from inside the core.
1266    ///
1267    /// `Ulid::from_timestamp_with_rng` panics above `0xFFFF_FFFF_FFFF` ms
1268    /// (year 10889); [`Self::now_wall_ms`] is a Unix-millisecond count, so
1269    /// reaching it requires a host clock set eight millennia ahead.
1270    pub fn next_request_id(&mut self) -> Ulid {
1271        Ulid::from_timestamp_with_rng(self.now_wall_ms, &mut self.request_id_rng)
1272    }
1273
1274    pub fn active_len(&self) -> usize {
1275        self.streams
1276            .iter()
1277            .filter(|s| !matches!(s.state, StreamState::Recycle))
1278            .count()
1279    }
1280
1281    /// Shared accessor for the [`HttpContext`] owned by a stream.
1282    ///
1283    /// Prefer this over `&self.streams[stream_id].context` at call sites
1284    /// that only need read access — it keeps the `Stream`/`HttpContext`
1285    /// relationship encapsulated and reads the same regardless of whether
1286    /// the caller is inside `Router::plan_connect`, the H2 mux, or a free
1287    /// helper. Panics on an out-of-bounds `stream_id`, which is the same
1288    /// behaviour as the raw `streams[sid]` indexing it replaces.
1289    pub fn http_context(&self, stream_id: GlobalStreamId) -> &HttpContext {
1290        &self.streams[stream_id].context
1291    }
1292
1293    /// Mutable sibling of [`Self::http_context`]. Use when routing
1294    /// decisions need to stamp `cluster_id` / `backend_id` on the stream's
1295    /// [`HttpContext`] (e.g. `Router::plan_connect` at the fill-cluster /
1296    /// fill-backend points).
1297    pub fn http_context_mut(&mut self, stream_id: GlobalStreamId) -> &mut HttpContext {
1298        &mut self.streams[stream_id].context
1299    }
1300
1301    /// Register a stream as linked to a backend token in the reverse index.
1302    pub fn link_stream(&mut self, stream_id: GlobalStreamId, token: Token) {
1303        self.streams[stream_id].state = StreamState::Linked(token);
1304        // An H1 backend carries one stream at a time, which a new entry
1305        // holds inline (#1613); an H2 backend spills into a `Vec`.
1306        self.backend_streams
1307            .get_or_insert_with(token, LinkedStreams::default)
1308            .push(stream_id);
1309    }
1310
1311    /// Remove a stream from the backend reverse index if it is currently
1312    /// `Linked`. Returns the backend token if one was removed.
1313    pub fn unlink_stream(&mut self, stream_id: GlobalStreamId) -> Option<Token> {
1314        if let StreamState::Linked(token) = self.streams[stream_id].state {
1315            remove_backend_stream(&mut self.backend_streams, token, stream_id);
1316            Some(token)
1317        } else {
1318            None
1319        }
1320    }
1321
1322    /// Open (or recycle) the stream slot for a request that has just arrived,
1323    /// capturing the listener configuration it will run on.
1324    ///
1325    /// This is the mux's single listener-snapshot point. `ConnectionH2` calls
1326    /// it from its HEADERS branch, so an H2 stream captures the listener as
1327    /// the operator had it configured when that request's HEADERS landed, and
1328    /// finishes on that capture whatever a concurrent reload does; the next
1329    /// HEADERS on the same connection takes a fresh one. The H1 sessions in
1330    /// `http.rs` / `https.rs` call it once per connection instead, because one
1331    /// H1 connection owns one slot that `HttpContext::reset` deliberately
1332    /// carries across keep-alive requests. `LIFECYCLE.md` §2.5 states the
1333    /// resulting staleness window for both.
1334    ///
1335    /// Every value read under the borrow below is re-read on each call for
1336    /// that reason. The two that are NOT read here — [`Self::protocol`] and
1337    /// the TLS fields — are connection-scoped by nature and captured once.
1338    ///
1339    /// `window` seeds [`Stream::front_window`], the stream's send window on
1340    /// the frontend connection. The backend leg's window is the backend's to
1341    /// size: `ConnectionH2::start_stream` sets [`Stream::back_window`].
1342    pub fn create_stream(&mut self, request_id: Ulid, window: u32) -> Option<GlobalStreamId> {
1343        let (http_context, answers) = {
1344            let listener = self.listener.borrow();
1345            let mut http_context = HttpContext::new(
1346                self.session_ulid,
1347                request_id,
1348                self.protocol,
1349                self.public_address,
1350                self.session_address,
1351                listener.get_sticky_name().to_string(),
1352                listener.get_sozu_id_header().to_string(),
1353                listener.get_elide_x_real_ip(),
1354                listener.get_send_x_real_ip(),
1355            );
1356            // Propagate the connection-scoped TLS SNI onto every per-stream
1357            // HttpContext so `route_from_request` can enforce the SNI ↔
1358            // `:authority` binding for each H2 stream independently.
1359            http_context.tls_server_name = self.tls_server_name.clone();
1360            // Mirror the frozen-at-handshake SAN snapshot. `Arc` clone is a
1361            // refcount bump, not a deep copy — every per-stream
1362            // `HttpContext` shares the same `Vec<String>`.
1363            http_context.tls_cert_names = self.tls_cert_names.clone();
1364            // Mirror the listener's strict_sni_binding flag onto each
1365            // HttpContext so the routing layer can honor operator opt-outs
1366            // without reaching back into the listener on every request.
1367            http_context.strict_sni_binding = listener.get_strict_sni_binding();
1368            // Mirror the listener's forwarding header family the same way:
1369            // a hot update reaches the next stream (H2) or connection (H1).
1370            http_context.forwarded_headers = listener.get_forwarded_headers();
1371            // The H1 trailer field bound is the listener's H2 one, so both
1372            // frontends admit the same number of trailer fields (#1701).
1373            http_context.max_trailer_fields = listener.get_h2_flood_config().max_header_fields();
1374            // Propagate the connection-scoped TLS metadata onto every
1375            // per-stream HttpContext so the access log can record it without
1376            // touching the rustls session on every request. These are
1377            // `&'static str` borrows from the rustls label tables — copy is
1378            // a pointer move.
1379            http_context.tls_version = self.tls_version;
1380            http_context.tls_cipher = self.tls_cipher;
1381            http_context.tls_alpn = self.tls_alpn;
1382            // The answer registry this request will render any default answer
1383            // from. Cloning the handle here — rather than borrowing the
1384            // listener again at each `set_default_answer` site — is what binds
1385            // a stream to the templates that were installed when it started.
1386            (http_context, listener.get_answers().clone())
1387        };
1388        let recycle_slot = self
1389            .streams
1390            .iter()
1391            .position(|s| s.state == StreamState::Recycle);
1392        if let Some(stream_id) = recycle_slot {
1393            let stream = &mut self.streams[stream_id];
1394            trace!("{} Reuse stream: {}", log_module_context!(), stream_id);
1395            stream.state = StreamState::Idle;
1396            stream.attempts = 0;
1397            stream.tried_backends.clear();
1398            stream.front_received_end_of_stream = false;
1399            stream.back_received_end_of_stream = false;
1400            stream.front_data_received = 0;
1401            stream.back_data_received = 0;
1402            // The ONE place `request_counted` is cleared without emitting the
1403            // paired `http.active_requests` decrement, and it must stay the
1404            // only one. A slot reaches `StreamState::Recycle` only through
1405            // `Stream::generate_access_log`, which already emitted the `-1`
1406            // and cleared the flag, so this is re-initialisation of an
1407            // already-false field rather than a decrement that went missing.
1408            // `Stream::new` asserts the same invariant for a fresh slot
1409            // ("new stream must not have a counted request"); this is its
1410            // reuse-path counterpart. A future site that clears this flag
1411            // without accounting for the gauge is a silent upward leak: the
1412            // `+1` stays in the aggregate with nothing left to pair it, and
1413            // an aggregate that only drifts up never underflows, so nothing
1414            // logs and nothing saturates.
1415            stream.request_counted = false;
1416            stream.front_window = i32::try_from(window).unwrap_or(i32::MAX);
1417            stream.back_window = i32::try_from(h2::DEFAULT_INITIAL_WINDOW_SIZE).unwrap_or(i32::MAX);
1418            stream.context = http_context;
1419            // A recycled slot takes the fresh capture too: the request that
1420            // released it ran on whatever the listener held then, and the one
1421            // taking it over must not inherit that.
1422            stream.answers = answers;
1423            stream.back.clear();
1424            stream.back.storage.clear();
1425            stream.front.clear();
1426            stream.front.storage.clear();
1427            // The request that released the slot may have been encoded for
1428            // an H2 backend; the one taking it over was not.
1429            stream.front_bound_to_backend = false;
1430            stream.forget_upstream_replay();
1431            stream.metrics.reset();
1432            stream.metrics.mark_request_start();
1433            // After recycling a slot, check if the Vec has excessive trailing
1434            // Recycle entries (more than 2x active streams of total capacity).
1435            let active = self.active_len();
1436            let total = self.streams.len();
1437            if total > 1 && active > 0 && total > active * self.h2_stream_shrink_ratio {
1438                self.shrink_trailing_recycle();
1439            }
1440            return Some(stream_id);
1441        }
1442        let stream = Stream::new(&mut *self.buffers, http_context, answers, window)?;
1443        self.streams.push(stream);
1444        Some(self.streams.len() - 1)
1445    }
1446
1447    /// Remove consecutive `Recycle` entries from the end of the streams Vec.
1448    ///
1449    /// This prevents unbounded growth when H2 streams are created and recycled
1450    /// over time, reclaiming memory from slots that are no longer needed.
1451    pub fn shrink_trailing_recycle(&mut self) {
1452        while self
1453            .streams
1454            .last()
1455            .is_some_and(|s| s.state == StreamState::Recycle)
1456        {
1457            self.streams.pop();
1458        }
1459    }
1460}
1461
1462/// A map keyed by [`Token`] that holds its lowest-token entry inline.
1463///
1464/// A session almost always has exactly one backend connection, so the two
1465/// per-backend indexes a mux session keeps — [`Context::backend_streams`] and
1466/// the backend half of [`MuxTimeouts`] — almost always hold one entry. A
1467/// `HashMap` allocated its table on that first insert, once per session and
1468/// so once per request on a one-request connection (#1613). Here the first
1469/// entry costs nothing and only a second backend reaches the overflow map.
1470///
1471/// The shape is [`BackendConnections`](router::BackendConnections)'s: `first`
1472/// is `None` only when `rest` is empty, and its token is below every token in
1473/// `rest`, so iteration runs in ascending `Token` order.
1474pub struct InlineTokenMap<V> {
1475    first: Option<(Token, V)>,
1476    rest: BTreeMap<Token, V>,
1477}
1478
1479impl<V> InlineTokenMap<V> {
1480    pub const fn new() -> Self {
1481        Self {
1482            first: None,
1483            rest: BTreeMap::new(),
1484        }
1485    }
1486
1487    pub fn is_empty(&self) -> bool {
1488        self.first.is_none()
1489    }
1490
1491    pub fn contains_key(&self, token: &Token) -> bool {
1492        self.get(token).is_some()
1493    }
1494
1495    pub fn get(&self, token: &Token) -> Option<&V> {
1496        match &self.first {
1497            Some((first, value)) if first == token => Some(value),
1498            _ => self.rest.get(token),
1499        }
1500    }
1501
1502    pub fn get_mut(&mut self, token: &Token) -> Option<&mut V> {
1503        match &mut self.first {
1504            Some((first, value)) if first == token => Some(value),
1505            _ => self.rest.get_mut(token),
1506        }
1507    }
1508
1509    /// The value under `token`, inserting `default()` first when there is none.
1510    pub fn get_or_insert_with(&mut self, token: Token, default: impl FnOnce() -> V) -> &mut V {
1511        let first_token = self.first.as_ref().map(|(first, _)| *first);
1512        if first_token.is_some_and(|first| token > first) {
1513            return self.rest.entry(token).or_insert_with(default);
1514        }
1515        if first_token.is_some_and(|first| token < first)
1516            && let Some((displaced_token, displaced)) = self.first.take()
1517        {
1518            self.rest.insert(displaced_token, displaced);
1519        }
1520        &mut self.first.get_or_insert_with(|| (token, default())).1
1521    }
1522
1523    /// Remove and return the value under `token`.
1524    pub fn remove(&mut self, token: &Token) -> Option<V> {
1525        match &self.first {
1526            Some((first, _)) if first == token => {
1527                let removed = std::mem::replace(&mut self.first, self.rest.pop_first());
1528                removed.map(|(_, value)| value)
1529            }
1530            _ => self.rest.remove(token),
1531        }
1532    }
1533
1534    /// Keep only the entries `keep` answers `true` for.
1535    pub fn retain(&mut self, mut keep: impl FnMut(&Token, &mut V) -> bool) {
1536        let keep_first = match &mut self.first {
1537            Some((token, value)) => keep(token, value),
1538            None => true,
1539        };
1540        self.rest.retain(|token, value| keep(token, value));
1541        if !keep_first {
1542            self.first = self.rest.pop_first();
1543        }
1544    }
1545
1546    pub fn clear(&mut self) {
1547        self.first = None;
1548        self.rest.clear();
1549    }
1550
1551    /// The entries in ascending `Token` order.
1552    pub fn iter(&self) -> impl Iterator<Item = (&Token, &V)> {
1553        self.first
1554            .iter()
1555            .map(|(token, value)| (token, value))
1556            .chain(self.rest.iter())
1557    }
1558
1559    pub fn values(&self) -> impl Iterator<Item = &V> {
1560        self.iter().map(|(_, value)| value)
1561    }
1562}
1563
1564impl<V> Default for InlineTokenMap<V> {
1565    fn default() -> Self {
1566        Self::new()
1567    }
1568}
1569
1570impl<V: Debug> Debug for InlineTokenMap<V> {
1571    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1572        f.debug_map().entries(self.iter()).finish()
1573    }
1574}
1575
1576/// The streams linked to one backend connection, in link order.
1577///
1578/// An H1 backend carries one stream at a time, so the first link is held
1579/// inline and allocates nothing; a second concurrent link (an H2 backend)
1580/// moves both into a `Vec` sized for four — the allocation the `Vec` this
1581/// replaces made at the same point — and the entry keeps that capacity for
1582/// the rest of the connection's life (#1583).
1583pub enum LinkedStreams {
1584    Inline(Option<GlobalStreamId>),
1585    Spilled(Vec<GlobalStreamId>),
1586}
1587
1588impl LinkedStreams {
1589    pub fn push(&mut self, stream_id: GlobalStreamId) {
1590        match self {
1591            LinkedStreams::Inline(None) => *self = LinkedStreams::Inline(Some(stream_id)),
1592            LinkedStreams::Inline(Some(held)) => {
1593                let mut ids = Vec::with_capacity(4);
1594                ids.push(*held);
1595                ids.push(stream_id);
1596                *self = LinkedStreams::Spilled(ids);
1597            }
1598            LinkedStreams::Spilled(ids) => ids.push(stream_id),
1599        }
1600    }
1601
1602    /// Drop `stream_id` from the list, keeping the order of the others.
1603    pub fn remove(&mut self, stream_id: GlobalStreamId) {
1604        match self {
1605            LinkedStreams::Inline(held) => {
1606                if *held == Some(stream_id) {
1607                    *held = None;
1608                }
1609            }
1610            LinkedStreams::Spilled(ids) => ids.retain(|&id| id != stream_id),
1611        }
1612    }
1613}
1614
1615impl Default for LinkedStreams {
1616    fn default() -> Self {
1617        LinkedStreams::Inline(None)
1618    }
1619}
1620
1621impl std::ops::Deref for LinkedStreams {
1622    type Target = [GlobalStreamId];
1623
1624    fn deref(&self) -> &[GlobalStreamId] {
1625        match self {
1626            LinkedStreams::Inline(held) => held.as_slice(),
1627            LinkedStreams::Spilled(ids) => ids,
1628        }
1629    }
1630}
1631
1632impl Debug for LinkedStreams {
1633    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1634        f.debug_list().entries(self.iter()).finish()
1635    }
1636}
1637
1638/// The `Mux` adapter's timer-wheel handles: the frontend's, and one per live
1639/// backend connection.
1640///
1641/// The frontend's key never changes for the life of a session, so its handle
1642/// is a named field; the backends' are an open set, held in an
1643/// [`InlineTokenMap`] so the usual single backend costs no allocation. The
1644/// `HashMap<Token, TimeoutContainer>` this replaces allocated its table when
1645/// the session was built (#1613).
1646///
1647/// HAProxy goes further: a stream's deadlines are fixed fields (`conn_exp` in
1648/// `include/haproxy/stream-t.h`, each channel's `analyse_exp` in
1649/// `include/haproxy/channel-t.h`) that `process_stream` folds into its one
1650/// task's `expire` with `tick_first` (`src/stream.c`), so there is no table at
1651/// all. A sozu session keeps one wheel entry per connection instead, because
1652/// each connection's core publishes its own deadline.
1653#[derive(Debug)]
1654pub struct MuxTimeouts {
1655    pub frontend: TimeoutContainer,
1656    pub backends: InlineTokenMap<TimeoutContainer>,
1657}
1658
1659impl MuxTimeouts {
1660    pub const fn new(frontend: TimeoutContainer) -> Self {
1661        Self {
1662            frontend,
1663            backends: InlineTokenMap::new(),
1664        }
1665    }
1666}
1667
1668/// Remove `stream_id` from the backend-token reverse index for `token`.
1669/// Free function to allow split borrows when `context.streams` is already
1670/// mutably borrowed (preventing a `Context::unlink_stream` call).
1671///
1672/// An emptied entry is kept, not removed: the next `Context::link_stream` on
1673/// the same backend connection — every request on a keep-alive socket or an
1674/// H2 multiplex slot — pushes into its retained capacity instead of
1675/// allocating a fresh `Vec` (#1583). The key leaves with the connection
1676/// itself, in the dead-backend sweep of `Mux::ready_inner`, and `Mux::close`
1677/// clears the whole index, so the index stays bounded by the session's live
1678/// backend connections.
1679/// [`connect_outcome`] against the three states a dialled socket can be in
1680/// when sozu sees WRITABLE on it (sozu-proxy/sozu#1800).
1681#[cfg(test)]
1682mod connect_outcome_tests {
1683    use std::{
1684        net::{SocketAddr, TcpListener},
1685        os::fd::AsRawFd,
1686        time::{Duration, Instant},
1687    };
1688
1689    use super::{ConnectOutcome, connect_outcome};
1690
1691    /// Poll `connect_outcome` until it leaves `Pending` or `deadline` passes.
1692    fn settle(socket: &mio::net::TcpStream, deadline: Duration) -> ConnectOutcome {
1693        let started = Instant::now();
1694        loop {
1695            let outcome = connect_outcome(socket);
1696            if outcome != ConnectOutcome::Pending || started.elapsed() >= deadline {
1697                return outcome;
1698            }
1699            std::thread::sleep(Duration::from_millis(10));
1700        }
1701    }
1702
1703    #[test]
1704    fn an_accepted_dial_is_connected() {
1705        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1706        let socket = mio::net::TcpStream::connect(listener.local_addr().unwrap()).unwrap();
1707        assert_eq!(
1708            settle(&socket, Duration::from_secs(2)),
1709            ConnectOutcome::Connected
1710        );
1711    }
1712
1713    /// A `unix_path` backend's socket carries an AF_UNIX peer, which std's
1714    /// `peer_addr` refuses: the outcome must still read Connected.
1715    #[test]
1716    fn an_accepted_unix_dial_is_connected() {
1717        let path =
1718            std::env::temp_dir().join(format!("sozu-connect-outcome-{}.sock", std::process::id()));
1719        let _ = std::fs::remove_file(&path);
1720        let _listener = std::os::unix::net::UnixListener::bind(&path).unwrap();
1721        let unused: SocketAddr = "127.0.0.1:1".parse().unwrap();
1722        let socket = crate::backends::connect_backend(unused, path.to_str()).unwrap();
1723        let outcome = settle(&socket, Duration::from_secs(2));
1724        std::fs::remove_file(&path).unwrap();
1725        assert_eq!(outcome, ConnectOutcome::Connected);
1726    }
1727
1728    #[test]
1729    fn a_refused_dial_failed() {
1730        let address: SocketAddr = {
1731            let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1732            listener.local_addr().unwrap()
1733        };
1734        // A loopback refusal may also be reported by `connect(2)` itself, in
1735        // which case there is no socket to read the outcome from.
1736        let Ok(socket) = mio::net::TcpStream::connect(address) else {
1737            return;
1738        };
1739        assert_eq!(
1740            settle(&socket, Duration::from_secs(2)),
1741            ConnectOutcome::Failed
1742        );
1743    }
1744
1745    #[test]
1746    fn a_dial_nobody_answers_is_pending() {
1747        // A full accept queue drops the SYN, so the dial stays in flight.
1748        let listener = TcpListener::bind("127.0.0.1:0").unwrap();
1749        // SAFETY: shrinks the backlog of a socket this test owns.
1750        assert_eq!(unsafe { libc::listen(listener.as_raw_fd(), 0) }, 0);
1751        let address = listener.local_addr().unwrap();
1752        let mut queued = Vec::new();
1753        while let Ok(stream) =
1754            std::net::TcpStream::connect_timeout(&address, Duration::from_millis(300))
1755        {
1756            queued.push(stream);
1757            assert!(queued.len() < 16, "the accept queue never filled");
1758        }
1759        let socket = mio::net::TcpStream::connect(address).unwrap();
1760        std::thread::sleep(Duration::from_millis(100));
1761        assert_eq!(connect_outcome(&socket), ConnectOutcome::Pending);
1762    }
1763}
1764
1765/// Where the non-blocking `connect(2)` behind a connecting backend socket
1766/// stands when its readiness carries WRITABLE.
1767#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1768enum ConnectOutcome {
1769    /// The connection is established.
1770    Connected,
1771    /// The connect is still in progress: the WRITABLE was not the kernel's
1772    /// completion edge but one sozu raised itself (`Readiness::
1773    /// signal_pending_write` to flush queued output). The completion edge is
1774    /// still to come.
1775    Pending,
1776    /// The connect failed.
1777    Failed,
1778}
1779
1780/// Read the outcome of the non-blocking `connect(2)` behind `socket`.
1781///
1782/// The kernel reports it through the socket's pending error (`SO_ERROR`,
1783/// POSIX `connect(2)`): a WRITABLE edge means the attempt finished, not that
1784/// it succeeded, and reading the error clears it. An empty error proves
1785/// nothing alone, because the WRITABLE may be one sozu signalled itself while
1786/// the dial is still in flight, so `getpeername(2)` then tells an
1787/// established connection from a pending one (`ENOTCONN`). An error reading
1788/// either is a failed connect: nothing past it could use the socket
1789/// (sozu-proxy/sozu#1800).
1790fn connect_outcome(socket: &TcpStream) -> ConnectOutcome {
1791    match socket.take_error() {
1792        Ok(None) => {}
1793        Ok(Some(error)) | Err(error) => {
1794            // The level of the synchronous refusal the pending-links loop
1795            // of `Mux::ready_inner` logs before it retries.
1796            debug!(
1797                "{} backend connect failed, retrying: {}",
1798                log_module_context!(),
1799                error
1800            );
1801            return ConnectOutcome::Failed;
1802        }
1803    }
1804    // `socket2` rather than std's `peer_addr`, which refuses the AF_UNIX
1805    // address of a `unix_path` backend (`backends::connect_backend`).
1806    match socket2::SockRef::from(socket).peer_addr() {
1807        Ok(_) => ConnectOutcome::Connected,
1808        Err(error) if error.kind() == ErrorKind::NotConnected => ConnectOutcome::Pending,
1809        Err(_) => ConnectOutcome::Failed,
1810    }
1811}
1812
1813pub(super) fn remove_backend_stream(
1814    index: &mut InlineTokenMap<LinkedStreams>,
1815    token: Token,
1816    stream_id: GlobalStreamId,
1817) {
1818    if let Some(ids) = index.get_mut(&token) {
1819        ids.remove(stream_id);
1820    }
1821}
1822
1823pub struct Mux<Front: SocketHandler, L: ListenerHandler + L7ListenerHandler> {
1824    pub configured_frontend_timeout: Duration,
1825    pub frontend_token: Token,
1826    pub frontend: Connection<Front>,
1827    pub router: Router,
1828    pub context: Context<L>,
1829    /// The adapter's wheel handles, one per live token (the frontend plus
1830    /// every backend in `router.backends`).
1831    ///
1832    /// The H1/H2 cores no longer own a `TimeoutContainer`: they publish the
1833    /// instant they want to be called back at through `Connection::poll_timeout`
1834    /// and `Mux::reschedule` reflects that onto `crate::timer`, arming only
1835    /// when what the wheel holds differs from what the core wants. This map is
1836    /// the ONLY thing in the mux that talks to the timer wheel.
1837    ///
1838    /// A handle for a token that has left `router.backends` is dropped by
1839    /// `reschedule`'s `retain`, and `TimeoutContainer::drop` cancels its entry
1840    /// — which is why no backend-removal site has to remember to cancel.
1841    pub timeouts: MuxTimeouts,
1842    /// The registry handles the core's [`BackendSlot`]s stand for.
1843    ///
1844    /// The embedder's half of Question 12's perimeter: the core carries an
1845    /// opaque slot, this resolves it, and `Mux::apply_backend_deltas` is
1846    /// what turns the core's [`BackendDelta`]s into writes on the worker's
1847    /// `Backend` values. Indexed by slot.
1848    ///
1849    /// A dial reuses an existing slot when `Rc::ptr_eq` matches, so the
1850    /// vector is bounded by the number of DISTINCT backends a session dials
1851    /// rather than by its request count. Entries are never removed while the
1852    /// session lives: a delta pushed before a backend connection closed must
1853    /// still resolve when the drain reaches it, and an index that moved would
1854    /// silently mis-charge it.
1855    pub(crate) backend_registry: BackendRegistry,
1856    /// The worker's backend set, which `Mux::dial_backend` lends to
1857    /// `Router::backend_from_request` through a `RegistrySelector`.
1858    ///
1859    /// Handed over when the session is built rather than read through
1860    /// `L7Proxy` on every dial: `HttpProxy` and `HttpsProxy` never reassign
1861    /// their `backends` field, so this is the same map, and a dial no longer
1862    /// borrows the proxy or clones the handle to reach it.
1863    pub(crate) backends: Rc<RefCell<BackendMap>>,
1864    /// Per-session correlation ID generated at construction time. Included in
1865    /// every log line emitted from this module so all events for a single
1866    /// frontend connection can be reassembled (independent of the ephemeral
1867    /// per-stream request id used by access logs).
1868    pub session_ulid: Ulid,
1869}
1870
1871/// Answer the per-`(cluster, source-IP)` limit step the core paused at.
1872///
1873/// The embedder half of [`router::ConnectStep::CheckIpLimit`]. The core
1874/// cannot do this itself: it is a consult of
1875/// `SessionManager::cluster_ip_at_limit` **and**, on the admitting path, a
1876/// call to `SessionManager::track_cluster_ip`, which mutates
1877/// `connections_per_cluster_ip` — worker-global state keyed on
1878/// `(cluster, ip)` across every session, and read again by
1879/// `lib/src/tcp.rs`'s own gate for the TCP proxy.
1880///
1881/// **The track happens here, before the decision resumes, and that ordering
1882/// is the point.** An `Admitted` verdict asserts to the core that the slot
1883/// has been taken. Resuming as `Admitted` without having tracked would let
1884/// the stream through uncounted, and because the counter is worker-global
1885/// the next session from the same IP would find a slot that should already
1886/// be gone. `an_admitted_stream_is_counted_before_the_decision_resumes`
1887/// pins it.
1888///
1889/// A free function rather than a `Mux` method so it can be driven directly:
1890/// the ordering above is the thing worth testing, and testing it through a
1891/// copy of this logic would prove nothing about this logic.
1892fn consult_ip_gate(
1893    sessions: &Rc<RefCell<crate::server::SessionManager>>,
1894    frontend_token: Token,
1895    // The cluster routing resolved, borrowed from the stream's `HttpContext`
1896    // where `Router::plan_connect` stored it (#1583).
1897    cluster_id: &sozu_command::state::ClusterId,
1898    resume: &router::ConnectResume,
1899) -> router::IpGateVerdict {
1900    // BOTH caps are consulted here, through the one combined gate:
1901    // the pre-existing per-(cluster, source-IP) cap and the
1902    // per-(cluster, source-SUBNET) cap that stands beside it. A
1903    // connection is admitted only when both allow it, which is what
1904    // makes "10 per IP AND 100 per /64" expressible. The subnet cap is
1905    // `0` (disabled) by default, in which case this is byte-for-byte
1906    // the previous per-IP consult.
1907    let at_limit = sessions.borrow().cluster_connection_at_limit(
1908        frontend_token,
1909        cluster_id,
1910        &resume.ip(),
1911        resume.max_connections_per_ip(),
1912        resume.max_connections_per_subnet(),
1913    );
1914    if at_limit {
1915        let retry_after = sessions
1916            .borrow()
1917            .effective_retry_after(resume.cluster_retry_after());
1918        return router::IpGateVerdict::AtLimit { retry_after };
1919    }
1920    // Idempotent track — H2 streams to the same `(cluster, ip)` share a
1921    // single slot in the per-token set. The decrement happens wholesale on
1922    // session close, via `untrack_all_cluster_ip`.
1923    sessions.borrow_mut().track_cluster_connection(
1924        frontend_token,
1925        cluster_id.clone(),
1926        resume.ip(),
1927        resume.max_connections_per_subnet(),
1928    );
1929    router::IpGateVerdict::Admitted
1930}
1931
1932impl<Front: SocketHandler, L: ListenerHandler + L7ListenerHandler> Mux<Front, L> {
1933    pub fn front_socket(&self) -> &TcpStream {
1934        self.frontend.socket()
1935    }
1936
1937    /// Take the flag the last `ready` pass set when it yielded at
1938    /// `MAX_LOOP_ITERATIONS` with work left (`Context::loop_budget_yielded`).
1939    /// The session answers `ProxySession::yielded` with it.
1940    pub fn take_loop_budget_yield(&mut self) -> bool {
1941        std::mem::take(&mut self.context.loop_budget_yielded)
1942    }
1943
1944    /// Whether a backend connection carries a HUP or ERROR event that no
1945    /// socket readiness will deliver again: the flag `Mux::timeout_inner`
1946    /// sets on a dial whose connect timed out. The session answers
1947    /// `ProxySession::needs_ready_pass` with this, so `Server::timeout` runs
1948    /// the `ready` pass that counts the failure and fails the request over
1949    /// (sozu-proxy/sozu#1800).
1950    pub fn has_pending_backend_failure(&self) -> bool {
1951        self.router.backends.iter().any(|(_, backend)| {
1952            let event = backend.readiness().event;
1953            event.is_error() || event.is_hup()
1954        })
1955    }
1956
1957    /// Whether the client has closed its side of the frontend connection, the
1958    /// `peer_closed` argument the session's close path hands to
1959    /// `shutdown_write`. See [`Connection::peer_closed`].
1960    pub fn frontend_peer_closed(&self) -> bool {
1961        self.frontend.peer_closed()
1962    }
1963
1964    /// Perform every backend accounting change the core has decided since the
1965    /// last drain, in the order it decided them.
1966    ///
1967    /// The ONLY writer of `Backend::active_requests` and
1968    /// `Backend::active_connections` on the mux path. The drain leaves the
1969    /// queue empty, so running this twice in a row is idempotent — the second
1970    /// call finds nothing and performs nothing.
1971    ///
1972    /// Called from the four `Mux` wrappers that already owe the timer wheel a
1973    /// `reschedule` on every exit, and once more immediately before each
1974    /// `Router::plan_connect`, because that call's load balancer is the only
1975    /// reader of the counters this drains.
1976    pub(crate) fn apply_backend_deltas(&mut self) {
1977        self.backend_registry
1978            .apply_all(&mut self.context.backend_deltas, &self.backends);
1979    }
1980}
1981
1982impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHandler> Mux<Front, L> {
1983    fn sync_upgrade_buffers(&mut self) {
1984        for stream in &mut self.context.streams {
1985            stream
1986                .front
1987                .storage
1988                .buffer
1989                .sync(stream.front.storage.end, stream.front.storage.head);
1990            stream
1991                .back
1992                .storage
1993                .buffer
1994                .sync(stream.back.storage.end, stream.back.storage.head);
1995        }
1996    }
1997
1998    /// Consume the wheel entry that delivered a timeout and say whether it was
1999    /// really due.
2000    ///
2001    /// Three things happen here, in this order, and all three are load-bearing.
2002    ///
2003    /// 1. **Consume.** The wheel has already handed the entry over, so
2004    ///    [`TimeoutContainer::triggered`] records that the handle holds
2005    ///    nothing. This is the consume-then-reschedule step, the direct
2006    ///    counterpart of `UdpManager::handle_timeout` clearing `armed_deadline`
2007    ///    on entry (`lib/src/protocol/udp/manager.rs`). It matters because
2008    ///    [`Mux::sync_timeout`] memoizes on "the handle already holds this
2009    ///    deadline": without the clearing, a delivery that changes nothing is
2010    ///    memoized away and the session is left with NO wheel entry at all — a
2011    ///    lost wakeup.
2012    ///
2013    /// 2. **Re-validate.** `crate::timer` rounds a delay to the NEAREST tick
2014    ///    (`timer.rs` `duration_to_tick`) and `Timer::poll` fires everything
2015    ///    whose tick has come, so an entry armed for deadline `D` is delivered
2016    ///    from `tick * round(D / tick) - tick/2` onwards. The earliness is
2017    ///    `(D_ms + 50) mod 100` at the default 100 ms tick, i.e. up to **99 ms**
2018    ///    — a full tick minus a millisecond, not half a tick; see
2019    ///    `timer.rs::duration_to_tick` for the two halves of that bound. An
2020    ///    early delivery is not an expiry: returning
2021    ///    `false` here leaves the core's deadline untouched, and the
2022    ///    `reschedule` on the way out of [`SessionState::timeout`] therefore
2023    ///    puts the entry straight back at the SAME instant.
2024    ///
2025    /// 3. **Clear the core's deadline on a real expiry.** The core asked to be
2026    ///    called back at `D` and has been; leaving `D` in place would have
2027    ///    `reschedule` re-arm an already-elapsed instant, which the wheel
2028    ///    re-delivers immediately — a busy loop. The timeout branches re-arm
2029    ///    with `arm_timeout` on every path that keeps the session alive, which
2030    ///    is what the old `triggered()` + `set(token)` pair did.
2031    ///
2032    /// A token with no core (an unknown token, or a backend already evicted)
2033    /// is reported due and handled by the caller's unknown-token branch.
2034    fn consume_timer_entry(&mut self, token: Token) -> bool {
2035        if token == self.frontend_token {
2036            self.timeouts.frontend.triggered();
2037        } else if let Some(container) = self.timeouts.backends.get_mut(&token) {
2038            container.triggered();
2039        }
2040        let core_deadline = if token == self.frontend_token {
2041            self.frontend.poll_timeout()
2042        } else {
2043            match self.router.backends.get(&token) {
2044                Some(backend) => backend.poll_timeout(),
2045                None => return true,
2046            }
2047        };
2048        if core_deadline.is_some_and(|deadline| self.context.now < deadline) {
2049            return false;
2050        }
2051        if token == self.frontend_token {
2052            self.frontend.clear_timeout();
2053        } else if let Some(backend) = self.router.backends.get_mut(&token) {
2054            backend.clear_timeout();
2055        }
2056        true
2057    }
2058
2059    /// Reflect every core's next deadline onto the timer wheel.
2060    ///
2061    /// This is the adapter half of the `poll_timeout()` split, and the direct
2062    /// analogue of `UdpManager::reschedule` (`lib/src/protocol/udp/manager.rs`).
2063    /// It runs at the end of every `SessionState` entry point that can change a
2064    /// deadline — `ready`, `timeout`, `shutting_down`, `cancel_timeouts` —
2065    /// through wrappers, so no early `return` inside those bodies can skip it.
2066    /// That structural placement is the point: a hand-placed re-arm at each
2067    /// exit is exactly what used to be forgotten.
2068    ///
2069    /// Departed tokens need no explicit cancel: `retain` drops their handle and
2070    /// `TimeoutContainer::drop` cancels the entry.
2071    fn reschedule(&mut self) {
2072        Self::sync_timeout(
2073            &mut self.timeouts.frontend,
2074            self.frontend_token,
2075            self.frontend.poll_timeout(),
2076            self.frontend.timeout_duration(),
2077        );
2078        for (token, backend) in &self.router.backends {
2079            let duration = backend.timeout_duration();
2080            Self::sync_timeout(
2081                self.timeouts
2082                    .backends
2083                    .get_or_insert_with(*token, || TimeoutContainer::new_empty(duration)),
2084                *token,
2085                backend.poll_timeout(),
2086                duration,
2087            );
2088        }
2089        let backends = &self.router.backends;
2090        self.timeouts
2091            .backends
2092            .retain(|token, _| backends.contains_key(token));
2093
2094        self.debug_assert_timer_coherence();
2095    }
2096
2097    /// Arm, re-arm or cancel one token's wheel entry so it matches `next`.
2098    ///
2099    /// The memoization — skip the wheel when nothing changed — is what makes
2100    /// this cheap enough to run on every pass, and it is also where the lost
2101    /// wakeup lives if it is written carelessly. It is guarded on BOTH halves:
2102    ///
2103    /// * `deadline() == next` — the instant has not moved; and
2104    /// * `is_armed() == next.is_some()` — the wheel actually still HOLDS it.
2105    ///
2106    /// The second is not redundant. A delivered entry is gone from the wheel,
2107    /// and `TimeoutContainer::triggered` clears the deadline alongside it, so
2108    /// the pair stays honest — that clearing is this design's
2109    /// consume-then-reschedule step, the same one `UdpManager::handle_timeout`
2110    /// performs by setting `armed_deadline = None` on entry. Memoize on the
2111    /// deadline alone, against a mirror that a firing does not clear, and every
2112    /// wheel delivery that changes nothing becomes a session with no timer.
2113    fn sync_timeout(
2114        container: &mut TimeoutContainer,
2115        token: Token,
2116        next: Option<Instant>,
2117        duration: Duration,
2118    ) {
2119        if container.duration() != duration {
2120            // Keep the handle's configured duration current without moving the
2121            // armed entry: the WebSocket upgrade hands these containers to
2122            // `Pipe`, which re-arms from `duration()`.
2123            container.retune(duration);
2124        }
2125        if container.deadline() == next && container.is_armed() == next.is_some() {
2126            return;
2127        }
2128        match next {
2129            Some(deadline) => container.set_at(token, deadline),
2130            None => {
2131                container.cancel();
2132            }
2133        }
2134    }
2135
2136    /// Timer coherence, carried over from `UdpManager::check_invariants` (6):
2137    /// for every live token the wheel handle must hold exactly what the core
2138    /// asked for — armed iff the core wants a timer, at the core's instant when
2139    /// it does — and no handle may outlive its connection.
2140    ///
2141    /// This is the invariant a missed `reschedule` breaks, and it is the reason
2142    /// the entry points are wrapped rather than trusted.
2143    fn debug_assert_timer_coherence(&self) {
2144        #[cfg(debug_assertions)]
2145        {
2146            let frontend_token = self.frontend_token;
2147            let frontend = std::iter::once((&frontend_token, &self.timeouts.frontend));
2148            for (token, container) in frontend.chain(self.timeouts.backends.iter()) {
2149                let core = if *token == frontend_token {
2150                    self.frontend.poll_timeout()
2151                } else {
2152                    match self.router.backends.get(token) {
2153                        Some(backend) => backend.poll_timeout(),
2154                        None => {
2155                            panic!("timer handle for {token:?} outlived its connection")
2156                        }
2157                    }
2158                };
2159                debug_assert_eq!(
2160                    container.is_armed(),
2161                    core.is_some(),
2162                    "timer coherence for {token:?}: armed={} but the core wants {core:?}",
2163                    container.is_armed(),
2164                );
2165                debug_assert_eq!(
2166                    container.deadline(),
2167                    core,
2168                    "timer coherence for {token:?}: the wheel holds {:?}, the core wants {core:?}",
2169                    container.deadline(),
2170                );
2171            }
2172        }
2173    }
2174
2175    /// Whether a frontend HUP still leaves an exchange to serve.
2176    ///
2177    /// The frontend HUP is not always a hang-up: `Ready::from(&Event)`
2178    /// (`command/src/ready.rs`) also raises it for `is_read_closed()`, the
2179    /// `EPOLLRDHUP` a client's half-close (`shutdown(SHUT_WR)`, RFC 9293
2180    /// §3.6) produces. Such a client has stopped sending, not receiving, and
2181    /// still expects the response to what it sent:
2182    ///
2183    /// - input not read yet may hold a request, or the rest of one: the FIN
2184    ///   can arrive with the request itself (`Connection::has_unread_input`);
2185    /// - a stream whose request was received whole is answered.
2186    ///
2187    /// Nothing else is: once the client's EOF is read, a request it left
2188    /// incomplete can never complete, and the session closes as on a hang-up
2189    /// instead of waiting for its backend.
2190    ///
2191    /// A full hang-up is never in flight: the client will read nothing more.
2192    /// It is ERROR (`EPOLLERR`, a reset) or WRITE_CLOSED (mio's
2193    /// `is_write_closed`: `EPOLLHUP` or `EPOLLERR`). ERROR alone is not
2194    /// enough: a reset whose error sozu's own `read` or `write` consumed
2195    /// first is reported as `EPOLLHUP` without `EPOLLERR`, which only
2196    /// WRITE_CLOSED tells from a half-close. A lingering frontend is the one
2197    /// exception: sozu shut its write side itself, so `EPOLLHUP` follows the
2198    /// client's FIN, and its drain still reads to that EOF.
2199    fn frontend_exchange_in_flight(&self) -> bool {
2200        let event = self.frontend.readiness().event;
2201        if event.is_error() || (event.is_write_closed() && !self.frontend.is_lingering()) {
2202            return false;
2203        }
2204        self.frontend.has_unread_input()
2205            || self
2206                .context
2207                .streams
2208                .iter()
2209                .any(|stream| stream.state.is_open() && stream.front.is_terminated())
2210    }
2211
2212    fn delay_close_for_frontend_flush(&mut self, reason: &'static str) -> bool {
2213        let _ = self.frontend.initiate_close_notify();
2214        // LIFECYCLE §9 invariant 16: consult per-stream back-buffers in
2215        // addition to the connection-level pending-write predicate so
2216        // shutdown does not close while any open H2 stream still has
2217        // kawa bytes queued after a voluntary scheduler yield.
2218        if self
2219            .frontend
2220            .has_pending_write_including_streams(&self.context)
2221        {
2222            let readiness = self.frontend.readiness_mut();
2223            readiness.interest = Ready::WRITABLE | Ready::HUP | Ready::ERROR;
2224            readiness.signal_pending_write();
2225            debug!(
2226                "{} Mux delaying close on {}: {:?}",
2227                log_context!(self),
2228                reason,
2229                self.frontend
2230            );
2231            true
2232        } else {
2233            false
2234        }
2235    }
2236
2237    /// The inner loop of `Mux::ready_inner` reached `MAX_LOOP_ITERATIONS`.
2238    /// Count and log it, then tell the loop whether to wait for the client
2239    /// (`true`) or close the session (`false`).
2240    ///
2241    /// Output still queued for the client, in the connection or in an open
2242    /// stream's response buffer, is not dropped while the client drains it.
2243    /// An H1 frontend's `has_pending_write_including_streams` does not see
2244    /// its stream's buffer, so the open streams are probed here
2245    /// (sozu-proxy/sozu#1819). The session then waits for the frontend's next
2246    /// writable event, with the frontend timeout as the bound if none comes.
2247    ///
2248    /// Waiting stays bounded: reaching the limit again with no byte written
2249    /// to the client since the last time (`Context::client_bytes_out`)
2250    /// closes the session, so a loop that spins on every wake-up without
2251    /// progress costs at most two budgets.
2252    fn wait_for_client_at_loop_limit(&mut self) -> bool {
2253        incr!(names::http::INFINITE_LOOP_ERROR);
2254        #[cfg(test)]
2255        LOOP_LIMIT_HITS.with(|hits| hits.set(hits.get() + 1));
2256        let bytes_out = self.context.client_bytes_out;
2257        let progressed = self.context.loop_limit_bytes_out != Some(bytes_out);
2258        let output_queued = progressed
2259            && (self
2260                .frontend
2261                .has_pending_write_including_streams(&self.context)
2262                || self.context.streams.iter().any(|stream| {
2263                    stream.state.is_open()
2264                        && (!stream.back.out.is_empty() || !stream.back.blocks.is_empty())
2265                }));
2266        warn!(
2267            "{} Mux loop reached {} iterations, {}: frontend={:?} backends={:?}",
2268            log_context!(self),
2269            MAX_LOOP_ITERATIONS,
2270            if output_queued {
2271                "waiting for the client to drain the queued output"
2272            } else if progressed {
2273                "closing"
2274            } else {
2275                "closing: no byte reached the client since the last time"
2276            },
2277            self.frontend.readiness(),
2278            self.router
2279                .backends
2280                .iter()
2281                .map(|(token, backend)| (*token, backend.readiness().clone()))
2282                .collect::<Vec<_>>()
2283        );
2284        if output_queued {
2285            self.context.loop_limit_bytes_out = Some(bytes_out);
2286            let readiness = self.frontend.readiness_mut();
2287            readiness.interest.insert(Ready::WRITABLE);
2288            readiness.event.remove(Ready::WRITABLE);
2289            self.frontend.arm_timeout(self.context.now);
2290        }
2291        output_queued
2292    }
2293
2294    /// Drive the frontend I/O path during shutdown, when the server is polling
2295    /// `shutting_down()` outside the normal epoll readiness loop.
2296    ///
2297    /// This is required for H2 graceful shutdown because a stream may still
2298    /// need one last readable pass to observe the peer's END_STREAM or one last
2299    /// writable pass to retire the stream, emit GOAWAY, or flush TLS records.
2300    ///
2301    /// An H2 frontend always gets that one writable pass. Further passes run
2302    /// only while the connection still has output queued AND the socket took
2303    /// the last write: the loop stops as soon as either is false. It must not
2304    /// wait for the WRITABLE *event* to clear, because the forced pass sets
2305    /// that bit itself (`signal_pending_write`) and `writable()` clears it only
2306    /// on `WouldBlock`. An idle draining session therefore never cleared it,
2307    /// and every `shut_down_sessions()` tick spun `MAX_LOOP_ITERATIONS` empty
2308    /// `writable()` calls per draining H2 session.
2309    fn drive_frontend_shutdown_io(&mut self) -> SessionIsToBeClosed {
2310        let force_h2_read = matches!(self.frontend, Connection::H2(_));
2311        let force_h2_write = matches!(self.frontend, Connection::H2(_));
2312        let readiness = self.frontend.readiness().clone();
2313        if !force_h2_read
2314            && !force_h2_write
2315            && readiness.event.is_empty()
2316            && !self.frontend.has_pending_write()
2317        {
2318            return false;
2319        }
2320
2321        if force_h2_read || self.frontend.readiness().event.is_readable() {
2322            self.frontend
2323                .readiness_mut()
2324                .interest
2325                .insert(Ready::READABLE);
2326            match self
2327                .frontend
2328                .readable(&mut self.context, EndpointClient(&mut self.router))
2329            {
2330                MuxResult::Continue => {}
2331                MuxResult::CloseSession | MuxResult::Upgrade => return true,
2332            }
2333        }
2334
2335        if !force_h2_write
2336            && !self.frontend.has_pending_write()
2337            && !self.frontend.readiness().event.is_writable()
2338        {
2339            return false;
2340        }
2341
2342        let mut iterations = 0;
2343        loop {
2344            self.frontend
2345                .readiness_mut()
2346                .interest
2347                .insert(Ready::WRITABLE);
2348            if force_h2_write {
2349                self.frontend.readiness_mut().signal_pending_write();
2350            }
2351            #[cfg(test)]
2352            SHUTDOWN_WRITABLE_PASSES.with(|passes| passes.set(passes.get() + 1));
2353            match self
2354                .frontend
2355                .writable(&mut self.context, EndpointClient(&mut self.router))
2356            {
2357                MuxResult::Continue => {}
2358                MuxResult::CloseSession | MuxResult::Upgrade => return true,
2359            }
2360
2361            iterations += 1;
2362            if iterations >= MAX_LOOP_ITERATIONS
2363                || !self.frontend.has_pending_write()
2364                || !self.frontend.readiness().event.is_writable()
2365            {
2366                break;
2367            }
2368        }
2369        false
2370    }
2371}
2372
2373impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHandler> SessionState
2374    for Mux<Front, L>
2375{
2376    /// Thin wrapper over `Mux::ready_inner` whose only job is to run
2377    /// `Mux::reschedule` on the way out. `ready_inner` has a dozen `return`
2378    /// sites; arming the wheel at each of them by hand is precisely the
2379    /// discipline this refactor exists to remove.
2380    fn ready(
2381        &mut self,
2382        session: Rc<RefCell<dyn ProxySession>>,
2383        proxy: Rc<RefCell<dyn L7Proxy>>,
2384        metrics: &mut SessionMetrics,
2385    ) -> SessionResult {
2386        let result = self.ready_inner(session, proxy, metrics);
2387        self.apply_backend_deltas();
2388        self.reschedule();
2389        result
2390    }
2391
2392    fn update_readiness(&mut self, token: Token, events: Ready) {
2393        trace!("{} EVENTS: {:?} on {:?}", log_context!(self), events, token);
2394        self.context.debug.push(DebugEvent::EV(token, events));
2395        if token == self.frontend_token {
2396            self.frontend.readiness_mut().event |= events;
2397        } else if let Some(c) = self.router.backends.get_mut(&token) {
2398            c.readiness_mut().event |= events;
2399        }
2400    }
2401
2402    /// Thin wrapper over `Mux::timeout_inner`: reschedule on the way out,
2403    /// then check the two properties carried over from `UdpManager`.
2404    fn timeout(&mut self, token: Token, metrics: &mut SessionMetrics) -> StateResult {
2405        let result = self.timeout_inner(token, metrics);
2406        self.apply_backend_deltas();
2407        self.reschedule();
2408
2409        // Strict-advance guard, carried over from `UdpManager::handle_timeout`
2410        // (`lib/src/protocol/udp/manager.rs`). After handling a firing at
2411        // `now`, the entry re-armed for the SAME token must be strictly in the
2412        // future. A deadline `<= now` makes the wheel re-deliver immediately
2413        // and the session spins — the canonical sans-io busy loop. The one
2414        // legitimate way to reach it is a configured timeout of zero, which is
2415        // an operator asking for exactly that.
2416        #[cfg(debug_assertions)]
2417        {
2418            let now = self.context.now;
2419            let (duration, container) = if token == self.frontend_token {
2420                (
2421                    Some(self.frontend.timeout_duration()),
2422                    Some(&self.timeouts.frontend),
2423                )
2424            } else {
2425                (
2426                    self.router
2427                        .backends
2428                        .get(&token)
2429                        .map(|b| b.timeout_duration()),
2430                    self.timeouts.backends.get(&token),
2431                )
2432            };
2433            if let Some(container) = container
2434                && let Some(next) = container.deadline()
2435                && duration.is_some_and(|d| !d.is_zero())
2436            {
2437                debug_assert!(
2438                    next > now,
2439                    "Mux::timeout must strictly advance past a firing on {token:?}: \
2440                     armed {next:?} <= fired_at {now:?} (busy-loop)"
2441                );
2442            }
2443        }
2444
2445        result
2446    }
2447
2448    fn cancel_timeouts(&mut self) {
2449        trace!("{} MuxState::cancel_timeouts", log_context!(self));
2450        self.frontend.clear_timeout();
2451        for backend in self.router.backends.values_mut() {
2452            backend.clear_timeout();
2453        }
2454        // Push the cleared deadlines onto the wheel in the same call: the
2455        // adapter's handles are the only thing holding real entries.
2456        self.reschedule();
2457    }
2458
2459    fn print_state(&self, _context: &str) {
2460        // The trait-required `context: &str` parameter (protocol tag like
2461        // "HTTPS"/"HTTP" passed by callers) predates the unified
2462        // `log_context!(self)` envelope. The canonical `MUX` tag lives
2463        // inside `log_context!`, so we ignore the parameter here and emit
2464        // the bracketed Session(...) block instead, mirroring the second
2465        // `error!` in this function.
2466        error!(
2467            "\
2468{} Session(Mux)
2469\tFrontend:
2470\t\ttoken: {:?}\treadiness: {:?}
2471\tBackend(s):",
2472            log_context!(self),
2473            self.frontend_token,
2474            self.frontend.readiness()
2475        );
2476        for (backend_token, backend) in &self.router.backends {
2477            error!(
2478                "{} \t\ttoken: {:?}\treadiness: {:?}",
2479                log_context!(self),
2480                backend_token,
2481                backend.readiness()
2482            )
2483        }
2484    }
2485
2486    fn close(&mut self, proxy: Rc<RefCell<dyn L7Proxy>>, _metrics: &mut SessionMetrics) {
2487        if self.context.debug.is_interesting() {
2488            warn!("{} {:?}", log_context!(self), self.context.debug.events);
2489        }
2490        debug!("{} MUX CLOSE", log_context!(self));
2491        trace!("{} FRONTEND: {:#?}", log_context!(self), self.frontend);
2492        trace!(
2493            "{} BACKENDS: {:#?}",
2494            log_context!(self),
2495            self.router.backends
2496        );
2497
2498        self.log_and_recycle_open_streams();
2499
2500        self.frontend
2501            .close(&mut self.context, EndpointClient(&mut self.router));
2502
2503        // Release every wheel handle: the session is going away and
2504        // `TimeoutContainer::drop` cancels each entry. This replaces the
2505        // per-backend `timeout_container().cancel()` the loop below used to do
2506        // and additionally covers the frontend, which it never did. The
2507        // frontend's handle is a field rather than a map entry, so it is
2508        // replaced by an unarmed one, which is what a later `reschedule`
2509        // would have inserted in its place.
2510        self.timeouts.frontend = TimeoutContainer::new_empty(self.timeouts.frontend.duration());
2511        self.timeouts.backends.clear();
2512
2513        self.close_backend_connections(&proxy);
2514
2515        // The session is going away, so this is the last chance to settle the
2516        // ledger against the worker's registry. Everything the frontend close
2517        // and the loop above decided is performed here.
2518        self.apply_backend_deltas();
2519    }
2520
2521    /// Thin wrapper over `Mux::shutting_down_inner`: it drives frontend I/O
2522    /// and can change a core deadline, so it owes the wheel a reschedule on
2523    /// every exit.
2524    fn shutting_down(&mut self) -> SessionIsToBeClosed {
2525        let result = self.shutting_down_inner();
2526        self.apply_backend_deltas();
2527        self.reschedule();
2528        result
2529    }
2530
2531    /// A mux session is targeted when one of its backend connections,
2532    /// connecting or connected, is: an H2 frontend multiplexing several
2533    /// clusters is targeted by each of them.
2534    fn targets(&self, target: &crate::SessionTarget) -> bool {
2535        self.router
2536            .backends
2537            .iter()
2538            .any(|(_, connection)| match connection.position() {
2539                Position::Client(cluster_id, backend, _) => {
2540                    target.matches(cluster_id, Some(&backend.backend_id), Some(backend.address))
2541                }
2542                Position::Server => false,
2543            })
2544    }
2545}
2546
2547/// The bodies the `SessionState` wrappers above drive. They are inherent
2548/// methods rather than trait methods so the wrappers can own the one thing
2549/// every exit path needs: the [`Mux::reschedule`] that pushes the cores'
2550/// deadlines onto the timer wheel.
2551impl<Front: SocketHandler + std::fmt::Debug, L: ListenerHandler + L7ListenerHandler> Mux<Front, L> {
2552    /// Emit the access log of every stream still open with a request start,
2553    /// with the connection overhead shared among them, and recycle it.
2554    /// Called by `Mux::close` at teardown and by
2555    /// [`Self::release_streams_and_backends_for_error_linger`].
2556    fn log_and_recycle_open_streams(&mut self) {
2557        // Log active streams at session teardown for timeout diagnosis
2558        let active_count = self
2559            .context
2560            .streams
2561            .iter()
2562            .filter(|s| s.state.is_open() && s.metrics.start.is_some())
2563            .count();
2564        if active_count > 0 {
2565            debug!(
2566                "{} Session close with {} active stream(s)",
2567                log_context!(self),
2568                active_count
2569            );
2570            for (idx, stream) in self
2571                .context
2572                .streams
2573                .iter()
2574                .enumerate()
2575                .filter(|(_, s)| s.state.is_open() && s.metrics.start.is_some())
2576            {
2577                let elapsed = stream.metrics.service_time();
2578                debug!(
2579                    "{}   active stream[{}]: state={:?} service_time={:?} method={:?} path={:?} status={:?}",
2580                    log_context!(self),
2581                    idx,
2582                    stream.state,
2583                    elapsed,
2584                    stream.context.method,
2585                    stream.context.path,
2586                    stream.context.status,
2587                );
2588            }
2589            incr!(names::h2::CLOSE_WITH_ACTIVE_STREAMS);
2590        }
2591
2592        // Distribute H2 connection-level overhead (control frames) across in-flight
2593        // streams so that access log bytes_in/bytes_out reflect actual wire cost.
2594        // Integer division may lose up to (active_count - 1) bytes, which is acceptable.
2595        let active_count = active_count.max(1);
2596        let (total_overhead_in, total_overhead_out) = self.frontend.overhead_bytes();
2597        let share_in = total_overhead_in / active_count;
2598        let share_out = total_overhead_out / active_count;
2599
2600        // Generate access logs for in-flight streams on session teardown.
2601        // Skip streams that already had their access log emitted (metrics.start is
2602        // set to None by metrics.reset() after generate_access_log in the happy path).
2603        // Each connection's RTT is its one memoized sample (`memoized_rtt`):
2604        // asked for only for a stream that still owes a log, so the ordinary
2605        // close, where every stream has already been logged, reads nothing
2606        // (issue #1590), and a connection that logged before reads nothing
2607        // either.
2608        for stream in &mut self.context.streams {
2609            if stream.state.is_open() && stream.metrics.start.is_some() {
2610                let client_rtt = self.frontend.rtt();
2611                stream.metrics.bin += share_in;
2612                stream.metrics.bout += share_out;
2613                stream.metrics.service_stop();
2614                if stream.metrics.backend_stop.is_none() {
2615                    stream.metrics.backend_stop();
2616                }
2617                // Only mark as error if the stream had an actual protocol/processing failure
2618                // (kawa parse error, backend error). Normal timeouts, client disconnects,
2619                // and graceful connection closures are not errors.
2620                let is_error = stream.front.is_error() || stream.back.is_error();
2621                let server_rtt = stream
2622                    .linked_token()
2623                    .and_then(|token| self.router.backends.get(&token).and_then(Connection::rtt));
2624                for event in stream
2625                    .generate_access_log(
2626                        is_error,
2627                        Some("session close"),
2628                        self.context.listener.clone(),
2629                        client_rtt,
2630                        server_rtt,
2631                    )
2632                    .into_iter()
2633                    .flatten()
2634                {
2635                    crate::protocol::mux::h2::record_metric(event);
2636                }
2637                stream.state = StreamState::Recycle;
2638            }
2639        }
2640    }
2641
2642    /// Shut down, deregister from the session slab and account for every
2643    /// backend connection (`StreamsEnded`, `ConnectionClosed`, the backend
2644    /// gauges), then clear the backend-stream reverse index. The connections
2645    /// stay in `router.backends`: at teardown they drop with the `Mux`.
2646    fn close_backend_connections(&mut self, proxy: &Rc<RefCell<dyn L7Proxy>>) {
2647        for (token, client) in &mut self.router.backends {
2648            let proxy_borrow = proxy.borrow();
2649            let peer_closed = client.peer_closed();
2650            let socket = client.socket_mut();
2651            // No `EPOLL_CTL_DEL`: the socket stays in `router.backends` until
2652            // this `Mux` drops with its session, before the event loop's next
2653            // `epoll_wait`, and its last close removes it from the epoll set —
2654            // the same invariant as the front socket in `HttpsSession::close`
2655            // (`lib/src/https.rs`).
2656            // invariant: write-only shutdown — Shutdown::Both on a TLS frontend
2657            // discards the receive buffer and elicits TCP RST, truncating the
2658            // already-queued response. Canonical write-up: `HttpsSession::close`
2659            // (`lib/src/https.rs`). Backend sockets follow the same discipline for symmetry.
2660            // Skipped once the backend has closed: see `shutdown_write`.
2661            if let Err(e) = shutdown_write(socket, peer_closed)
2662                && e.kind() != ErrorKind::NotConnected
2663            {
2664                error!(
2665                    "{} error shutting down back socket({:?}): {:?}",
2666                    log_context_lite!(self),
2667                    socket,
2668                    e
2669                );
2670            }
2671            if !proxy_borrow.remove_session(*token) {
2672                error!(
2673                    "{} session {:?} was already removed!",
2674                    log_context_lite!(self),
2675                    token
2676                );
2677            }
2678
2679            match client.position() {
2680                Position::Client(cluster_id, backend, _) => {
2681                    // Session teardown iterates the backends map directly
2682                    // instead of routing through `Connection::close`, so it
2683                    // emits the same two accounting changes that path emits.
2684                    // Going through the ledger rather than mutating here is
2685                    // what lets the invariant-14 balance be read off one
2686                    // stream of deltas — see `Mux::apply_backend_deltas`.
2687                    let count = self
2688                        .context
2689                        .backend_streams
2690                        .get(token)
2691                        .map_or(0, |ids| ids.len());
2692                    let (slot, backend_id, metric_incarnation) = (
2693                        backend.slot(),
2694                        backend.backend_id.clone(),
2695                        backend.metric_incarnation(),
2696                    );
2697                    self.context.backend_deltas.push(BackendDelta {
2698                        slot,
2699                        change: BackendChange::StreamsEnded(count),
2700                    });
2701                    self.context.backend_deltas.push(BackendDelta {
2702                        slot,
2703                        change: BackendChange::ConnectionClosed,
2704                    });
2705                    gauge_add!(names::backend::CONNECTIONS, -1);
2706                    // Second `-1` site for `backend.pool.size` (the first is
2707                    // in `connection.rs::pre_close_client_bookkeeping`). This
2708                    // path runs during session teardown when the frontend
2709                    // session iterates the backends map directly without
2710                    // routing through `Connection::close`. Both `-1` sites
2711                    // mirror the single `+1` in router.rs::connect and the
2712                    // matching `backend.connections, -1` calls already
2713                    // present here, so symmetry follows from
2714                    // `backend.connections` correctness.
2715                    gauge_add!(names::backend::POOL_SIZE, -1);
2716                    gauge_add!(
2717                        names::backend::CONNECTIONS_PER_BACKEND,
2718                        -1,
2719                        Some(cluster_id),
2720                        Some(&backend_id),
2721                        metric_incarnation
2722                    );
2723                    trace!(
2724                        "{} connection (session) closed: {:?}",
2725                        log_context_lite!(self),
2726                        backend
2727                    );
2728                }
2729                Position::Server => {
2730                    error!(
2731                        "{} close_backend called on Server position",
2732                        log_context_lite!(self)
2733                    );
2734                }
2735            }
2736        }
2737        // Clear the reverse index after all backends have charged their
2738        // `StreamsEnded` deltas (whose counts come from that index).
2739        self.context.backend_streams.clear();
2740    }
2741
2742    /// Release the streams and backends of an H2 frontend that lingers after
2743    /// a final GOAWAY carrying an error code, in the pass that decides the
2744    /// linger rather than at its end, the order
2745    /// `ngx_http_v2_finalize_connection` follows. A linger still pending on
2746    /// its TLS `close_notify` counts as decided
2747    /// (`Connection::lingers_after_error_goaway`): the pass that queues the
2748    /// alert can end with the kernel refusing it and nothing re-armed, and
2749    /// then reach the `pending_links` pass with its streams still open.
2750    ///
2751    /// Such a GOAWAY can leave streams open (a flood, a protocol or flow
2752    /// control error, a SETTINGS timeout), unlike a final GOAWAY(NO_ERROR),
2753    /// which waits for the stream table to empty. None of them will be
2754    /// answered, so their access logs are emitted now with the same outcome
2755    /// the immediate close gave them before (`session close`), the slots
2756    /// recycled, and the backend connections shut down, accounted for and
2757    /// dropped, closing their sockets. With every stream recycled, nothing
2758    /// can be re-linked: the `pending_links` pass of `Mux::ready_inner`
2759    /// skips any stream not in `StreamState::Link`, and no backend is left
2760    /// to report a failure that would queue one. The recycled slots keep
2761    /// their buffers until the `Mux` drops, at the latest when the linger's
2762    /// short deadline (`shared::ERROR_GOAWAY_LINGER_TIMEOUT`) closes it.
2763    ///
2764    /// Idempotent: a second call finds nothing open and no backend. A
2765    /// graceful linger is left alone: its stream table is already empty.
2766    fn release_streams_and_backends_for_error_linger(&mut self, proxy: &Rc<RefCell<dyn L7Proxy>>) {
2767        if !self.frontend.lingers_after_error_goaway() {
2768            return;
2769        }
2770        // Pre: the `metrics.start` filter below cannot skip a stream that
2771        // could reach a backend. Every entry into `StreamState::Link` arms
2772        // the start first (`SessionMetrics::service_start`) or re-links a
2773        // stream that was already linked, and no `SessionMetrics::reset`
2774        // leaves a stream in either state.
2775        debug_assert!(
2776            !self.context.streams.iter().any(|s| {
2777                matches!(s.state, StreamState::Link | StreamState::Linked(_))
2778                    && s.metrics.start.is_none()
2779            }),
2780            "a stream queued for or linked to a backend carries its request start"
2781        );
2782        let has_open_stream = self
2783            .context
2784            .streams
2785            .iter()
2786            .any(|s| s.state.is_open() && s.metrics.start.is_some());
2787        if !has_open_stream && self.router.backends.is_empty() {
2788            return;
2789        }
2790        debug!(
2791            "{} H2 error GOAWAY linger: releasing {} backend connection(s) and the open streams",
2792            log_context!(self),
2793            self.router.backends.len()
2794        );
2795        self.log_and_recycle_open_streams();
2796        self.close_backend_connections(proxy);
2797        let tokens: Vec<Token> = (&self.router.backends)
2798            .into_iter()
2799            .map(|(token, _)| *token)
2800            .collect();
2801        for token in tokens {
2802            // Dropping the connection closes its socket, which leaves the
2803            // epoll set with its last close (see `Mux::close`).
2804            drop(self.router.backends.remove(&token));
2805        }
2806        self.timeouts.backends.clear();
2807        debug_assert!(
2808            self.router.backends.is_empty() && self.context.backend_streams.is_empty(),
2809            "an error-GOAWAY linger holds no backend connection"
2810        );
2811        debug_assert!(
2812            !self
2813                .context
2814                .streams
2815                .iter()
2816                .any(|s| matches!(s.state, StreamState::Link | StreamState::Linked(_))),
2817            "an error-GOAWAY linger keeps no stream that could reach a backend"
2818        );
2819    }
2820
2821    /// Fulfil a [`router::ConnectPlan::Dial`]: select a backend, dial it,
2822    /// build the connection, register it, and hand it to
2823    /// `Router::commit_dialed`.
2824    ///
2825    /// This is the embedder half of Question 6's split
2826    /// ([#1340](https://github.com/sozu-proxy/sozu/issues/1340)). Every step
2827    /// here reaches something the core has no business holding — the
2828    /// `BackendMap`, a `connect(2)`, `setsockopt`, the session slab, the mio
2829    /// registry — and the order is the one `Router::plan_connect` used, unchanged.
2830    ///
2831    /// The one side-effect taken before the dial is the reservation selection
2832    /// raises on the backend (`active_connections += 1`,
2833    /// `BackendMap::reserve_backend`), and this function releases it on
2834    /// every exit that returns `Err`: a failed `connect(2)` through
2835    /// `BackendChange::DialFailed`, which also records the backend's
2836    /// failure, and every later exit — all of [`Self::attach_dialed`]'s —
2837    /// through one `BackendChange::ConnectionClosed`, which records none,
2838    /// because there the backend did not fail (#1713).
2839    ///
2840    /// Taken as free functions over `&mut` fields rather than `&mut self` so
2841    /// the caller can keep its `&mut self.context` binding across the call.
2842    #[allow(clippy::too_many_arguments)]
2843    fn dial_backend(
2844        router: &mut Router,
2845        backend_registry: &mut BackendRegistry,
2846        backends: &RefCell<BackendMap>,
2847        stream_id: GlobalStreamId,
2848        context: &mut Context<L>,
2849        session: &Rc<RefCell<dyn ProxySession>>,
2850        proxy: &Rc<RefCell<dyn L7Proxy>>,
2851        // Owned: moved into the new connection's `Position::Client`, a handle
2852        // on the stream's routed id rather than a copy of it (#1583).
2853        cluster_id: sozu_command::state::ClusterId,
2854        h2: bool,
2855        frontend_should_stick: bool,
2856    ) -> Result<(), BackendConnectionError> {
2857        // One `borrow_mut` of the worker's `BackendMap` for exactly the
2858        // selection and its reservation, and no longer: nothing else on this
2859        // path borrows the map, and the `RefMut` drops before the dial and
2860        // before `add_session` and `register_socket` borrow the proxy below.
2861        let mut backend = {
2862            let mut backends = backends.borrow_mut();
2863            let stream = &mut context.streams[stream_id];
2864            let mut selector = RegistrySelector {
2865                backends: &mut backends,
2866                registry: backend_registry,
2867                now: context.now,
2868                exclude: &stream.tried_backends,
2869            };
2870            router.backend_from_request(
2871                &cluster_id,
2872                frontend_should_stick,
2873                &mut stream.context,
2874                &mut selector,
2875            )?
2876        };
2877        backend.set_metric_incarnation(
2878            context.streams[stream_id]
2879                .context
2880                .cluster_metrics_incarnation(),
2881        );
2882
2883        // Selection reserved a connection on `backend`; the dial is ours
2884        // (#1684). A `connect(2)` that fails releases that reservation and
2885        // records the failure through the ledger, which the drain before the
2886        // next selection applies — the backend is then exactly where a failed
2887        // `Backend::try_connect` used to leave it.
2888        // A backend with a `unix_path` is dialed over that UNIX socket, and
2889        // TCP_NODELAY means nothing there. An id without a registry handle
2890        // (an embedder's own selector) names a TCP backend.
2891        let (dialed, nodelay) = match backend_registry.handle(&backend) {
2892            Some(handle) => {
2893                let handle = handle.borrow();
2894                (handle.connect_socket(), handle.unix_path.is_none())
2895            }
2896            None => (mio::net::TcpStream::connect(backend.address), true),
2897        };
2898        let socket = match dialed {
2899            Ok(socket) => socket,
2900            Err(io_error) => {
2901                context.backend_deltas.push(BackendDelta {
2902                    slot: backend.slot(),
2903                    change: BackendChange::DialFailed(context.now),
2904                });
2905                context.streams[stream_id]
2906                    .tried_backends
2907                    .push(backend.address);
2908                // The count the backend will hold once the delta is applied,
2909                // as `Backend::try_connect`'s error reported it.
2910                let failures = backend_registry
2911                    .handle(&backend)
2912                    .map_or(1, |handle| handle.borrow().failures + 1);
2913                return Err(BackendConnectionError::Backend(
2914                    BackendError::ConnectionFailures {
2915                        cluster_id: cluster_id.to_string(),
2916                        backend_address: backend.address,
2917                        failures,
2918                        error: io_error.to_string(),
2919                    },
2920                ));
2921            }
2922        };
2923
2924        // Past the connect, the reservation belongs to the connection built
2925        // on `socket`. An exit that abandons it drops the connection
2926        // unregistered, and `Connection` has no `Drop` that would release
2927        // anything, so the release is taken here, once, for every such exit:
2928        // `Connection::new_h2_client` finding no buffer, a refused
2929        // `Connection::start_stream`, and the `register_socket` rollback
2930        // (#1713). It is `ConnectionClosed`, not `DialFailed`: each of those
2931        // is a local failure, and charging it to the backend's failure count
2932        // and backoff would steer traffic away from a healthy backend.
2933        let slot = backend.slot();
2934        let attached = Self::attach_dialed(
2935            router, stream_id, context, session, proxy, cluster_id, h2, backend, socket, nodelay,
2936        );
2937        if attached.is_err() {
2938            context.backend_deltas.push(BackendDelta {
2939                slot,
2940                change: BackendChange::ConnectionClosed,
2941            });
2942            debug_assert!(
2943                !matches!(context.streams[stream_id].state, StreamState::Linked(_)),
2944                "an abandoned dial must not have linked its stream"
2945            );
2946        }
2947        attached
2948    }
2949
2950    /// Build, start and register the connection of a dial whose `connect(2)`
2951    /// succeeded, and hand it to `Router::commit_dialed`.
2952    ///
2953    /// SECURITY (CWE-400): every stateful side-effect (the
2954    /// `backend.connections` / `backend.pool.size` /
2955    /// `connections_per_backend` gauges, the slab session, the mio
2956    /// registration, the router-map entry, `stream.metrics.backend_start`) is
2957    /// deferred until AFTER both the `Connection` constructor and
2958    /// `start_stream` have succeeded. If either fails this returns `Err`
2959    /// without leaking a slab entry, an epoll registration, a gauge counter
2960    /// or a router-map entry. The `TcpStream` lives on the stack and is moved
2961    /// into the `Connection`; on failure the `Connection` — or the raw
2962    /// `TcpStream`, for the pool-exhaustion branch that drops inside
2963    /// `Connection::new_h2_client` — is dropped, closing the fd. No token is
2964    /// allocated before that point, so there is nothing to roll back. The
2965    /// backend's reservation is the caller's to release: every `Err` here
2966    /// drops the connection, and [`Self::dial_backend`] releases it once for
2967    /// all of them.
2968    #[allow(clippy::too_many_arguments)]
2969    fn attach_dialed(
2970        router: &mut Router,
2971        stream_id: GlobalStreamId,
2972        context: &mut Context<L>,
2973        session: &Rc<RefCell<dyn ProxySession>>,
2974        proxy: &Rc<RefCell<dyn L7Proxy>>,
2975        cluster_id: sozu_command::state::ClusterId,
2976        h2: bool,
2977        backend: BackendId,
2978        socket: mio::net::TcpStream,
2979        nodelay: bool,
2980    ) -> Result<(), BackendConnectionError> {
2981        if nodelay && let Err(e) = socket.set_nodelay(true) {
2982            error!(
2983                "{} error setting nodelay on back socket({:?}): {:?}",
2984                log_module_context!(context.http_context(stream_id)),
2985                socket,
2986                e
2987            );
2988        }
2989
2990        // Cache the backend's configured address so SOCKET log lines fired on
2991        // ECONNREFUSED (or any failed async `connect()`) can still render
2992        // `peer=<backend>` — `getpeername(2)` returns ENOTCONN in that state,
2993        // so the live lookup path would show `peer=None` exactly when the
2994        // operator needs the backend id.
2995        let backend_peer = Some(backend.address);
2996        let socket = SessionTcpStream::new(socket, context.session_ulid, backend_peer);
2997
2998        let flood_config = context.listener.borrow().get_h2_flood_config();
2999        let connection_config = context.listener.borrow().get_h2_connection_config();
3000        let stream_idle_timeout = context.listener.borrow().get_h2_stream_idle_timeout();
3001        let graceful_shutdown_deadline = context
3002            .listener
3003            .borrow()
3004            .get_h2_graceful_shutdown_deadline();
3005        let backend_id_for_gauge = Rc::clone(&backend.backend_id);
3006        let metric_incarnation = backend.metric_incarnation();
3007        let mut connection = if h2 {
3008            match Connection::new_h2_client(
3009                context.session_ulid,
3010                socket,
3011                cluster_id,
3012                backend,
3013                &mut *context.buffers,
3014                router.configured_connect_timeout,
3015                flood_config,
3016                connection_config,
3017                stream_idle_timeout,
3018                graceful_shutdown_deadline,
3019            ) {
3020                Some(connection) => connection,
3021                // Pool exhaustion: the socket was already dropped by
3022                // `new_h2_client` and no side-effect has been committed.
3023                None => return Err(BackendConnectionError::MaxBuffers),
3024            }
3025        } else {
3026            Connection::new_h1_client(
3027                context.session_ulid,
3028                socket,
3029                cluster_id,
3030                backend,
3031                router.configured_connect_timeout,
3032            )
3033        };
3034
3035        // Check the backend can accept a new stream BEFORE committing any
3036        // registry state. `start_stream` records its `StreamsStarted` delta
3037        // only once the start has succeeded, so a refusal leaves the backend
3038        // accounting untouched rather than needing a rollback.
3039        if !connection.start_stream(stream_id, context) {
3040            error!(
3041                "{} Backend rejected stream start (max concurrent streams reached)",
3042                log_module_context!(context.http_context(stream_id))
3043            );
3044            // `connection` (socket + pending timeout deadline) drops here; no
3045            // wheel entry was ever armed for it.
3046            return Err(BackendConnectionError::MaxSessionsMemory);
3047        }
3048
3049        // --- Happy path: commit side-effects in one atomic-ish block ---
3050        let stream = &mut context.streams[stream_id];
3051        stream.metrics.backend_start();
3052        stream.metrics.backend_id = stream.context.backend_id.clone();
3053        gauge_add!(names::backend::CONNECTIONS, 1);
3054        // `backend.pool.size` mirrors `backend.connections` exactly: one entry
3055        // per `Router::backends` token. The `-1` partners live in
3056        // `connection.rs::pre_close_client_bookkeeping` (graceful close) and
3057        // in `Mux::close` (session teardown). Symmetric pairing with both
3058        // decrement sites is the only defence against the gauge underflow
3059        // class of bug fixed by ff401b54 / aadb3fa4.
3060        gauge_add!(names::backend::POOL_SIZE, 1);
3061        // The cluster id moved into the connection; the stream's own copy,
3062        // stored by `Router::plan_connect`, names the same cluster.
3063        gauge_add!(
3064            names::backend::CONNECTIONS_PER_BACKEND,
3065            1,
3066            stream.context.cluster_id.as_deref(),
3067            Some(&backend_id_for_gauge),
3068            metric_incarnation
3069        );
3070
3071        let token = proxy.borrow().add_session(session.clone());
3072
3073        {
3074            let socket_ref = connection.socket_mut();
3075            if let Err(e) = proxy.borrow().register_socket(
3076                socket_ref,
3077                token,
3078                Interest::READABLE | Interest::WRITABLE,
3079            ) {
3080                // SECURITY (CWE-400): treat mio registration failure as a hard
3081                // connect failure. Without this rollback the gauges
3082                // (`backend.connections`, `backend.pool.size`,
3083                // `connections_per_backend`) and the slab session leak until
3084                // the connect timeout fires. Under fd pressure
3085                // (EMFILE/ENFILE) this can occur in tight bursts and poison
3086                // capacity dashboards.
3087                error!(
3088                    "{} error registering back socket: {:?} — rolling back",
3089                    log_module_context!(context.http_context(stream_id)),
3090                    e
3091                );
3092                // Undo the gauge increments committed above.
3093                gauge_add!(names::backend::CONNECTIONS, -1);
3094                gauge_add!(names::backend::POOL_SIZE, -1);
3095                gauge_add!(
3096                    names::backend::CONNECTIONS_PER_BACKEND,
3097                    -1,
3098                    context.http_context(stream_id).cluster_id.as_deref(),
3099                    Some(&backend_id_for_gauge),
3100                    metric_incarnation
3101                );
3102                // Release the `active_requests` charge `start_stream` took.
3103                //
3104                // The comment this replaces claimed the drop released it "via
3105                // the regular session drop path
3106                // (`pre_close_client_bookkeeping`)". That was never true:
3107                // `Connection` has no `impl Drop`, so dropping it runs no
3108                // bookkeeping at all. It was harmless only because the charge
3109                // is guarded on `BackendStatus::Connected` and a
3110                // freshly-dialled connection is `Connecting`, so there was
3111                // nothing to release. Emitting the release explicitly, under
3112                // the same guard the charge uses, makes the pair hold however
3113                // the status is reached rather than by accident.
3114                connection.release_start_stream_charge(context);
3115                proxy.borrow().remove_session(token);
3116                return Err(BackendConnectionError::MaxSessionsMemory);
3117            }
3118        }
3119
3120        router.commit_dialed(stream_id, context, token, connection);
3121        Ok(())
3122    }
3123
3124    fn ready_inner(
3125        &mut self,
3126        session: Rc<RefCell<dyn ProxySession>>,
3127        proxy: Rc<RefCell<dyn L7Proxy>>,
3128        _metrics: &mut SessionMetrics,
3129    ) -> SessionResult {
3130        let mut counter = 0;
3131        // How far this call forwarded stream bytes, read at the loop budget:
3132        // a session that reaches `MAX_LOOP_ITERATIONS` while progressing
3133        // yields to the event loop instead of closing (sozu-proxy/sozu#1906).
3134        let bytes_out_at_entry = self.context.stream_bytes_out;
3135        self.context.loop_budget_yielded = false;
3136
3137        // A linger decided outside this function (a timeout, a soft stop)
3138        // releases at the next pass; see the call after the frontend write.
3139        self.release_streams_and_backends_for_error_linger(&proxy);
3140
3141        if self.frontend.is_lingering() && self.frontend.readiness().event.is_hup() {
3142            // A lingering frontend reads to the client's EOF before it closes:
3143            // a close with its last bytes unread would still reset.
3144            self.frontend.readiness_mut().event.insert(Ready::READABLE);
3145        } else if self.frontend.readiness().event.is_hup() && self.frontend_exchange_in_flight() {
3146            // A half-closed client still gets the response to what it sent:
3147            // neither close nor queue `close_notify` while it is in flight.
3148            // What is already pending is flushed as a delayed close would.
3149            if self
3150                .frontend
3151                .has_pending_write_including_streams(&self.context)
3152            {
3153                self.frontend.readiness_mut().arm_writable();
3154            }
3155        } else if self.frontend.readiness().event.is_hup()
3156            && !self.delay_close_for_frontend_flush("frontend HUP")
3157        {
3158            debug!(
3159                "{} Mux closing on frontend HUP: {:?}",
3160                log_context!(self),
3161                self.frontend
3162            );
3163            return SessionResult::Close;
3164        }
3165
3166        // Start service timers on all active streams after the HUP check.
3167        // This mirrors session-level service_start/service_stop in Http(s)Session::ready()
3168        // to measure only CPU processing time, excluding epoll wait between cycles.
3169        for stream in &mut self.context.streams {
3170            if stream.state.is_open() {
3171                stream.metrics.service_start();
3172            }
3173        }
3174
3175        let start = Instant::now();
3176        self.context.debug.push(DebugEvent::ReadyTimestamp(
3177            std::time::SystemTime::now()
3178                .duration_since(std::time::UNIX_EPOCH)
3179                .unwrap_or_default()
3180                .as_millis() as usize,
3181        ));
3182        trace!("{} {:?}", log_context!(self), start);
3183        // Set once the inner loop has spent its budget and waits for the
3184        // client. The outer loop then re-enters the inner one at most once
3185        // more, for a stream this sweep just linked (an attached pooled
3186        // backend is armed by a synthetic event no epoll edge repeats), and
3187        // that iteration stops at the spent budget without counting or
3188        // logging the same limit again.
3189        let mut budget_spent = false;
3190        let mut budget_extra_pass = false;
3191        loop {
3192            // The mux's clock sample for this pass. Everything time-based
3193            // below — flood and back-pressure windows, the SETTINGS-ACK
3194            // deadline, per-stream liveness and flow-control-stall deadlines
3195            // — reads `context.now` (mirrored into `ConnectionH2::now` at
3196            // each entry point) rather than calling `Instant::now()` itself.
3197            //
3198            // Sampled per OUTER iteration so the inner loop DELIBERATELY
3199            // shares one instant: that is the property the snapshot exists
3200            // for — every frame processed in one inner sweep is weighed
3201            // against the same "now", so a burst cannot decay a rate window
3202            // out from under itself mid-sweep. It is not a bound on how long
3203            // the sweep may take: `MAX_LOOP_ITERATIONS` is a count, and a
3204            // count bounds iterations, not wall clock. `counter` is declared
3205            // above BOTH loops, so that budget is shared across every outer
3206            // iteration of one `ready()` call rather than reset per pass.
3207            // `start` above stays a separate sample so the trace reports the
3208            // true entry instant.
3209            self.context.now = Instant::now();
3210            self.context.now_wall_ms = unix_epoch_ms();
3211            self.context.debug.push(DebugEvent::LoopStart);
3212            loop {
3213                self.context.debug.push(DebugEvent::LoopIteration(counter));
3214                if self.frontend.readiness().filter_interest().is_readable() {
3215                    let res = {
3216                        let context = &mut self.context;
3217                        let res = self
3218                            .frontend
3219                            .readable(context, EndpointClient(&mut self.router));
3220                        context.debug.push(DebugEvent::SR(
3221                            self.frontend_token,
3222                            res,
3223                            self.frontend.readiness().clone(),
3224                        ));
3225                        res
3226                    };
3227                    match res {
3228                        MuxResult::Continue => {}
3229                        MuxResult::CloseSession => {
3230                            if !self.delay_close_for_frontend_flush("frontend readable") {
3231                                debug!(
3232                                    "{} Mux close from frontend readable: {:?}",
3233                                    log_context!(self),
3234                                    self.frontend
3235                                );
3236                                return SessionResult::Close;
3237                            }
3238                        }
3239                        MuxResult::Upgrade => {
3240                            self.sync_upgrade_buffers();
3241                            return SessionResult::Upgrade;
3242                        }
3243                    }
3244                }
3245
3246                let mut all_backends_readiness_are_empty = true;
3247                self.router.dead_backends.clear();
3248                let mut backend_close: Option<(&'static str, Token)> = None;
3249                for (token, client) in self.router.backends.iter_mut() {
3250                    // A WRITABLE on a connecting socket is not proof it
3251                    // connected: a dial that failed can surface as WRITABLE
3252                    // alone, its error in `SO_ERROR`, and sozu raises
3253                    // WRITABLE itself to flush queued output, as an HTTP/2
3254                    // backend's preface, while the dial is still in flight.
3255                    // Read the outcome before the WRITABLE arm below would
3256                    // mark the connection established: a failure takes the
3257                    // same connect-failure branch as a refused dial, a dial
3258                    // still in flight waits for the kernel's completion edge
3259                    // (sozu-proxy/sozu#1800).
3260                    if client.readiness().event.is_writable()
3261                        && matches!(
3262                            client.position(),
3263                            Position::Client(_, _, BackendStatus::Connecting(_))
3264                        )
3265                    {
3266                        match connect_outcome(client.socket()) {
3267                            ConnectOutcome::Connected => {}
3268                            ConnectOutcome::Pending => {
3269                                client.readiness_mut().event.remove(Ready::WRITABLE);
3270                            }
3271                            ConnectOutcome::Failed => {
3272                                client.readiness_mut().event.insert(Ready::ERROR);
3273                            }
3274                        }
3275                    }
3276                    let readiness = client.readiness_mut();
3277                    // Check the raw event for HUP/ERROR, not filter_interest():
3278                    // the event is what the poller reported, whatever the
3279                    // interest holds.
3280                    let dead = readiness.event.is_hup() || readiness.event.is_error();
3281                    if dead {
3282                        trace!(
3283                            "{} Backend({:?}) -> {:?}",
3284                            log_context_lite!(self),
3285                            token,
3286                            readiness
3287                        );
3288                        readiness.event.remove(Ready::WRITABLE);
3289                    }
3290
3291                    if client.readiness().filter_interest().is_writable() {
3292                        let position = client.position_mut();
3293                        match position {
3294                            Position::Client(
3295                                cluster_id,
3296                                backend,
3297                                BackendStatus::Connecting(start),
3298                            ) => {
3299                                #[cfg(debug_assertions)]
3300                                self.context
3301                                    .debug
3302                                    .push(DebugEvent::CCS(*token, cluster_id.clone()));
3303
3304                                // Health and latency are not invariant 14 and
3305                                // the load balancer READS `retry_policy`, so
3306                                // these three writes stay immediate, against
3307                                // the handle the slot resolves to. Only the
3308                                // `active_requests` charge below is a delta.
3309                                let Some(handle) = self.backend_registry.get(backend.slot()) else {
3310                                    error!(
3311                                        "{} connected backend names a slot this session never                                          minted: {:?}",
3312                                        log_context_lite!(self),
3313                                        backend
3314                                    );
3315                                    continue;
3316                                };
3317                                let mut backend_borrow = handle.borrow_mut();
3318                                if backend_borrow.retry_policy.is_down() {
3319                                    info!(
3320                                        "{} backend server {} at {} is up",
3321                                        log_context_lite!(self),
3322                                        backend.backend_id,
3323                                        backend.address
3324                                    );
3325                                    incr!(
3326                                        names::backend::UP,
3327                                        Some(cluster_id),
3328                                        Some(&backend.backend_id),
3329                                        backend.metric_incarnation()
3330                                    );
3331                                    gauge!(
3332                                        names::backend::AVAILABLE,
3333                                        1,
3334                                        Some(cluster_id),
3335                                        Some(&backend.backend_id),
3336                                        backend.metric_incarnation()
3337                                    );
3338                                    push_event(Event {
3339                                        kind: EventKind::BackendUp as i32,
3340                                        backend_id: Some(backend.backend_id.to_string()),
3341                                        address: Some(backend.address.into()),
3342                                        cluster_id: Some(cluster_id.to_string()),
3343                                        metric_detail: None,
3344                                    });
3345                                }
3346
3347                                //successful connection, reset failure counter
3348                                backend_borrow.failures = 0;
3349                                backend_borrow
3350                                    .set_connection_time(start.elapsed(), self.context.now);
3351                                backend_borrow.retry_policy.succeed(self.context.now);
3352                                drop(backend_borrow);
3353
3354                                // These streams linked while the connection
3355                                // was still `Connecting`, so
3356                                // `Connection::start_stream`'s `Connected`
3357                                // guard skipped their charge. This is where
3358                                // every first stream of a fresh dial is
3359                                // charged — which is why the ledger cannot be
3360                                // core-only and still balance.
3361                                let mut started = 0;
3362                                if let Some(ids) = self.context.backend_streams.get(token) {
3363                                    started = ids.len();
3364                                    for &stream_id in ids.iter() {
3365                                        self.context.streams[stream_id].metrics.backend_connected();
3366                                    }
3367                                }
3368                                if started > 0 {
3369                                    let slot = backend.slot();
3370                                    self.context.backend_deltas.push(BackendDelta {
3371                                        slot,
3372                                        change: BackendChange::StreamsStarted(started),
3373                                    });
3374                                }
3375                                trace!(
3376                                    "{} connection success: {:?}",
3377                                    log_context_lite!(self),
3378                                    backend
3379                                );
3380                                *position = Position::Client(
3381                                    std::mem::take(cluster_id),
3382                                    backend.clone(),
3383                                    BackendStatus::Connected,
3384                                );
3385                                client.set_timeout_duration(
3386                                    self.router.configured_backend_timeout,
3387                                    self.context.now,
3388                                );
3389                            }
3390                            Position::Client(..) => {}
3391                            Position::Server => {
3392                                error!(
3393                                    "{} backend connection cannot be in Server position",
3394                                    log_context_lite!(self)
3395                                );
3396                            }
3397                        }
3398                        let res = {
3399                            let context = &mut self.context;
3400                            let res = client.writable(context, EndpointServer(&mut self.frontend));
3401                            context.debug.push(DebugEvent::CW(
3402                                *token,
3403                                res,
3404                                client.readiness().clone(),
3405                            ));
3406                            res
3407                        };
3408                        match res {
3409                            MuxResult::Continue => {}
3410                            MuxResult::Upgrade => {
3411                                error!(
3412                                    "{} only frontend connections can trigger Upgrade",
3413                                    log_context_lite!(self)
3414                                );
3415                            }
3416                            MuxResult::CloseSession => {
3417                                backend_close = Some(("backend writable", *token));
3418                                break;
3419                            }
3420                        }
3421                        // Cross-readiness: backend wrote → wake frontend reader
3422                        let context = &mut self.context;
3423                        self.frontend.try_resume_reading(context);
3424                    }
3425
3426                    if client.readiness().filter_interest().is_readable() {
3427                        let res = {
3428                            let context = &mut self.context;
3429                            let res = client.readable(context, EndpointServer(&mut self.frontend));
3430                            context.debug.push(DebugEvent::CR(
3431                                *token,
3432                                res,
3433                                client.readiness().clone(),
3434                            ));
3435                            res
3436                        };
3437                        match res {
3438                            MuxResult::Continue => {}
3439                            MuxResult::Upgrade => {
3440                                error!(
3441                                    "{} only frontend connections can trigger Upgrade (readable)",
3442                                    log_context_lite!(self)
3443                                );
3444                            }
3445                            MuxResult::CloseSession => {
3446                                backend_close = Some(("backend readable", *token));
3447                                break;
3448                            }
3449                        }
3450                    }
3451
3452                    if dead
3453                        && !client.readiness().filter_interest().is_readable()
3454                        && !client.has_buffer_pressure(&self.context)
3455                    {
3456                        self.context
3457                            .debug
3458                            .push(DebugEvent::CH(*token, client.readiness().clone()));
3459                        trace!("{} Closing {:#?}", log_context_lite!(self), client);
3460                        match client.position() {
3461                            Position::Client(cluster_id, backend, BackendStatus::Connecting(_)) => {
3462                                // A dial that never completed charged nothing
3463                                // to `active_requests`, so this arm touches
3464                                // only the health counters — not the ledger.
3465                                let Some(handle) = self.backend_registry.get(backend.slot()) else {
3466                                    error!(
3467                                        "{} failed backend names a slot this session never                                          minted: {:?}",
3468                                        log_context_lite!(self),
3469                                        backend
3470                                    );
3471                                    continue;
3472                                };
3473                                let mut backend_borrow = handle.borrow_mut();
3474                                backend_borrow.failures += 1;
3475
3476                                let already_unavailable = backend_borrow.retry_policy.is_down();
3477                                // The jitter comes from the worker's backend
3478                                // map, the one generator selection draws
3479                                // from; nothing else borrows the map here.
3480                                backend_borrow
3481                                    .retry_policy
3482                                    .fail(self.context.now, self.backends.borrow_mut().rng());
3483                                incr!(
3484                                    names::backend::CONNECTIONS_ERROR,
3485                                    Some(cluster_id),
3486                                    Some(&backend.backend_id),
3487                                    backend.metric_incarnation()
3488                                );
3489                                if !already_unavailable && backend_borrow.retry_policy.is_down() {
3490                                    error!(
3491                                        "{} backend server {} at {} is down",
3492                                        log_context_lite!(self),
3493                                        backend.backend_id,
3494                                        backend.address
3495                                    );
3496                                    incr!(
3497                                        names::backend::DOWN,
3498                                        Some(cluster_id),
3499                                        Some(&backend.backend_id),
3500                                        backend.metric_incarnation()
3501                                    );
3502                                    gauge!(
3503                                        names::backend::AVAILABLE,
3504                                        0,
3505                                        Some(cluster_id),
3506                                        Some(&backend.backend_id),
3507                                        backend.metric_incarnation()
3508                                    );
3509                                    push_event(Event {
3510                                        kind: EventKind::BackendDown as i32,
3511                                        backend_id: Some(backend.backend_id.to_string()),
3512                                        address: Some(backend.address.into()),
3513                                        cluster_id: Some(cluster_id.to_string()),
3514                                        metric_detail: None,
3515                                    });
3516                                }
3517                                drop(backend_borrow);
3518                                // Every stream this dial carried failed to
3519                                // reach `backend`: a retry of any of them
3520                                // prefers another one (sozu-proxy/sozu#1800).
3521                                if let Some(ids) = self.context.backend_streams.get(token) {
3522                                    for &stream_id in ids.iter() {
3523                                        self.context.streams[stream_id]
3524                                            .tried_backends
3525                                            .push(backend.address);
3526                                    }
3527                                }
3528                                trace!(
3529                                    "{} connection fail: {:?}",
3530                                    log_context_lite!(self),
3531                                    backend
3532                                );
3533                            }
3534                            Position::Client(_, backend, _) => {
3535                                // A backend that dies past `Connecting` still
3536                                // owes the charge every stream it carries
3537                                // took — the same `StreamsEnded(count)` the
3538                                // teardown path emits.
3539                                let count = self
3540                                    .context
3541                                    .backend_streams
3542                                    .get(token)
3543                                    .map_or(0, |ids| ids.len());
3544                                self.context.backend_deltas.push(BackendDelta {
3545                                    slot: backend.slot(),
3546                                    change: BackendChange::StreamsEnded(count),
3547                                });
3548                            }
3549                            Position::Server => {
3550                                error!(
3551                                    "{} dead backend cannot be in Server position",
3552                                    log_context_lite!(self)
3553                                );
3554                            }
3555                        }
3556                        client.close(&mut self.context, EndpointServer(&mut self.frontend));
3557                        self.router.dead_backends.push(*token);
3558                    }
3559
3560                    if client.has_loop_work(&self.context) {
3561                        all_backends_readiness_are_empty = false;
3562                    }
3563                }
3564                // Remove dead backends from the map BEFORE handling
3565                // backend_close. client.close() already decremented
3566                // connections_per_backend / backend.connections gauges in
3567                // the loop above; if we return SessionResult::Close before
3568                // removing them, Mux::close() would decrement again
3569                // (double-decrement → gauge underflow).
3570                if !self.router.dead_backends.is_empty() {
3571                    for token in &self.router.dead_backends {
3572                        let proxy_borrow = proxy.borrow();
3573                        // The reverse-index entry `remove_backend_stream`
3574                        // kept for its capacity leaves with its connection.
3575                        // Only an emptied one: an entry still naming a stream
3576                        // is the drift the debug check at the end of this
3577                        // pass reports, and removing it here would hide it.
3578                        if self
3579                            .context
3580                            .backend_streams
3581                            .get(token)
3582                            .is_some_and(|ids| ids.is_empty())
3583                        {
3584                            self.context.backend_streams.remove(token);
3585                        }
3586                        if let Some(mut client) = self.router.backends.remove(token) {
3587                            // No explicit timer cancel: the token has left
3588                            // `router.backends`, so `Mux::reschedule` drops its
3589                            // handle on the way out of `ready` and
3590                            // `TimeoutContainer::drop` cancels the entry.
3591                            // No `EPOLL_CTL_DEL`: `client` drops at the end of
3592                            // this block, and closing its socket removes it from
3593                            // the epoll set — see `HttpsSession::close`
3594                            // (`lib/src/https.rs`).
3595                            let peer_closed = client.peer_closed();
3596                            let socket = client.socket_mut();
3597                            // invariant: write-only shutdown — Shutdown::Both on a TLS frontend
3598                            // discards the receive buffer and elicits TCP RST, truncating the
3599                            // already-queued response. Canonical write-up: `HttpsSession::close`
3600                            // (`lib/src/https.rs`). Backend sockets follow the same discipline for symmetry.
3601                            // Skipped once the backend has closed: see `shutdown_write`.
3602                            if let Err(e) = shutdown_write(socket, peer_closed)
3603                                && e.kind() != ErrorKind::NotConnected
3604                            {
3605                                error!(
3606                                    "{} error shutting down back socket({:?}): {:?}",
3607                                    log_context!(self),
3608                                    socket,
3609                                    e
3610                                );
3611                            }
3612                        } else {
3613                            error!("{} session {:?} has no backend!", log_context!(self), token);
3614                        }
3615                        if !proxy_borrow.remove_session(*token) {
3616                            error!(
3617                                "{} session {:?} was already removed!",
3618                                log_context!(self),
3619                                token
3620                            );
3621                        }
3622                    }
3623                    trace!("{} FRONTEND: {:#?}", log_context!(self), self.frontend);
3624                    trace!(
3625                        "{} BACKENDS: {:#?}",
3626                        log_context!(self),
3627                        self.router.backends
3628                    );
3629                }
3630                if let Some((reason, token)) = backend_close {
3631                    if !self.delay_close_for_frontend_flush(reason) {
3632                        debug!(
3633                            "{} Mux close from {} token={:?}: frontend={:?}",
3634                            log_context!(self),
3635                            reason,
3636                            token,
3637                            self.frontend
3638                        );
3639                        return SessionResult::Close;
3640                    }
3641                    all_backends_readiness_are_empty = false;
3642                }
3643
3644                if self.frontend.readiness().filter_interest().is_writable() {
3645                    let res = {
3646                        let context = &mut self.context;
3647                        let res = self
3648                            .frontend
3649                            .writable(context, EndpointClient(&mut self.router));
3650                        context.debug.push(DebugEvent::SW(
3651                            self.frontend_token,
3652                            res,
3653                            self.frontend.readiness().clone(),
3654                        ));
3655                        res
3656                    };
3657                    match res {
3658                        MuxResult::Continue => {}
3659                        MuxResult::CloseSession => {
3660                            if !self.delay_close_for_frontend_flush("frontend writable") {
3661                                debug!(
3662                                    "{} Mux close from frontend writable: {:?}",
3663                                    log_context!(self),
3664                                    self.frontend
3665                                );
3666                                return SessionResult::Close;
3667                            }
3668                        }
3669                        MuxResult::Upgrade => {
3670                            self.sync_upgrade_buffers();
3671                            return SessionResult::Upgrade;
3672                        }
3673                    }
3674                    // The pass that decides an error-GOAWAY linger, pending
3675                    // on `close_notify` or already draining, releases its
3676                    // streams and backends before anything below can touch
3677                    // them, `pending_links` included.
3678                    self.release_streams_and_backends_for_error_linger(&proxy);
3679                    // Cross-readiness: the frontend write can hand work back to
3680                    // a backend — a parked one resumes once its peer drained
3681                    // buffer space, and forwarding an interim 1xx re-arms the
3682                    // backend read so the response bytes already buffered
3683                    // behind it get parsed (`ConnectionH1::writable`'s interim
3684                    // arms, `ConnectionH2::handle_1xx_reset`). The
3685                    // `all_backends_readiness_are_empty` flag was recorded
3686                    // before this write, so it is refreshed from every
3687                    // backend's readiness now: a stale `true` would exit the
3688                    // loop with a backend armed and no socket event left to
3689                    // bring it back.
3690                    let context = &mut self.context;
3691                    for backend in self.router.backends.values_mut() {
3692                        if backend.try_resume_reading(context) || backend.has_loop_work(context) {
3693                            all_backends_readiness_are_empty = false;
3694                        }
3695                    }
3696                }
3697
3698                // A pooled connection whose backend left the configuration
3699                // is never reused (`Router::decide_after_gate`); once it
3700                // carries no stream it has nothing left to do, so drop it
3701                // rather than leave it idle until the session ends. The HUP
3702                // `force_disconnect` raises sends it through the dead-backend
3703                // sweep above on the next iteration, which releases its
3704                // connection on the backend and lets a `Closing` backend
3705                // reach `Closed`.
3706                for (token, backend) in self.router.backends.iter_mut() {
3707                    let retired = backend
3708                        .idle_pooled_backend()
3709                        .is_some_and(|id| self.backend_registry.is_retired(id))
3710                        && self
3711                            .context
3712                            .backend_streams
3713                            .get(token)
3714                            .is_none_or(|ids| ids.is_empty());
3715                    if retired {
3716                        debug!(
3717                            "{} closing idle backend connection {:?}: its backend was removed",
3718                            log_context_lite!(self),
3719                            token
3720                        );
3721                        backend.force_disconnect();
3722                        all_backends_readiness_are_empty = false;
3723                    }
3724                }
3725
3726                // A frontend HUP is handled here as it is on entry: nothing
3727                // else in this loop acts on one. Once no output is left to
3728                // flush, this check closes the session, including after a
3729                // flush the entry check delayed the close for and the
3730                // writable arm above completed. A lingering frontend is left
3731                // to its drain: `ConnectionH1::start_linger` raises READABLE,
3732                // which `Mux::frontend_exchange_in_flight` counts as unread
3733                // input until the drain reads the client's EOF, so the
3734                // client's last bytes are never left unread to reset the
3735                // connection. While output is
3736                // pending, the HUP alone is not work this loop can progress:
3737                // it stops when the frontend has nothing to read or write,
3738                // and the next readiness event resumes the flush. Counting
3739                // the HUP as work spun the loop to `MAX_LOOP_ITERATIONS`
3740                // after every client hang-up. A HUP that comes from a
3741                // half-close leaves the exchange in flight to be served
3742                // (`Mux::frontend_exchange_in_flight`): the session closes
3743                // here once its response is complete.
3744                if self.frontend.readiness().event.is_hup()
3745                    && !self
3746                        .frontend
3747                        .has_pending_write_including_streams(&self.context)
3748                    && !self.frontend_exchange_in_flight()
3749                    && !self.delay_close_for_frontend_flush("frontend HUP")
3750                {
3751                    debug!(
3752                        "{} Mux closing on frontend HUP: {:?}",
3753                        log_context!(self),
3754                        self.frontend
3755                    );
3756                    return SessionResult::Close;
3757                }
3758                let frontend_work = self.frontend.readiness().filter_interest();
3759                if !frontend_work.is_readable()
3760                    && !frontend_work.is_writable()
3761                    && !frontend_work.is_error()
3762                    && all_backends_readiness_are_empty
3763                {
3764                    break;
3765                }
3766
3767                counter += 1;
3768                if counter >= MAX_LOOP_ITERATIONS {
3769                    if budget_spent {
3770                        break;
3771                    }
3772                    // A call that forwarded at least a byte per budgeted
3773                    // iteration is a transfer whose peers
3774                    // keep up, not a spin. Closing it cut a fast download or
3775                    // upload at a random offset with a clean close
3776                    // (sozu-proxy/sozu#1906); running on would hold the
3777                    // single-threaded worker for the whole transfer. Leave
3778                    // every readiness bit as it is and yield:
3779                    // `Server::resume_yielded_sessions` calls `ready` again
3780                    // after the next poll, which serves the other sessions
3781                    // first.
3782                    if progressed_through_loop_budget(
3783                        bytes_out_at_entry,
3784                        self.context.stream_bytes_out,
3785                    ) {
3786                        incr!(names::http::MUX_LOOP_BUDGET_YIELD);
3787                        debug!(
3788                            "{} Mux loop reached {} iterations while forwarding, yielding: frontend={:?}",
3789                            log_context!(self),
3790                            MAX_LOOP_ITERATIONS,
3791                            self.frontend.readiness()
3792                        );
3793                        self.context.loop_budget_yielded = true;
3794                        budget_spent = true;
3795                        break;
3796                    }
3797                    if self.wait_for_client_at_loop_limit() {
3798                        budget_spent = true;
3799                        break;
3800                    }
3801                    return SessionResult::Close;
3802                }
3803            }
3804
3805            let context = &mut self.context;
3806            let mut dirty = false;
3807            while let Some(stream_id) = context.pending_links.pop_front() {
3808                let Some(stream) = context.streams.get(stream_id) else {
3809                    continue;
3810                };
3811                if stream.state != StreamState::Link {
3812                    continue;
3813                }
3814                // Before the first request triggers a stream Link, the frontend timeout is set
3815                // to a shorter request_timeout, here we switch to the longer nominal timeout.
3816                //
3817                // Only on a request's first link. A re-link — failover after a
3818                // failed dial, a stale-upstream replay — keeps the deadline armed
3819                // then, so `front_timeout` bounds the whole failover: re-arming it
3820                // on every attempt let a request spend `max_connection_attempts`
3821                // times `connect_timeout` before any answer. When it expires
3822                // during failover, the frontend-timeout branch of
3823                // `Mux::timeout_inner` answers as for any request still waiting on
3824                // a backend (sozu-proxy/sozu#1800).
3825                if stream.attempts == 0 {
3826                    self.frontend
3827                        .set_timeout_duration(self.configured_frontend_timeout, context.now);
3828                }
3829                let front_readiness = self.frontend.readiness_mut();
3830                dirty = true;
3831                // Settle the ledger before the ONLY reader of backend load
3832                // state on this path runs: the load balancer reached from
3833                // `Mux::dial_backend` below weighs `active_requests` and
3834                // `connection_time`. Draining here is what keeps a second
3835                // stream linking in the same pass seeing the first stream's
3836                // charge, exactly as it did when the charge was applied in
3837                // place. It is also what makes the view each selection reads
3838                // current: selection reserves `active_connections` itself
3839                // (`BackendMap::reserve_backend`), and a `connect(2)` that
3840                // failed in the previous iteration left a `DialFailed`
3841                // release on the ledger, applied here before the next
3842                // selection reads the counters (#1684).
3843                self.backend_registry
3844                    .apply_all(&mut context.backend_deltas, &self.backends);
3845                // Build the routing view once for this decision, from a single
3846                // `proxy.borrow()`. Holding one `Ref` for the call is what
3847                // gives every read inside it the same cluster map; the two
3848                // sites used to take separate borrows. A second immutable
3849                // borrow inside (`plan_connect`'s limit gate, and
3850                // `attach_dialed`'s `add_session` / `register_socket`) is fine
3851                // — nothing on this path borrows the proxy mutably.
3852                let proxy_ref = proxy.borrow();
3853                let backend_registry = &self.backend_registry;
3854                let retired = |backend: &BackendId| backend_registry.is_retired(backend);
3855                let max_connection_attempts = proxy_ref.sessions().borrow().max_connection_attempts;
3856                let view = router::RoutingView::new(proxy_ref.clusters(), proxy_ref.kind())
3857                    .with_backend_retired(&retired)
3858                    .with_max_connection_attempts(max_connection_attempts);
3859                match self
3860                    .router
3861                    .plan_connect(stream_id, context, &view)
3862                    .and_then(|step| match step {
3863                        router::ConnectStep::Decided(plan) => Ok(plan),
3864                        // The core resolved a cluster and paused: the
3865                        // per-(cluster, source-IP) gate is a consult AND a
3866                        // mutation of worker-global session state, so the core
3867                        // cannot perform it. Answer it here, then resume.
3868                        //
3869                        // The track must land BEFORE the decision resumes, not
3870                        // after: `cluster_ip_at_limit` is read by every other
3871                        // session on this worker and by `lib/src/tcp.rs`'s own
3872                        // gate, so a stream admitted but not yet counted is a
3873                        // slot a concurrent stream can take twice.
3874                        router::ConnectStep::CheckIpLimit(resume) => {
3875                            // `plan_connect` stored the routed cluster before
3876                            // it paused; an unset one is refused, not gated
3877                            // against an empty key.
3878                            let Some(cluster_id) =
3879                                context.http_context(stream_id).cluster_id.as_ref()
3880                            else {
3881                                return Err(BackendConnectionError::MaxSessionsMemory);
3882                            };
3883                            let verdict = consult_ip_gate(
3884                                &proxy.borrow().sessions(),
3885                                self.frontend_token,
3886                                cluster_id,
3887                                &resume,
3888                            );
3889                            self.router
3890                                .plan_connect_resume(stream_id, context, &view, resume, verdict)
3891                        }
3892                    })
3893                    .and_then(|plan| match plan {
3894                        router::ConnectPlan::Attached => Ok(()),
3895                        router::ConnectPlan::Dial {
3896                            cluster_id,
3897                            h2,
3898                            frontend_should_stick,
3899                        } => Self::dial_backend(
3900                            &mut self.router,
3901                            &mut self.backend_registry,
3902                            &self.backends,
3903                            stream_id,
3904                            context,
3905                            &session,
3906                            &proxy,
3907                            cluster_id,
3908                            h2,
3909                            frontend_should_stick,
3910                        ),
3911                    }) {
3912                    Ok(_) => {
3913                        let state = context.streams[stream_id].state;
3914                        context.debug.push(DebugEvent::CC(stream_id, state));
3915                    }
3916                    Err(error) => {
3917                        trace!("{} Connection error: {}", log_module_context!(), error);
3918                        let stream = &mut context.streams[stream_id];
3919                        // The registry this stream captured when its request
3920                        // arrived, not whatever the listener holds now.
3921                        let answers_rc = stream.answers.clone();
3922                        let answers = answers_rc.borrow();
3923                        use BackendConnectionError as BE;
3924                        match error {
3925                            BE::MaxConnectionRetries(_)
3926                            | BE::MaxSessionsMemory
3927                            | BE::MaxBuffers => {
3928                                warn!(
3929                                    "{} backend retry budget exhausted: {}",
3930                                    log_module_context!(stream.context),
3931                                    error
3932                                );
3933                                set_default_answer(stream, front_readiness, 503, &answers);
3934                            }
3935                            BE::Backend(BackendError::NoBackendForCluster(ref cluster_id)) => {
3936                                // A cluster that has backends, none of them
3937                                // selectable, is in an outage: its backends
3938                                // failed their health checks or the
3939                                // connections Sōzu opened to them. A cluster
3940                                // with no backend at all is a routing miss.
3941                                stream.context.backends_unavailable = self
3942                                    .backends
3943                                    .borrow()
3944                                    .backends
3945                                    .get(cluster_id.as_str())
3946                                    .is_some_and(|list| !list.backends.is_empty());
3947                                set_default_answer(stream, front_readiness, 503, &answers);
3948                            }
3949                            BE::RetrieveClusterError(RetrieveClusterError::RetrieveFrontend(
3950                                ref err,
3951                            )) => {
3952                                // RFC 9110 §15.5.1: a malformed authority is a
3953                                // 400. A syntactically valid authority that
3954                                // simply has no matching frontend stays on the
3955                                // historical 404 path.
3956                                let code = match err {
3957                                    FrontendFromRequestError::HostParse { .. }
3958                                    | FrontendFromRequestError::InvalidCharsAfterHost(_) => 400,
3959                                    FrontendFromRequestError::NoClusterFound(_) => 404,
3960                                };
3961                                set_default_answer(stream, front_readiness, code, &answers);
3962                            }
3963                            BE::RetrieveClusterError(RetrieveClusterError::UnauthorizedRoute) => {
3964                                set_default_answer(stream, front_readiness, 401, &answers);
3965                            }
3966                            BE::RetrieveClusterError(RetrieveClusterError::SilentClose) => {
3967                                // No response byte (sozu-proxy/sozu#767): an
3968                                // H2 frontend resets this stream with
3969                                // RST_STREAM(CANCEL) while the connection's
3970                                // other streams continue; an H1 frontend
3971                                // closes the connection without writing.
3972                                // The RTT is sampled for this refusal only,
3973                                // as the access-log sites do (#1590).
3974                                let client_rtt = self.frontend.rtt();
3975                                for event in silent_close_answer(
3976                                    stream,
3977                                    self.frontend.readiness_mut(),
3978                                    context.listener.clone(),
3979                                    client_rtt,
3980                                )
3981                                .into_iter()
3982                                .flatten()
3983                                {
3984                                    crate::protocol::mux::h2::record_metric(event);
3985                                }
3986                            }
3987                            BE::RetrieveClusterError(
3988                                RetrieveClusterError::SniAuthorityMismatch { .. },
3989                            ) => {
3990                                // RFC 9110 §15.5.20: 421 Misdirected Request is the
3991                                // semantically correct status for an authority that
3992                                // does not belong to this TLS connection. The
3993                                // http.sni_authority_mismatch metric emitted in
3994                                // `route_from_request` remains the durable signal;
3995                                // the 421 body here is what a client sees and may
3996                                // retry on a fresh TLS connection with a matching SNI.
3997                                set_default_answer(stream, front_readiness, 421, &answers);
3998                            }
3999                            BE::RetrieveClusterError(RetrieveClusterError::HttpsRedirect) => {
4000                                // Use the redirect status stashed by `Router::route_from_request`
4001                                // (#1009). Falls back to 301 for the legacy
4002                                // `cluster.https_redirect = true` path that does
4003                                // not set the field.
4004                                let code = stream.context.redirect_status.unwrap_or(301);
4005                                set_default_answer(stream, front_readiness, code, &answers);
4006                            }
4007
4008                            // `connect(2)` itself failed: the dial was refused
4009                            // before it could be registered. `Mux::dial_backend`
4010                            // recorded the failure and the address; like a dial
4011                            // refused later, the request goes to another
4012                            // backend while its attempts last, and
4013                            // `Router::plan_connect` answers 503 once they are
4014                            // spent (sozu-proxy/sozu#1800). The stream is still
4015                            // `Link`.
4016                            BE::Backend(BackendError::ConnectionFailures { ref error, .. }) => {
4017                                debug!(
4018                                    "{} backend connect failed, retrying: {}",
4019                                    log_module_context!(stream.context),
4020                                    error
4021                                );
4022                                debug_assert_eq!(stream.state, StreamState::Link);
4023                                context.pending_links.push_back(stream_id);
4024                            }
4025                            BE::Backend(ref e) => {
4026                                error!("{} backend connection error: {}", log_module_context!(), e);
4027                                set_default_answer(stream, front_readiness, 503, &answers);
4028                            }
4029                            BE::RetrieveClusterError(ref other) => {
4030                                error!(
4031                                    "{} unexpected RetrieveClusterError variant: {:?}",
4032                                    log_module_context!(),
4033                                    other
4034                                );
4035                                set_default_answer(stream, front_readiness, 503, &answers);
4036                            }
4037                            // TCP specific error
4038                            BE::NotFound(ref msg) => {
4039                                error!(
4040                                    "{} NotFound is TCP-specific, not reachable in mux: {:?}",
4041                                    log_module_context!(),
4042                                    msg
4043                                );
4044                                set_default_answer(stream, front_readiness, 503, &answers);
4045                            }
4046                            // Per-(cluster, source-IP) connection limit reached.
4047                            // Emit HTTP 429 with the resolved `Retry-After`. The
4048                            // value is computed in `Router::plan_connect` (where the
4049                            // SessionManager + cluster override are reachable)
4050                            // and stashed on the stream context just before the
4051                            // error is returned, so the answer engine can render
4052                            // (or elide) the header without re-deriving the
4053                            // resolution chain here.
4054                            BE::TooManyConnectionsPerIp { ref cluster_id } => {
4055                                debug!(
4056                                    "{} per-(cluster, source-IP) limit hit for cluster {:?}",
4057                                    log_module_context!(),
4058                                    cluster_id
4059                                );
4060                                let retry_after = stream.context.retry_after_seconds;
4061                                set_default_answer_with_retry_after(
4062                                    stream,
4063                                    front_readiness,
4064                                    429,
4065                                    &answers,
4066                                    retry_after,
4067                                );
4068                            }
4069                            // A stale-upstream replay whose preconditions no
4070                            // longer hold (sozu-proxy/sozu#1442). 502 is the
4071                            // answer this stream received before the replay
4072                            // existed, so refusing costs the client nothing it
4073                            // would not already have paid — and costs it far
4074                            // less than H1 wire bytes framed as an HTTP/2 DATA
4075                            // payload, or a second partial copy of the request
4076                            // trailing the first.
4077                            BE::ReplayRefused(reason) => {
4078                                warn!(
4079                                    "{} stale-upstream replay refused: {}",
4080                                    log_module_context!(stream.context),
4081                                    reason
4082                                );
4083                                set_default_answer(stream, front_readiness, 502, &answers);
4084                            }
4085                        }
4086                        context.debug.push(DebugEvent::CCF(stream_id, error));
4087                    }
4088                }
4089                // Every routing error arm sets a default answer, or for a
4090                // silent close an errored empty one, moving the stream out of
4091                // Link state, except a failed `connect(2)`, which re-queues it
4092                // above for another backend.
4093            }
4094            if !dirty || budget_extra_pass {
4095                break;
4096            }
4097            budget_extra_pass = budget_spent;
4098        }
4099
4100        // Stop service timers before yielding to epoll, so idle wait time is excluded
4101        // from the service_time metric. For Close/Upgrade returns, close() handles cleanup.
4102        for stream in &mut self.context.streams {
4103            if stream.state.is_open() {
4104                stream.metrics.service_stop();
4105            }
4106        }
4107
4108        #[cfg(debug_assertions)]
4109        {
4110            use std::collections::HashMap;
4111
4112            // Verify backend_streams index matches actual stream states.
4113            let mut expected: HashMap<Token, Vec<GlobalStreamId>> = HashMap::new();
4114            for (id, stream) in self.context.streams.iter().enumerate() {
4115                if let StreamState::Linked(token) = stream.state {
4116                    expected.entry(token).or_default().push(id);
4117                }
4118            }
4119            // An emptied entry is capacity kept for the next link on the same
4120            // connection (#1583), not a link: it is left out of the count,
4121            // and it must name a connection the router still holds, which is
4122            // what keeps the index bounded by the live backends.
4123            for (token, ids) in self.context.backend_streams.iter() {
4124                assert!(
4125                    !ids.is_empty() || self.router.backends.contains_key(token),
4126                    "backend_streams keeps an emptied entry for {token:?}, \
4127                     which is not a live backend connection"
4128                );
4129            }
4130            assert_eq!(
4131                expected.len(),
4132                self.context
4133                    .backend_streams
4134                    .values()
4135                    .filter(|ids| !ids.is_empty())
4136                    .count(),
4137                "backend_streams index key count mismatch: expected={:?}, actual={:?}",
4138                expected,
4139                self.context.backend_streams
4140            );
4141            for (token, mut expected_ids) in expected {
4142                let mut actual_ids = self
4143                    .context
4144                    .backend_streams
4145                    .get(&token)
4146                    .map_or_else(Vec::new, |ids| ids.to_vec());
4147                expected_ids.sort();
4148                actual_ids.sort();
4149                assert_eq!(
4150                    expected_ids, actual_ids,
4151                    "backend_streams index mismatch for token {token:?}",
4152                );
4153            }
4154        }
4155
4156        SessionResult::Continue
4157    }
4158
4159    fn timeout_inner(&mut self, token: Token, _metrics: &mut SessionMetrics) -> StateResult {
4160        trace!("{} MuxState::timeout({:?})", log_context!(self), token);
4161        // `timeout` runs outside `ready()`, so it is its own sampling point.
4162        // `cancel_timed_out_streams` below reaps on this snapshot.
4163        self.context.now = Instant::now();
4164        self.context.now_wall_ms = unix_epoch_ms();
4165        // Consume the wheel entry and re-validate it BEFORE the per-token
4166        // branches, so both of them share one gate. See
4167        // `Mux::consume_timer_entry`.
4168        if !self.consume_timer_entry(token) {
4169            trace!(
4170                "{} MuxState::timeout: early wheel delivery on {:?}, re-armed",
4171                log_context!(self),
4172                token
4173            );
4174            return StateResult::Continue;
4175        }
4176        let front_is_h2 = match self.frontend {
4177            Connection::H1(_) => false,
4178            Connection::H2(_) => true,
4179        };
4180        if self.frontend_token == token && self.frontend.is_lingering() {
4181            // The lingering close's deadline: stop draining and close.
4182            debug!(
4183                "{} Mux lingering close deadline reached: {:?}",
4184                log_context!(self),
4185                self.frontend
4186            );
4187            return StateResult::CloseSession;
4188        }
4189        let mut should_close = true;
4190        let mut should_write = false;
4191        if self.frontend_token == token {
4192            trace!(
4193                "{} MuxState::timeout_frontend({:#?})",
4194                log_context!(self),
4195                self.frontend
4196            );
4197            // The per-stream reaper (bidirectional-idle + outbound
4198            // flow-control-stall guards) normally runs only from `readable()`,
4199            // but a fully-silent peer never triggers a read event. Run it on the
4200            // connection-timeout path too so a window-stalled stream — a buffered
4201            // response the peer refuses to drain by holding its receive window
4202            // shut — is reaped and its MAX_CONCURRENT_STREAMS slot freed, instead
4203            // of lingering until the 30-minute zombie checker. The reaper queues
4204            // an `RST_STREAM(CANCEL)`; because `has_pending_write()` does NOT
4205            // observe the `H2ControlTx` queue (it gates connection close, so a
4206            // queued RST must not read as "keep open"), set `should_write` via
4207            // the dedicated `has_pending_control_write()` probe so the reset is
4208            // actually flushed to the peer before the connection closes — without
4209            // it, a fully-silent peer's stalled stream is freed but the peer sees
4210            // only EOF, never the RST(CANCEL).
4211            if let Connection::H2(h2) = &mut self.frontend {
4212                h2.cancel_timed_out_streams(
4213                    &mut self.context,
4214                    &mut EndpointClient(&mut self.router),
4215                );
4216                if h2.core.has_pending_control_write() {
4217                    should_write = true;
4218                }
4219            }
4220            if self.frontend.has_pending_write() {
4221                should_write = true;
4222            }
4223            // Stream-specific on an H1 frontend only; see
4224            // `response_waits_on_client`.
4225            let front_pending_write = !front_is_h2 && self.frontend.has_pending_write();
4226            // Streams whose response waits on their backend, not on the
4227            // client: the frontend timeout leaves them linked, to their
4228            // backend's timeout (sozu-proxy/sozu#1906).
4229            let mut waiting_on_backend: Vec<GlobalStreamId> = Vec::new();
4230            let front_readiness = self.frontend.readiness_mut();
4231            for stream_id in 0..self.context.streams.len() {
4232                match self.context.streams[stream_id].state {
4233                    StreamState::Idle => {
4234                        // In h1 an Idle stream is always the first request, so we can send a 408
4235                        // In h2 an Idle stream doesn't necessarily hold a request yet,
4236                        // in most cases it was just reserved, so we can just ignore them.
4237                        if !front_is_h2 {
4238                            // Each stream answers out of the registry it
4239                            // captured when its request arrived; a listener
4240                            // reload since then belongs to the next one.
4241                            let answers_rc = self.context.streams[stream_id].answers.clone();
4242                            let answers = answers_rc.borrow();
4243                            let stream = &mut self.context.streams[stream_id];
4244                            stream.context.access_log_message = Some("client_timeout");
4245                            set_default_answer(stream, front_readiness, 408, &answers);
4246                            should_write = true;
4247                        }
4248                    }
4249                    StreamState::Link => {
4250                        // This is an unusual case, as we have both a complete request and no
4251                        // available backend yet. For now, we answer with 503.
4252                        // Not a timeout-driven outcome from the operator's
4253                        // perspective — leave access_log_message as None.
4254                        let answers_rc = self.context.streams[stream_id].answers.clone();
4255                        let answers = answers_rc.borrow();
4256                        let stream = &mut self.context.streams[stream_id];
4257                        set_default_answer(stream, front_readiness, 503, &answers);
4258                        should_write = true;
4259                    }
4260                    StreamState::Linked(_) => {
4261                        // The frontend timed out while a stream is linked to a backend.
4262                        // The backend timeout should handle this, but in case the backend
4263                        // is also stalled, send a 504 and terminate the stream.
4264                        if !self.context.streams[stream_id].back.consumed {
4265                            self.context.unlink_stream(stream_id);
4266                            let answers_rc = self.context.streams[stream_id].answers.clone();
4267                            let answers = answers_rc.borrow();
4268                            let stream = &mut self.context.streams[stream_id];
4269                            stream.context.access_log_message =
4270                                Some("client_timeout_during_response");
4271                            set_default_answer(stream, front_readiness, 504, &answers);
4272                            should_write = true;
4273                        } else if self.context.streams[stream_id].back.is_completed()
4274                            && (self.context.streams[stream_id].back.is_terminated()
4275                                || self.context.streams[stream_id].back.is_error())
4276                        {
4277                            // Response fully proxied, stream can be closed.
4278                            // kawa's `is_completed` only says nothing is queued:
4279                            // without the terminated/error half, a response
4280                            // still waiting on its backend took this arm and
4281                            // was cut, silently (sozu-proxy/sozu#1906).
4282                        } else if self.context.streams[stream_id].back.is_terminated()
4283                            || self.context.streams[stream_id].back.is_error()
4284                        {
4285                            // Response is terminated/error but not fully written to frontend.
4286                            // Keep the session alive briefly to flush remaining data.
4287                            should_close = false;
4288                        } else if !response_waits_on_client(
4289                            &self.context.streams[stream_id],
4290                            front_pending_write,
4291                        ) {
4292                            // The client has taken every byte so far: the
4293                            // response waits on its backend, whose own
4294                            // timeout bounds that wait. Cutting it here
4295                            // charged a slow backend to the client.
4296                            trace!(
4297                                "{} frontend timeout while stream {} waits on its backend, deferring to back_timeout",
4298                                log_module_context!(),
4299                                stream_id
4300                            );
4301                            waiting_on_backend.push(stream_id);
4302                            should_close = false;
4303                        } else {
4304                            // Partial response in progress — forcefully terminate
4305                            self.context.unlink_stream(stream_id);
4306                            let stream = &mut self.context.streams[stream_id];
4307                            stream.context.access_log_message =
4308                                Some("client_timeout_during_response");
4309                            forcefully_terminate_answer(
4310                                stream,
4311                                front_readiness,
4312                                H2Error::InternalError,
4313                            );
4314                            should_write = true;
4315                        }
4316                        // end_stream is called in a second pass below to avoid
4317                        // borrow conflicts on context.streams.
4318                    }
4319                    StreamState::Unlinked => {
4320                        // A stream Unlinked already has a response and its backend closed.
4321                        // In case it hasn't finished proxying we wait. Otherwise it is a stream
4322                        // kept alive for a new request, which can be killed.
4323                        if !self.context.streams[stream_id].back.is_completed() {
4324                            should_close = false;
4325                        }
4326                    }
4327                    StreamState::Recycle => {
4328                        // A recycled stream is an h2 stream which doesn't hold a request anymore.
4329                        // We can ignore it.
4330                    }
4331                }
4332            }
4333            // Second pass: end streams that were linked to backends.
4334            // This is done separately to avoid borrow conflicts on context.streams.
4335            let linked_streams: Vec<(GlobalStreamId, Token)> = self
4336                .context
4337                .streams
4338                .iter()
4339                .enumerate()
4340                .filter_map(|(id, stream)| {
4341                    if let StreamState::Linked(back_token) = stream.state
4342                        && !waiting_on_backend.contains(&id)
4343                    {
4344                        Some((id, back_token))
4345                    } else {
4346                        None
4347                    }
4348                })
4349                .collect();
4350            for (stream_id, back_token) in linked_streams {
4351                if let Some(backend) = self.router.backends.get_mut(&back_token) {
4352                    #[cfg(debug_assertions)]
4353                    debug_assert_h1_owns_stream(backend, stream_id);
4354                    backend.end_stream(stream_id, &mut self.context);
4355                }
4356            }
4357        } else if let Some(backend) = self.router.backends.get_mut(&token) {
4358            // Captured before the backend borrow so the re-arm below can reach
4359            // the pass's clock snapshot.
4360            let now = self.context.now;
4361            trace!(
4362                "{} MuxState::timeout_backend({:#?})",
4363                log_context_lite!(self),
4364                backend
4365            );
4366            // A dial still `Connecting` when its timer fires is the
4367            // `connect_timeout` armed by `Mux::dial_backend`: the backend
4368            // accepted nothing, so no byte of any request reached it. That is
4369            // a backend connect failure exactly like a refused dial, and it
4370            // takes the same path: flag the connection dead as the kernel
4371            // flags a refused one, and let `ready_inner`'s dead-backend branch
4372            // count the failure (`failures`, `retry_policy`, `backend.down`),
4373            // close the connection and re-queue its streams for another
4374            // backend through `end_stream_decision`'s `Reconnect`. Answering
4375            // 504 here left the failure uncounted, so the load balancer kept
4376            // choosing the dead backend (sozu-proxy/sozu#1800).
4377            //
4378            // The pass that does this runs right after this timeout:
4379            // `Mux::has_pending_backend_failure` reports the flag and
4380            // `Server::timeout` calls `ready` on it (`ProxySession::
4381            // needs_ready_pass`). Nothing is re-armed for this backend: that
4382            // pass removes it.
4383            if matches!(
4384                backend.position(),
4385                Position::Client(_, _, BackendStatus::Connecting(_))
4386            ) {
4387                debug!(
4388                    "{} backend connect timeout on {:?}, failing over",
4389                    log_context_lite!(self),
4390                    token
4391                );
4392                backend.readiness_mut().event.insert(Ready::ERROR);
4393                return StateResult::Continue;
4394            }
4395            // Stream-specific on an H1 frontend only; see
4396            // `response_waits_on_client`.
4397            let front_pending_write = !front_is_h2 && self.frontend.has_pending_write();
4398            // Set when a stream's response waits on the client: the backend
4399            // timeout leaves it linked, to the frontend's timeout
4400            // (sozu-proxy/sozu#1906).
4401            let mut waiting_on_client = false;
4402            let front_readiness = self.frontend.readiness_mut();
4403            let linked_ids: Vec<GlobalStreamId> = self
4404                .context
4405                .backend_streams
4406                .get(&token)
4407                .map_or_else(Vec::new, |ids| ids.to_vec());
4408            for stream_id in linked_ids {
4409                // This stream is linked to the backend that timedout
4410                if self.context.streams[stream_id].back.is_terminated()
4411                    || self.context.streams[stream_id].back.is_error()
4412                {
4413                    trace!(
4414                        "{} Stream terminated or in error, do nothing, just wait a bit more",
4415                        log_module_context!()
4416                    );
4417                    // Nothing to do, simply wait for the remaining bytes to be proxied
4418                    if !self.context.streams[stream_id].back.is_completed() {
4419                        should_close = false;
4420                    }
4421                } else if !self.context.streams[stream_id].back.consumed {
4422                    // The response has not started yet
4423                    trace!(
4424                        "{} Stream still waiting for response, send 504",
4425                        log_module_context!()
4426                    );
4427                    self.context.unlink_stream(stream_id);
4428                    let answers_rc = self.context.streams[stream_id].answers.clone();
4429                    let answers = answers_rc.borrow();
4430                    let stream = &mut self.context.streams[stream_id];
4431                    stream.context.access_log_message = Some("backend_timeout");
4432                    set_default_answer(stream, front_readiness, 504, &answers);
4433                    should_write = true;
4434                } else if response_waits_on_client(
4435                    &self.context.streams[stream_id],
4436                    front_pending_write,
4437                ) {
4438                    // Sōzu holds response bytes the client has not taken, so
4439                    // it stopped reading this backend itself: the backend is
4440                    // not inactive, the client is slow. The frontend timeout
4441                    // bounds that; cutting here truncated a large download to
4442                    // a client that paused longer than `back_timeout`.
4443                    trace!(
4444                        "{} backend timeout while stream {} waits on the client, deferring to front_timeout",
4445                        log_module_context!(),
4446                        stream_id
4447                    );
4448                    waiting_on_client = true;
4449                    should_close = false;
4450                    continue;
4451                } else {
4452                    trace!(
4453                        "{} Stream waiting for end of response, forcefully terminate it",
4454                        log_module_context!()
4455                    );
4456                    self.context.unlink_stream(stream_id);
4457                    let stream = &mut self.context.streams[stream_id];
4458                    stream.context.access_log_message = Some("backend_response_timeout");
4459                    forcefully_terminate_answer(stream, front_readiness, H2Error::InternalError);
4460                    should_write = true;
4461                }
4462                #[cfg(debug_assertions)]
4463                debug_assert_h1_owns_stream(backend, stream_id);
4464                backend.end_stream(stream_id, &mut self.context);
4465            }
4466            // Re-arm the backend timeout if the session stays alive (draining streams).
4467            // Without this, the timeout is consumed and the session becomes immortal
4468            // until the zombie checker runs.
4469            //
4470            // The `!should_close` condition looks like a hole, and is not. On
4471            // `should_close`, control falls through to the shared tail below,
4472            // which can still return `Continue` — through the `should_write`
4473            // writable loop, or through `delay_close_for_frontend_flush` —
4474            // having re-armed only the FRONTEND container. That would be a
4475            // lost wakeup if this backend could still be carrying work with no
4476            // timer of its own. It cannot, and the chain is four links, all in
4477            // this file and `connection.rs`/`h1.rs`/`h2.rs`:
4478            //
4479            // 1. the loop above iterates `context.backend_streams[&token]` —
4480            //    every stream currently linked to this backend, and nothing
4481            //    else — and calls `Connection::end_stream` on EVERY iteration,
4482            //    outside the if/else chain, so no arm can skip it;
4483            // 2. `ConnectionH2::end_stream` (`h2.rs`) starts with
4484            //    `context.unlink_stream(...)` unconditionally.
4485            //    `ConnectionH1::end_stream` (`h1.rs`) does NOT: it first guards
4486            //    `if self.stream != Some(stream) { error!(..); return; }` and
4487            //    that early return skips the unlink. The two are dispatched by
4488            //    `Connection::end_stream` in `connection.rs`;
4489            // 3. the H1 guard cannot reject a stream this loop passes it,
4490            //    because an H1 connection carries at most one linked stream and
4491            //    it is exactly `self.stream`. That holds because
4492            //    `self.stream = Some(..)` occurs only in
4493            //    `ConnectionH1::start_stream`, paired with
4494            //    `Context::link_stream`, and `self.stream = None` occurs only
4495            //    inside `end_stream` itself, after the unlink has run. So while
4496            //    `backend_streams[&token]` names a stream, `self.stream` names
4497            //    the same one — and `debug_assert_h1_owns_stream`, called
4498            //    just above every `end_stream` in this function, is the
4499            //    tripwire that fires if it ever stops being true;
4500            // 4. so by the time the tail runs, this backend has no linked
4501            //    stream left. The index-consistency `assert_eq!` at the end of
4502            //    `Mux::ready` pins the other direction (`backend_streams` is
4503            //    exactly the set of `StreamState::Linked` streams).
4504            //
4505            // `Context::unlink_stream` is the eviction point this path relies
4506            // on, not the only one in the module: `remove_backend_stream` also
4507            // has direct callers in `h1.rs` and `h2.rs`. Extra eviction can
4508            // only strengthen the conclusion, but the premise is "this loop
4509            // evicts", not "nothing else can".
4510            //
4511            // Work attached to this backend AFTERWARDS does not depend on
4512            // this site either, and since the adapter landed it no longer even
4513            // depends on the connection doing I/O: a pooled backend stays in
4514            // `router.backends`, so `Mux::reschedule` keeps its handle armed at
4515            // whatever deadline the core holds, every pass. (`arm_timeout()` at
4516            // the top of `ConnectionH1::{readable,writable}` and in
4517            // `ConnectionH2::{handle_write,handle_headers_frame}` pushes that
4518            // deadline out on the first pass that moves a stream's bytes — not
4519            // on every pass, LIFECYCLE §9 invariant 9 — but nothing hangs on
4520            // it.) That is what covers the pool-reuse branch of
4521            // `Router::plan_connect` never arming a timeout, unlike the fresh-dial
4522            // branch.
4523            //
4524            // The assertion below is the cheap tripwire for link 3: if the
4525            // impossible becomes possible (a new arm that skips `end_stream`,
4526            // or an `end_stream` that stops unlinking), it fires here rather
4527            // than as an immortal backend in production.
4528            #[cfg(debug_assertions)]
4529            debug_assert!(
4530                waiting_on_client
4531                    || self
4532                        .context
4533                        .backend_streams
4534                        .get(&token)
4535                        .is_none_or(|ids| ids.is_empty()),
4536                "backend {token:?} still holds linked streams {:?} after its \
4537                 timeout reaped them: on the should_close path nothing re-arms \
4538                 its container, so those streams would have no backend timer",
4539                self.context.backend_streams.get(&token),
4540            );
4541            if !should_close {
4542                backend.arm_timeout(now);
4543            }
4544            // A deferral must not reach the shared tail, which re-arms the
4545            // FRONTEND timer: re-armed on every backend timeout, a client that
4546            // stopped reading would never time out at all.
4547            if waiting_on_client && !should_write {
4548                return StateResult::Continue;
4549            }
4550        } else {
4551            // Session received a timeout for an unknown token, ignore it
4552            return StateResult::Continue;
4553        }
4554        if should_write {
4555            // Drain as much pending data as possible before closing.
4556            // A single writable() call is insufficient for large responses —
4557            // the TLS buffer may need multiple flushes. Without this loop,
4558            // the session is killed with unflushed TLS data, causing the
4559            // client to receive a truncated TLS record ("decode error").
4560            //
4561            // The constant 16 is empirical: it papers over a missing
4562            // invariant-15 hop in the H2 mux state machine where the
4563            // writable readiness signal is not always re-armed after a
4564            // partial flush. Long-term plan: reach invariant-15 closure
4565            // and remove this loop. See `lib/src/protocol/mux/LIFECYCLE.md`.
4566            let mut result = StateResult::Continue;
4567            for _ in 0..16 {
4568                result = match self
4569                    .frontend
4570                    .writable(&mut self.context, EndpointClient(&mut self.router))
4571                {
4572                    MuxResult::Continue => StateResult::Continue,
4573                    MuxResult::Upgrade => StateResult::Upgrade,
4574                    MuxResult::CloseSession => StateResult::CloseSession,
4575                };
4576                if result != StateResult::Continue
4577                    || !self.frontend.readiness_mut().interest.is_writable()
4578                {
4579                    break;
4580                }
4581            }
4582            // Re-arm the frontend timeout so the session doesn't become immortal.
4583            // The writable call may have partially flushed the response — we need
4584            // the timeout to fire again if the flush stalls.
4585            if result == StateResult::Continue {
4586                self.frontend.arm_timeout(self.context.now);
4587            }
4588            return result;
4589        }
4590        if should_close {
4591            if self.delay_close_for_frontend_flush("timeout") {
4592                debug!(
4593                    "{} Mux timeout delaying close for frontend flush: token={:?}, frontend={:?}",
4594                    log_context!(self),
4595                    token,
4596                    self.frontend
4597                );
4598                self.frontend.arm_timeout(self.context.now);
4599                return StateResult::Continue;
4600            }
4601            if front_is_h2 {
4602                debug!(
4603                    "{} Mux timeout returning CloseSession: token={:?}, frontend={:?}",
4604                    log_context!(self),
4605                    token,
4606                    self.frontend
4607                );
4608                for (idx, stream) in self.context.streams.iter().enumerate() {
4609                    if stream.state != StreamState::Recycle {
4610                        debug!(
4611                            "{}   timeout stream[{}]: state={:?}, front_phase={:?}, back_phase={:?}, front_completed={}, back_completed={}",
4612                            log_context!(self),
4613                            idx,
4614                            stream.state,
4615                            stream.front.parsing_phase,
4616                            stream.back.parsing_phase,
4617                            stream.front.is_completed(),
4618                            stream.back.is_completed()
4619                        );
4620                    }
4621                }
4622            }
4623            StateResult::CloseSession
4624        } else {
4625            // Re-arm the frontend timeout. Without this, the timeout is consumed
4626            // by triggered() and the session stays alive indefinitely until the
4627            // zombie checker runs (default: 30 minutes).
4628            self.frontend.arm_timeout(self.context.now);
4629            StateResult::Continue
4630        }
4631    }
4632
4633    fn shutting_down_inner(&mut self) -> SessionIsToBeClosed {
4634        // `shut_down_sessions()` drives this outside `ready()`, so it is its
4635        // own sampling point, and it is load-bearing rather than belt-and-braces.
4636        // The chain is: this line refreshes `context.now`;
4637        // `drive_frontend_shutdown_io` below always reaches `readable()` for an
4638        // H2 frontend (`force_h2_read` is unconditionally true, so the
4639        // early return above it cannot fire), and `readable` mirrors
4640        // `context.now` into `ConnectionH2::now`; the forced-close check that
4641        // follows then reads a fresh snapshot. Drop this line and a silent
4642        // draining session propagates the snapshot of its last `ready()` pass
4643        // forever, so the graceful-shutdown budget never expires. Pinned by
4644        // `shutting_down_refreshes_the_snapshot_so_the_drain_budget_expires`.
4645        let now = Instant::now();
4646        self.context.now = now;
4647        self.context.now_wall_ms = unix_epoch_ms();
4648        // RFC 9113 §6.8: initiate graceful shutdown with double-GOAWAY pattern.
4649        // Only send the initial GOAWAY once. The final GOAWAY (with the real
4650        // last_stream_id) is handled by finalize_write() when all streams drain.
4651        // Calling graceful_goaway() again would send the final GOAWAY
4652        // prematurely and force-disconnect before in-flight streams complete.
4653        if !self.frontend.is_draining() {
4654            match self.frontend.graceful_goaway(now) {
4655                MuxResult::CloseSession => return true,
4656                MuxResult::Continue => {
4657                    // graceful_goaway() queued a GOAWAY frame. Flush it directly
4658                    // since the event loop uses edge-triggered epoll and won't
4659                    // deliver a new WRITABLE event for an already-writable socket.
4660                    self.frontend.flush_output_buffer();
4661                }
4662                _ => {}
4663            }
4664        } else {
4665            trace!(
4666                "{} shutting_down: already draining, skipping duplicate GOAWAY",
4667                log_context!(self)
4668            );
4669            // A peer GOAWAY drains without arming the graceful-shutdown
4670            // budget, which only `graceful_goaway` arms: arm it here, once,
4671            // so the forced-close check below still bounds this session.
4672            self.frontend.arm_graceful_shutdown_deadline(now);
4673            // shut_down_sessions() runs outside ready(), so retry flushing any
4674            // previously-buffered GOAWAY/TLS records on each pass.
4675            self.frontend.flush_output_buffer();
4676        }
4677        if self.drive_frontend_shutdown_io() {
4678            return true;
4679        }
4680        // Forced-close deadline: once the H2 listener's
4681        // `h2_graceful_shutdown_deadline_seconds` budget has elapsed from
4682        // the moment `graceful_goaway` armed `drain.started_at`, stop
4683        // waiting for streams and tear the session down. `drive_frontend_
4684        // shutdown_io` above already had a chance to flush any pending
4685        // TLS/GOAWAY records; this branch accepts that some bytes may be
4686        // lost in exchange for honoring the operator-configured SLA.
4687        // Listeners that disable the knob (`= 0` → `None`) short-circuit
4688        // the check inside `graceful_shutdown_deadline_elapsed`.
4689        //
4690        // RFC 9113 §6.8: the close is still preceded by the final GOAWAY, so
4691        // the peer learns which streams were processed; the advisory one at
4692        // `2^31-1` says nothing about it (sozu-proxy/sozu#1647). It is flushed
4693        // once, best effort, like the advisory GOAWAY above.
4694        if self.frontend.graceful_shutdown_deadline_elapsed() {
4695            debug!(
4696                "{} Mux shutting_down: graceful-shutdown deadline elapsed, forcing close",
4697                log_context!(self)
4698            );
4699            // A lingering connection already sent its final GOAWAY and shut
4700            // its write side down: the operator's budget closes it as is.
4701            if !self.frontend.is_lingering() {
4702                let _ = self.frontend.goaway_before_forced_close(&self.context);
4703                self.frontend.flush_output_buffer();
4704            }
4705            return true;
4706        }
4707        if self.frontend.is_lingering() {
4708            // Bounded by its own deadline (`request_timeout` from the
4709            // linger's start), which the timer enforces. For H2 also by the
4710            // graceful-shutdown budget just above, once that budget is armed:
4711            // `graceful_goaway` arms it when this soft stop sends the first
4712            // GOAWAY, and the lingering passes keep the connection's clock
4713            // fresh so it can elapse. A drain the client's own GOAWAY started
4714            // never armed it, so such a linger runs to its own deadline.
4715            return false;
4716        }
4717        if matches!(self.frontend, Connection::H2(_)) && self.frontend.is_draining() {
4718            for stream in &mut self.context.streams {
4719                if stream.front_received_end_of_stream {
4720                    continue;
4721                }
4722                if !matches!(stream.state, StreamState::Linked(_) | StreamState::Unlinked) {
4723                    continue;
4724                }
4725                if stream.front.consumed
4726                    && stream.front.storage.is_empty()
4727                    && stream.front.is_completed()
4728                {
4729                    stream.front_received_end_of_stream = true;
4730                    self.frontend
4731                        .readiness_mut()
4732                        .interest
4733                        .insert(Ready::WRITABLE);
4734                    self.frontend.readiness_mut().signal_pending_write();
4735                }
4736            }
4737        }
4738        // An H2 request that has not reached a backend yet still holds the
4739        // session, bounded by the forced-close deadline above: a field block
4740        // still being reassembled (its stream is `StreamState::Idle`), and a
4741        // complete one awaiting `Mux::ready`'s link pass (`StreamState::Link`),
4742        // which is where a CONTINUATION read by `drive_frontend_shutdown_io`
4743        // leaves it. Closing on either loses a request the advisory GOAWAY
4744        // invited, and skips the final GOAWAY `ConnectionH2::finalize_write`
4745        // sends once the stream is done (sozu-proxy/sozu#1647).
4746        let front_is_h2 = matches!(self.frontend, Connection::H2(_));
4747        let mut can_stop = !self.frontend.peer_header_block_in_progress();
4748        for stream in &mut self.context.streams {
4749            match stream.state {
4750                StreamState::Linked(_) => {
4751                    // An HTTP/1.1 response still to come announces the close
4752                    // (`Connection: close`, in `HttpContext::on_response_headers`
4753                    // of `lib/src/protocol/kawa_h1/editor.rs`), and the
4754                    // frontend closes once it is sent instead of waiting for
4755                    // the next pass. HTTP/2 announces it with the GOAWAY.
4756                    if !front_is_h2 {
4757                        stream.context.closing = true;
4758                    }
4759                    can_stop = false;
4760                }
4761                StreamState::Link if front_is_h2 => {
4762                    can_stop = false;
4763                }
4764                StreamState::Unlinked => {
4765                    kawa::debug_kawa(&stream.front);
4766                    kawa::debug_kawa(&stream.back);
4767                    if stream.is_quiesced() {
4768                        continue;
4769                    }
4770                    stream.context.closing = true;
4771                    can_stop = false;
4772                }
4773                _ => {}
4774            }
4775        }
4776        if self.frontend.has_pending_write() {
4777            return false;
4778        }
4779        if can_stop {
4780            let active_h2_streams = self
4781                .context
4782                .streams
4783                .iter()
4784                .enumerate()
4785                .filter(|(_, s)| {
4786                    if s.state == StreamState::Recycle {
4787                        return false;
4788                    }
4789                    if s.state == StreamState::Unlinked && s.is_quiesced() {
4790                        return false;
4791                    }
4792                    true
4793                })
4794                .collect::<Vec<_>>();
4795            if matches!(self.frontend, Connection::H2(_)) && !active_h2_streams.is_empty() {
4796                debug!(
4797                    "{} Mux shutting_down returning true with active H2 streams: {:?}",
4798                    log_context!(self),
4799                    self.frontend
4800                );
4801                for (idx, stream) in active_h2_streams {
4802                    debug!(
4803                        "{}   shutdown stream[{}]: state={:?}, front_phase={:?}, back_phase={:?}, front_completed={}, back_completed={}",
4804                        log_context!(self),
4805                        idx,
4806                        stream.state,
4807                        stream.front.parsing_phase,
4808                        stream.back.parsing_phase,
4809                        stream.front.is_completed(),
4810                        stream.back.is_completed()
4811                    );
4812                }
4813            }
4814        }
4815        can_stop
4816    }
4817}
4818
4819/// The embedder's [`router::BackendSelector`]: the worker's backend set and
4820/// this session's registry, lent together for one selection.
4821///
4822/// It is the one place a registry handle becomes an opaque [`BackendId`]: the
4823/// backend is chosen and reserved by the `BackendMap`, named by the session's
4824/// table, and the `Rc` goes no further. Everything past
4825/// [`router::BackendSelector::select`] — the dial in `Mux::dial_backend`, and
4826/// every `Position::Client` built from it — holds the id, never the handle.
4827struct RegistrySelector<'a> {
4828    backends: &'a mut BackendMap,
4829    registry: &'a mut BackendRegistry,
4830    /// The pass's clock sample, [`Context::now`]: selection judges backoff
4831    /// windows and connection-time decay against it rather than reading a
4832    /// clock (#1684).
4833    now: Instant,
4834    /// The backends the stream being linked already failed to connect to,
4835    /// [`Stream::tried_backends`]: selection prefers any other one
4836    /// (sozu-proxy/sozu#1800).
4837    exclude: &'a [std::net::SocketAddr],
4838}
4839
4840impl router::BackendSelector for RegistrySelector<'_> {
4841    fn select(
4842        &mut self,
4843        cluster_id: &str,
4844        affinity: router::Affinity<'_>,
4845        key: Option<u64>,
4846    ) -> Result<router::SelectedBackend, BackendError> {
4847        let handle = match affinity {
4848            router::Affinity::Sticky(Some(cookie)) => {
4849                self.backends.reserve_sticky_backend_excluding(
4850                    cluster_id,
4851                    cookie,
4852                    key,
4853                    self.now,
4854                    self.exclude,
4855                )?
4856            }
4857            router::Affinity::Sticky(None) | router::Affinity::Unpinned => self
4858                .backends
4859                .reserve_backend_excluding(cluster_id, key, self.now, self.exclude)?,
4860        };
4861        let sticky_session = match affinity {
4862            router::Affinity::Sticky(_) => {
4863                let chosen = handle.borrow();
4864                Some(
4865                    chosen
4866                        .sticky_id
4867                        .clone()
4868                        .unwrap_or_else(|| chosen.backend_id.to_owned()),
4869                )
4870            }
4871            router::Affinity::Unpinned => None,
4872        };
4873        let backend = self.registry.id_for(&handle);
4874        // The id names the handle the map just reserved, not a copy of it.
4875        debug_assert!(
4876            self.registry
4877                .get(backend.slot())
4878                .is_some_and(|known| Rc::ptr_eq(known, &handle)),
4879            "the minted id must resolve to the very backend that was reserved"
4880        );
4881        Ok(router::SelectedBackend {
4882            backend,
4883            sticky_session,
4884        })
4885    }
4886}
4887
4888/// Minimal listener + `Mux` scaffolding shared by the `mod.rs` and `h2.rs`
4889/// test modules. Lives here rather than in either test module because
4890/// `Context<L>` is generic over the listener and both need the same `L`.
4891#[cfg(test)]
4892pub(crate) mod test_support {
4893    use std::{
4894        cell::RefCell,
4895        collections::BTreeMap,
4896        net::{SocketAddr, TcpListener as StdTcpListener},
4897        rc::Rc,
4898    };
4899
4900    use sozu_command::logging::CachedTags;
4901
4902    use super::*;
4903    use crate::{
4904        FrontendFromRequestError, L7ListenerHandler, ListenerHandler, Protocol,
4905        protocol::http::{answers::HttpAnswers, parser::Method},
4906        router::RouteResult,
4907    };
4908
4909    /// Implements exactly the nine required methods of `ListenerHandler` +
4910    /// `L7ListenerHandler`; every other knob keeps its trait default.
4911    pub(crate) struct TestListener {
4912        address: SocketAddr,
4913        answers: Rc<RefCell<HttpAnswers>>,
4914        sticky_name: String,
4915        elide_x_real_ip: bool,
4916    }
4917
4918    pub(crate) const TEST_STICKY_NAME: &str = "SOZUBALANCEID";
4919
4920    impl TestListener {
4921        pub(crate) fn new() -> Self {
4922            Self {
4923                address: "127.0.0.1:1".parse().expect("test address must parse"),
4924                answers: test_answers(),
4925                sticky_name: TEST_STICKY_NAME.to_owned(),
4926                elide_x_real_ip: false,
4927            }
4928        }
4929
4930        /// Stand in for an operator listener reload: change the per-request
4931        /// knobs and publish a brand-new answer registry under a new `Rc`,
4932        /// exactly as `HttpListener::update_config` does. Returns the newly
4933        /// published registry.
4934        ///
4935        /// The registry must be *published*, not edited: a test that mutated
4936        /// the existing one in place would prove nothing about the per-stream
4937        /// capture, because both streams would still be holding the same
4938        /// object.
4939        pub(crate) fn reload(
4940            &mut self,
4941            sticky_name: &str,
4942            elide_x_real_ip: bool,
4943        ) -> Rc<RefCell<HttpAnswers>> {
4944            self.sticky_name = sticky_name.to_owned();
4945            self.elide_x_real_ip = elide_x_real_ip;
4946            self.answers = test_answers();
4947            self.answers.clone()
4948        }
4949    }
4950
4951    /// A fresh, empty answer registry — what a test `Stream` renders from
4952    /// when the test is not about templates.
4953    pub(crate) fn test_answers() -> Rc<RefCell<HttpAnswers>> {
4954        Rc::new(RefCell::new(
4955            HttpAnswers::new(&BTreeMap::new()).expect("default answers must build"),
4956        ))
4957    }
4958
4959    impl ListenerHandler for TestListener {
4960        fn get_addr(&self) -> &SocketAddr {
4961            &self.address
4962        }
4963        fn get_tags(&self, _key: &str) -> Option<&CachedTags> {
4964            None
4965        }
4966        fn set_tags(&mut self, _key: String, _tags: Option<BTreeMap<String, String>>) {}
4967        fn protocol(&self) -> Protocol {
4968            Protocol::HTTP
4969        }
4970        fn public_address(&self) -> SocketAddr {
4971            self.address
4972        }
4973    }
4974
4975    impl L7ListenerHandler for TestListener {
4976        fn get_sticky_name(&self) -> &str {
4977            &self.sticky_name
4978        }
4979        fn get_elide_x_real_ip(&self) -> bool {
4980            self.elide_x_real_ip
4981        }
4982        fn get_connect_timeout(&self) -> u32 {
4983            10
4984        }
4985        fn frontend_from_request(
4986            &self,
4987            _host: &str,
4988            _uri: &str,
4989            _method: &Method,
4990            _headers: &dyn crate::router::RequestHeaders,
4991        ) -> Result<RouteResult, FrontendFromRequestError> {
4992            Err(FrontendFromRequestError::InvalidCharsAfterHost(
4993                "test listener routes nothing".to_owned(),
4994            ))
4995        }
4996        fn get_answers(&self) -> &Rc<RefCell<HttpAnswers>> {
4997            &self.answers
4998        }
4999    }
5000
5001    /// Whether the epoll instance behind `epoll` watches descriptor `fd`,
5002    /// read from the `tfd:` lines Linux lists in `/proc/self/fdinfo`.
5003    ///
5004    /// Reading the kernel's own table, rather than counting calls, is what
5005    /// lets a test tell "never deregistered" from "removed by the close".
5006    #[cfg(target_os = "linux")]
5007    pub(crate) fn epoll_watches(epoll: &impl std::os::fd::AsRawFd, fd: std::os::fd::RawFd) -> bool {
5008        let fdinfo = std::fs::read_to_string(format!("/proc/self/fdinfo/{}", epoll.as_raw_fd()))
5009            .expect("the epoll instance must expose its fdinfo");
5010        fdinfo.lines().any(|line| {
5011            line.strip_prefix("tfd:")
5012                .and_then(|rest| rest.split_whitespace().next())
5013                .and_then(|target| target.parse::<std::os::fd::RawFd>().ok())
5014                == Some(fd)
5015        })
5016    }
5017
5018    /// A live, connected, non-blocking loopback socket. Reads return
5019    /// `WouldBlock` rather than `ECONNREFUSED`, so a `readable()` pass over it
5020    /// is a no-op instead of a forced disconnect. The accepted peer is returned
5021    /// so the caller can keep it alive for the duration of the test.
5022    pub(crate) fn connected_socket() -> (mio::net::TcpStream, std::net::TcpStream) {
5023        let listener = StdTcpListener::bind("127.0.0.1:0").expect("bind test listener");
5024        let address = listener.local_addr().expect("listener local addr");
5025        let client = mio::net::TcpStream::connect(address).expect("connect test client");
5026        let (peer, _) = listener.accept().expect("accept test peer");
5027        peer.set_nonblocking(true).expect("peer nonblocking");
5028        (client, peer)
5029    }
5030
5031    /// A `Context` backed by [`TestListener`] and a small buffer pool.
5032    pub(crate) fn test_context(pool: &Rc<RefCell<Pool>>) -> Context<TestListener> {
5033        Context::new(
5034            Ulid::generate(),
5035            Rc::downgrade(pool),
5036            Rc::new(RefCell::new(TestListener::new())),
5037            None,
5038            "127.0.0.1:1".parse().expect("public address must parse"),
5039        )
5040    }
5041}
5042
5043// Counts `writable()` calls made by `Mux::drive_frontend_shutdown_io`, so a
5044// test can pin how much work one shutdown pass does. Thread-local: each test
5045// runs on its own thread.
5046#[cfg(test)]
5047thread_local! {
5048    static SHUTDOWN_WRITABLE_PASSES: std::cell::Cell<u32> = const { std::cell::Cell::new(0) };
5049    /// Times `Mux::wait_for_client_at_loop_limit` ran on this thread, in
5050    /// every build (the debug history is empty in release builds).
5051    static LOOP_LIMIT_HITS: std::cell::Cell<u32> = const { std::cell::Cell::new(0) };
5052}
5053
5054#[cfg(test)]
5055mod tests {
5056    use std::collections::HashMap;
5057
5058    use super::test_support::{TEST_STICKY_NAME, connected_socket, test_context};
5059    use super::*;
5060    use crate::{
5061        pool::Pool,
5062        protocol::http::parser::Method,
5063        protocol::mux::{
5064            h2::{H2ConnectionConfig, H2State},
5065            h2_flood_detector::H2FloodConfig,
5066        },
5067        timer::TimeoutContainer,
5068    };
5069
5070    /// After its final GOAWAY, an H2 frontend closes while the client may
5071    /// still be sending frames. A close with those frames unread, or followed
5072    /// by more, makes the kernel reset the connection and discard the part of
5073    /// the response still queued in the frontend's socket. The connection
5074    /// must shut its write side down and drain instead (RFC 9112 §9.6, the
5075    /// same lingering close as H1), so the client reads all of it, then EOF.
5076    ///
5077    /// The test emulates the session close `HttpsSession::close` performs —
5078    /// `shutdown_write`, then `close(2)` — whenever `writable` asks for it.
5079    #[test]
5080    fn an_h2_frontend_lingers_after_its_final_goaway_so_the_client_reads_everything() {
5081        use std::io::{Read, Write};
5082
5083        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5084        let (socket, mut peer) = connected_socket();
5085        // A client that is slow to read: most of the response stays queued
5086        // in the frontend's socket.
5087        socket2::SockRef::from(&peer)
5088            .set_recv_buffer_size(32 * 1024)
5089            .expect("the client receive buffer must be settable");
5090
5091        let mut h2 = h2::H2Shell::new(
5092            Ulid::generate(),
5093            socket,
5094            Position::Server,
5095            &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5096            H2FloodConfig::default(),
5097            H2ConnectionConfig::default(),
5098            Duration::from_secs(30),
5099            None,
5100            Duration::from_secs(10),
5101            None,
5102            Ready::WRITABLE | Ready::HUP | Ready::ERROR,
5103        )
5104        .expect("a pool with free buffers must yield an H2 connection");
5105        // The final GOAWAY is out: the next writable pass closes.
5106        h2.core.state = H2State::GoAway;
5107        h2.core.drain.__test_set_draining();
5108        h2.core.graceful_goaway = true;
5109
5110        // The response, already handed to the kernel.
5111        let mut queued = 0;
5112        let chunk = [b'r'; 16 * 1024];
5113        loop {
5114            match (&h2.socket).write(&chunk) {
5115                Ok(size) => queued += size,
5116                Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => break,
5117                Err(e) => panic!("the response write failed: {e}"),
5118            }
5119            if queued >= 1024 * 1024 {
5120                break;
5121            }
5122        }
5123        // The client still sends frames after the GOAWAY.
5124        peer.write_all(&[0u8; 1024])
5125            .expect("the client may still send after the GOAWAY");
5126        std::thread::sleep(Duration::from_millis(20));
5127
5128        let mut frontend = Connection::H2(h2);
5129        let mut context = test_context(&pool);
5130        let mut router = Router::new(Duration::from_secs(30), Duration::from_secs(30));
5131        context.now = Instant::now();
5132        let result = frontend.writable(&mut context, EndpointClient(&mut router));
5133        let lingering = frontend.is_lingering();
5134        let mut frontend = Some(frontend);
5135        if matches!(result, MuxResult::CloseSession)
5136            && let Some(closed) = frontend.take()
5137        {
5138            let _ = shutdown_write(closed.socket(), false);
5139            drop(closed);
5140        }
5141
5142        peer.set_nonblocking(false)
5143            .expect("the client reads blocking");
5144        peer.set_read_timeout(Some(Duration::from_secs(5)))
5145            .expect("the client read is bounded");
5146        let mut received = 0;
5147        let mut buf = [0u8; 16 * 1024];
5148        let end = loop {
5149            match peer.read(&mut buf) {
5150                Ok(0) => break Ok(()),
5151                Ok(size) => received += size,
5152                Err(e) => break Err(e),
5153            }
5154        };
5155        assert_eq!(
5156            received, queued,
5157            "the client reads the whole response (end: {end:?})"
5158        );
5159        assert!(
5160            end.is_ok(),
5161            "the response ends with a clean EOF, got {end:?}"
5162        );
5163        assert!(
5164            lingering && matches!(result, MuxResult::Continue),
5165            "the frontend lingers instead of closing, got {result:?}"
5166        );
5167
5168        // The client closes: the drain meets its EOF and the session closes.
5169        let mut frontend = frontend.expect("a lingering frontend is kept");
5170        drop(peer);
5171        let mut after_client_close = MuxResult::Continue;
5172        for _ in 0..1000 {
5173            frontend.readiness_mut().event.insert(Ready::READABLE);
5174            after_client_close = frontend.readable(&mut context, EndpointClient(&mut router));
5175            if matches!(after_client_close, MuxResult::CloseSession) {
5176                break;
5177            }
5178            std::thread::sleep(Duration::from_millis(1));
5179        }
5180        assert!(
5181            matches!(after_client_close, MuxResult::CloseSession),
5182            "the client's EOF ends the drain, got {after_client_close:?}"
5183        );
5184    }
5185
5186    /// Start the lingering close of an H2 frontend that queued its final
5187    /// GOAWAY through `ConnectionH2::goaway` — `NO_ERROR` when `graceful`,
5188    /// `ENHANCE_YOUR_CALM` (a flood) otherwise — with
5189    /// a 10 s `request_timeout`; then have its client send `sent` bytes
5190    /// without closing, move the drain's clock `elapsed` past the linger's
5191    /// start, and drive the drain until it closes or 200 passes go by. Answers the linger the close started with, the instants that
5192    /// bracket it, and whether the drain closed.
5193    fn h2_linger_after_goaway(
5194        graceful: bool,
5195        sent: usize,
5196        elapsed: Duration,
5197    ) -> (shared::Linger, Instant, Instant, bool) {
5198        use std::io::Write;
5199
5200        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5201        let (socket, peer) = connected_socket();
5202        let mut h2 = h2::H2Shell::new(
5203            Ulid::generate(),
5204            socket,
5205            Position::Server,
5206            &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5207            H2FloodConfig::default(),
5208            H2ConnectionConfig::default(),
5209            Duration::from_secs(30),
5210            None,
5211            Duration::from_secs(10),
5212            None,
5213            Ready::WRITABLE | Ready::HUP | Ready::ERROR,
5214        )
5215        .expect("a pool with free buffers must yield an H2 connection");
5216        // The real entry point: `goaway` queues the final GOAWAY, records
5217        // its error code and asks for the write that flushes it.
5218        let queued = h2.core.goaway(if graceful {
5219            H2Error::NoError
5220        } else {
5221            H2Error::EnhanceYourCalm
5222        });
5223        assert!(
5224            matches!(queued, MuxResult::Continue),
5225            "the GOAWAY is queued"
5226        );
5227
5228        let mut frontend = Connection::H2(h2);
5229        let mut context = test_context(&pool);
5230        let mut router = Router::new(Duration::from_secs(30), Duration::from_secs(30));
5231        let before = Instant::now();
5232        context.now = before;
5233        let result = frontend.writable(&mut context, EndpointClient(&mut router));
5234        let after = Instant::now();
5235        assert!(
5236            frontend.is_lingering() && matches!(result, MuxResult::Continue),
5237            "premise: the frontend lingers after its final GOAWAY, got {result:?}"
5238        );
5239        let Connection::H2(h2) = &frontend else {
5240            unreachable!("built as H2")
5241        };
5242        let linger = h2.core.linger;
5243
5244        // The client keeps sending and never closes; it is kept open until
5245        // the drain has decided.
5246        peer.set_nonblocking(false)
5247            .expect("the client writes blocking");
5248        let mut writer = peer.try_clone().expect("the client socket clones");
5249        let sender = std::thread::spawn(move || {
5250            let _ = writer.write_all(&vec![0u8; sent]);
5251        });
5252        context.now = after + elapsed;
5253        let mut closed = false;
5254        for _ in 0..200 {
5255            frontend.readiness_mut().event.insert(Ready::READABLE);
5256            if matches!(
5257                frontend.readable(&mut context, EndpointClient(&mut router)),
5258                MuxResult::CloseSession
5259            ) {
5260                closed = true;
5261                break;
5262            }
5263            std::thread::sleep(Duration::from_millis(1));
5264        }
5265        drop(frontend);
5266        let _ = sender.join();
5267        drop(peer);
5268        (linger, before, after, closed)
5269    }
5270
5271    /// The lingering close after a GOAWAY carrying an error code exists only
5272    /// to deliver that GOAWAY (sozu-proxy/sozu#1861): its deadline is
5273    /// `shared::ERROR_GOAWAY_LINGER_TIMEOUT` from its start and its byte
5274    /// budget `shared::ERROR_GOAWAY_LINGER_MAX_BYTES`, so a client that keeps
5275    /// sending is cut off once that much is read. A graceful GOAWAY keeps the
5276    /// listener's `request_timeout` and `shared::LINGER_MAX_BYTES`, and the
5277    /// same traffic does not close it. With no traffic at all, the short
5278    /// deadline closes the error linger and leaves the graceful one open.
5279    ///
5280    /// TO SEE THIS RED: in `ConnectionH2::linger_budget` (`h2.rs`), return
5281    /// the graceful pair for every GOAWAY.
5282    #[test]
5283    fn an_error_goaway_lingers_under_the_short_budget_and_a_graceful_one_keeps_its_own() {
5284        let sent = shared::ERROR_GOAWAY_LINGER_MAX_BYTES + 64 * 1024;
5285
5286        let (linger, before, after, closed) = h2_linger_after_goaway(false, sent, Duration::ZERO);
5287        let shared::Linger::Draining {
5288            deadline,
5289            remaining,
5290        } = linger
5291        else {
5292            panic!("an error GOAWAY drains at once on a plain socket, got {linger:?}");
5293        };
5294        assert_eq!(remaining, shared::ERROR_GOAWAY_LINGER_MAX_BYTES);
5295        assert!(
5296            deadline >= before + shared::ERROR_GOAWAY_LINGER_TIMEOUT
5297                && deadline <= after + shared::ERROR_GOAWAY_LINGER_TIMEOUT,
5298            "the error linger's deadline is the short one, {:?} past its start",
5299            deadline.saturating_duration_since(before)
5300        );
5301        assert!(
5302            closed,
5303            "the byte budget ends the error linger while the client still sends"
5304        );
5305
5306        let (linger, before, after, closed) = h2_linger_after_goaway(true, sent, Duration::ZERO);
5307        let shared::Linger::Draining {
5308            deadline,
5309            remaining,
5310        } = linger
5311        else {
5312            panic!("a graceful GOAWAY drains at once on a plain socket, got {linger:?}");
5313        };
5314        assert_eq!(remaining, shared::LINGER_MAX_BYTES);
5315        assert!(
5316            deadline >= before + Duration::from_secs(10)
5317                && deadline <= after + Duration::from_secs(10),
5318            "the graceful linger keeps request_timeout, {:?} past its start",
5319            deadline.saturating_duration_since(before)
5320        );
5321        assert!(
5322            !closed,
5323            "a graceful linger keeps draining a client that is still sending"
5324        );
5325
5326        let (.., closed) = h2_linger_after_goaway(false, 0, shared::ERROR_GOAWAY_LINGER_TIMEOUT);
5327        assert!(closed, "the short deadline ends a silent error linger");
5328        let (.., closed) = h2_linger_after_goaway(true, 0, shared::ERROR_GOAWAY_LINGER_TIMEOUT);
5329        assert!(!closed, "the short deadline does not end a graceful linger");
5330    }
5331
5332    /// Run one `Mux::ready` pass over an H2 frontend that queued its final
5333    /// GOAWAY (`NO_ERROR` when `graceful`, `ENHANCE_YOUR_CALM` otherwise)
5334    /// while one stream is still linked to a backend. Answers the `Mux`
5335    /// after the pass and the backend's peer socket.
5336    fn mux_after_final_goaway_with_a_linked_stream(
5337        pool: &Rc<RefCell<Pool>>,
5338        graceful: bool,
5339    ) -> (
5340        Mux<mio::net::TcpStream, test_support::TestListener>,
5341        std::net::TcpStream,
5342        std::net::TcpStream,
5343    ) {
5344        let (socket, front_peer) = connected_socket();
5345        let mut h2 = h2::H2Shell::new(
5346            Ulid::generate(),
5347            socket,
5348            Position::Server,
5349            &mut PoolBufferSource::new(Rc::downgrade(pool)),
5350            H2FloodConfig::default(),
5351            H2ConnectionConfig::default(),
5352            Duration::from_secs(30),
5353            None,
5354            Duration::from_secs(10),
5355            None,
5356            Ready::WRITABLE | Ready::HUP | Ready::ERROR,
5357        )
5358        .expect("a pool with free buffers must yield an H2 connection");
5359        let queued = h2.core.goaway(if graceful {
5360            H2Error::NoError
5361        } else {
5362            H2Error::EnhanceYourCalm
5363        });
5364        assert!(
5365            matches!(queued, MuxResult::Continue),
5366            "the GOAWAY is queued"
5367        );
5368
5369        let mut context = test_context(pool);
5370        context
5371            .create_stream(Ulid::generate(), 1 << 16)
5372            .expect("test context must create a stream");
5373        let mut mux = Mux {
5374            configured_frontend_timeout: Duration::from_secs(10),
5375            frontend_token: Token(0),
5376            frontend: Connection::H2(h2),
5377            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5378            context,
5379            session_ulid: Ulid::generate(),
5380            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(10))),
5381            backend_registry: BackendRegistry::default(),
5382            backends: Rc::default(),
5383        };
5384        let (backend, back_peer) = test_backend_connection(&mut mux, Duration::from_secs(30));
5385        mux.router.backends.insert(Token(1), backend);
5386        mux.context.link_stream(0, Token(1));
5387        mux.timeouts.backends.get_or_insert_with(Token(1), || {
5388            TimeoutContainer::new_empty(Duration::from_secs(30))
5389        });
5390
5391        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
5392        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
5393        let mut metrics = SessionMetrics::new(None);
5394        let result = mux.ready(session, proxy, &mut metrics);
5395        assert!(
5396            matches!(result, SessionResult::Continue) && mux.frontend.is_lingering(),
5397            "premise: the pass that flushes the final GOAWAY starts the linger, got {result:?}"
5398        );
5399        (mux, front_peer, back_peer)
5400    }
5401
5402    /// A final GOAWAY carrying an error code can leave streams open, and none
5403    /// of them will be answered: the pass that starts its linger logs and
5404    /// recycles them and shuts down, accounts for and drops their backend
5405    /// connections, instead of holding them until the linger ends. A graceful
5406    /// linger is left as it was.
5407    ///
5408    /// TO SEE THIS RED: remove both calls to
5409    /// `Mux::release_streams_and_backends_for_error_linger` from
5410    /// `Mux::ready_inner`.
5411    #[test]
5412    fn an_error_goaway_linger_releases_its_streams_and_backends_when_it_starts() {
5413        use std::io::Read;
5414
5415        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
5416        let (mux, _front_peer, mut back_peer) =
5417            mux_after_final_goaway_with_a_linked_stream(&pool, false);
5418        assert!(
5419            mux.router.backends.is_empty(),
5420            "the backend connection is released when the error linger starts"
5421        );
5422        assert!(
5423            mux.context.backend_streams.is_empty() && mux.timeouts.backends.is_empty(),
5424            "no reverse-index entry or backend timer survives the release"
5425        );
5426        assert_eq!(
5427            mux.context.streams[0].state,
5428            StreamState::Recycle,
5429            "the stream that will get no answer is logged and recycled"
5430        );
5431        back_peer
5432            .set_nonblocking(false)
5433            .expect("the backend peer reads blocking");
5434        back_peer
5435            .set_read_timeout(Some(Duration::from_secs(5)))
5436            .expect("the backend peer read is bounded");
5437        let mut buf = [0u8; 64];
5438        assert_eq!(
5439            back_peer.read(&mut buf).ok(),
5440            Some(0),
5441            "the backend socket is closed: its peer reads EOF"
5442        );
5443
5444        let (mux, _front_peer, _back_peer) =
5445            mux_after_final_goaway_with_a_linked_stream(&pool, true);
5446        assert_eq!(
5447            mux.router.backends.len(),
5448            1,
5449            "a graceful linger keeps its backend, as before"
5450        );
5451        assert_eq!(mux.context.streams[0].state, StreamState::Linked(Token(1)));
5452    }
5453
5454    /// A TLS frontend whose final GOAWAY carries an error code and leaves a
5455    /// stream open sends that GOAWAY without `close_notify` behind it
5456    /// (`ConnectionH2::output_flush_closes_connection` wants an empty stream
5457    /// table). The close its next writable pass decides queues the alert and
5458    /// turns into a pending linger (`H2Shell::linger_instead_of_closing`);
5459    /// when the kernel then refuses the alert's record, nothing re-arms the
5460    /// write and `Mux::ready_inner`'s loop ends with the linger still
5461    /// pending. The release must already have run by then: a stream queued
5462    /// in `pending_links` by the same pass is never routed nor dialed.
5463    ///
5464    /// TO SEE THIS RED: gate `Connection::lingers_after_error_goaway` on
5465    /// `is_lingering()` alone, as before. The `pending_links` pass then
5466    /// reaches the routing view for the stream and panics in
5467    /// `NoRoutingProxy` ("pending_links routed a stream of an error-GOAWAY
5468    /// linger").
5469    #[test]
5470    fn an_error_goaway_linger_pending_on_close_notify_releases_before_pending_links() {
5471        use std::io::Write;
5472
5473        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
5474        let (mut h2, _peer, _client) = h2::tests::rustls_h2_connection(&pool, H2State::Header);
5475        let mut context = test_context(&pool);
5476        let stream_id = context
5477            .create_stream(Ulid::generate(), 1 << 16)
5478            .expect("test context must create a stream");
5479        // A request the same read pass parsed whole and queued for a backend.
5480        context.streams[stream_id].state = StreamState::Link;
5481        context.pending_links.push_back(stream_id);
5482        h2.core.__test_insert_wire_mapping_only(1, stream_id);
5483
5484        assert!(
5485            matches!(
5486                h2.core.goaway(H2Error::EnhanceYourCalm),
5487                MuxResult::Continue
5488            ),
5489            "the GOAWAY is queued"
5490        );
5491        h2.flush_output_buffer();
5492        assert!(
5493            h2.core.output_pending().is_empty() && !h2.socket.socket_wants_write(),
5494            "premise: the GOAWAY reached the kernel, with no close_notify behind it"
5495        );
5496        // The client stops reading: fill the frontend's send queue so the
5497        // kernel refuses the close_notify record.
5498        let junk = [0u8; 4096];
5499        let mut filled = false;
5500        for _ in 0..4096 {
5501            match h2.socket.stream.write(&junk) {
5502                Ok(_) => {}
5503                Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => {
5504                    filled = true;
5505                    break;
5506                }
5507                Err(e) => panic!("filling the frontend send queue failed: {e:?}"),
5508            }
5509        }
5510        assert!(filled, "premise: the frontend send queue is full");
5511        h2.core.readiness.event.insert(Ready::WRITABLE);
5512
5513        let mut mux = Mux {
5514            configured_frontend_timeout: Duration::from_secs(10),
5515            frontend_token: Token(0),
5516            frontend: Connection::H2(h2),
5517            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5518            context,
5519            session_ulid: Ulid::generate(),
5520            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(10))),
5521            backend_registry: BackendRegistry::default(),
5522            backends: Rc::default(),
5523        };
5524        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
5525        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(NoRoutingProxy));
5526        let mut metrics = SessionMetrics::new(None);
5527        let result = mux.ready(session, proxy, &mut metrics);
5528
5529        let Connection::H2(h2) = &mux.frontend else {
5530            unreachable!("the frontend was built as H2")
5531        };
5532        assert!(
5533            matches!(result, SessionResult::Continue)
5534                && matches!(h2.core.linger, shared::Linger::Pending { .. }),
5535            "premise: the pass ends with the linger pending on close_notify, got \
5536             {result:?} with {:?}",
5537            h2.core.linger
5538        );
5539        assert_eq!(
5540            mux.context.streams[stream_id].state,
5541            StreamState::Recycle,
5542            "the stream that will get no answer is released, not linked"
5543        );
5544        assert!(
5545            mux.context.pending_links.is_empty() && mux.router.backends.is_empty(),
5546            "nothing is left to link and no backend was dialed"
5547        );
5548    }
5549
5550    /// A soft stop must still close a lingering H2 session once the H2
5551    /// listener's graceful-shutdown budget has elapsed, even when the linger
5552    /// began before the budget ran out. The lingering `readable`/`writable`
5553    /// passes must adopt the pass's clock: `graceful_shutdown_deadline_elapsed`
5554    /// reads the connection's own snapshot, and one frozen at the linger's
5555    /// start would only ever see the budget as it stood then.
5556    ///
5557    /// TO SEE THIS RED: drop the `self.core.adopt_now(context.now)` from the
5558    /// lingering early returns of `H2Shell::readable` and `H2Shell::writable`.
5559    /// `shutting_down` then keeps the session.
5560    #[test]
5561    fn a_soft_stop_closes_a_lingering_h2_session_once_its_budget_elapses() {
5562        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5563        let (socket, _peer) = connected_socket();
5564
5565        let mut h2 = h2::H2Shell::new(
5566            Ulid::generate(),
5567            socket,
5568            Position::Server,
5569            &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5570            H2FloodConfig::default(),
5571            H2ConnectionConfig::default(),
5572            Duration::from_secs(30),
5573            Some(Duration::from_secs(1)),
5574            Duration::from_secs(30),
5575            None,
5576            Ready::READABLE | Ready::HUP | Ready::ERROR,
5577        )
5578        .expect("a pool with free buffers must yield an H2 connection");
5579
5580        // A 1 s budget armed 3 s ago; the linger began 100 ms into it, with a
5581        // linger deadline far away.
5582        let armed_at = Instant::now() - Duration::from_secs(3);
5583        let linger_started = armed_at + Duration::from_millis(100);
5584        h2.core.state = H2State::Error;
5585        h2.core.drain.__test_arm_draining(armed_at);
5586        h2.core.adopt_now(linger_started);
5587        h2.core.linger = shared::Linger::Draining {
5588            deadline: linger_started + Duration::from_secs(60),
5589            remaining: shared::LINGER_MAX_BYTES,
5590        };
5591        let frontend = Connection::H2(h2);
5592        assert!(frontend.is_lingering(), "premise: the session lingers");
5593
5594        let mut mux = Mux {
5595            configured_frontend_timeout: Duration::from_secs(30),
5596            frontend_token: Token(0),
5597            frontend,
5598            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5599            context: test_context(&pool),
5600            session_ulid: Ulid::generate(),
5601            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
5602            backend_registry: BackendRegistry::default(),
5603            backends: Rc::default(),
5604        };
5605
5606        assert!(
5607            mux.shutting_down(),
5608            "the elapsed graceful-shutdown budget closes the lingering session"
5609        );
5610    }
5611
5612    /// `Mux::shutting_down` runs outside `ready()`, driven by
5613    /// `shut_down_sessions()`. It is therefore its own clock-sampling point,
5614    /// and that sample is load-bearing: it refreshes `context.now`, which
5615    /// `drive_frontend_shutdown_io` -> `readable()` mirrors into
5616    /// `ConnectionH2::now`, which the forced-close check then reads.
5617    ///
5618    /// A silent draining session gets no read/write events, so nothing else
5619    /// refreshes the snapshot. Without this sample the connection would carry
5620    /// the snapshot of its last `ready()` pass forever and the
5621    /// graceful-shutdown budget would never expire — the session would hang
5622    /// past the operator-configured SLA, which is the exact failure the
5623    /// forced-close deadline exists to prevent.
5624    ///
5625    /// The frontend is already draining, so `graceful_goaway` is NOT called on
5626    /// this path — that is what makes the test discriminating, because the
5627    /// `now` parameter of `graceful_goaway` cannot cover it.
5628    ///
5629    /// To SEE THIS RED: delete `self.context.now = now;` from
5630    /// `Mux::shutting_down` (keeping `let now` so `graceful_goaway(now)` still
5631    /// compiles). `readable()` then mirrors the frozen snapshot, the budget
5632    /// reads as 0s elapsed against a 5s deadline, and the final assertion fails
5633    /// with `assertion failed: mux.shutting_down()`.
5634    /// One `shutting_down()` pass on an idle draining H2 session makes exactly
5635    /// one `writable()` call: the forced pass, with nothing queued after it.
5636    ///
5637    /// `shut_down_sessions()` calls `shutting_down()` on every draining session
5638    /// at each 100 ms shutdown tick, so the work in one pass is multiplied by
5639    /// both. The loop in `Mux::drive_frontend_shutdown_io` used to wait for the
5640    /// WRITABLE event bit to clear, a bit that its own `signal_pending_write`
5641    /// had just set and that `writable()` clears only on `WouldBlock`, so an
5642    /// idle session with a writable socket ran all `MAX_LOOP_ITERATIONS`.
5643    /// Measured on the e2e drain test in a debug build: 10 000 calls and about
5644    /// 29 ms of CPU per pass, uncontended.
5645    ///
5646    /// To SEE THIS RED: restore the old exit condition,
5647    /// `!has_pending_write() && !readiness().event.is_writable()`; the count
5648    /// becomes `MAX_LOOP_ITERATIONS`.
5649    #[test]
5650    fn an_idle_draining_h2_session_makes_one_writable_pass_per_shutdown_tick() {
5651        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5652        let (socket, _peer) = connected_socket();
5653
5654        let mut h2 = h2::H2Shell::new(
5655            Ulid::generate(),
5656            socket,
5657            Position::Server,
5658            &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5659            H2FloodConfig::default(),
5660            H2ConnectionConfig::default(),
5661            Duration::from_secs(30),
5662            Some(Duration::from_secs(3600)),
5663            Duration::from_secs(30),
5664            None,
5665            Ready::READABLE | Ready::HUP | Ready::ERROR,
5666        )
5667        .expect("a pool with free buffers must yield an H2 connection");
5668
5669        // Already draining, budget far from elapsed: the pass must neither
5670        // send a GOAWAY nor force-close, only drive the frontend I/O.
5671        let armed_at = Instant::now();
5672        h2.core.state = H2State::Header;
5673        h2.core.drain.__test_arm_draining(armed_at);
5674        let mut frontend = Connection::H2(h2);
5675
5676        let mut context = test_context(&pool);
5677        context.now = armed_at;
5678        // One live stream, so the session keeps draining instead of closing.
5679        let stream_id = context
5680            .create_stream(Ulid::generate(), 1 << 16)
5681            .expect("test context must create a stream");
5682        context.streams[stream_id].state = StreamState::Linked(Token(1));
5683        let Connection::H2(h2) = &mut frontend else {
5684            unreachable!("frontend was built as H2")
5685        };
5686        h2.core.__test_insert_wire_mapping_only(1, stream_id);
5687
5688        let mut mux = Mux {
5689            configured_frontend_timeout: Duration::from_secs(30),
5690            frontend_token: Token(0),
5691            frontend,
5692            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5693            context,
5694            session_ulid: Ulid::generate(),
5695            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
5696            backend_registry: BackendRegistry::default(),
5697            backends: Rc::default(),
5698        };
5699        assert!(
5700            !mux.frontend.has_pending_write(),
5701            "precondition: the idle draining session has nothing queued"
5702        );
5703
5704        SHUTDOWN_WRITABLE_PASSES.with(|passes| passes.set(0));
5705        assert!(
5706            !mux.shutting_down(),
5707            "a draining session with a live stream and budget left must stay open"
5708        );
5709        let passes = SHUTDOWN_WRITABLE_PASSES.with(|passes| passes.get());
5710        assert_eq!(
5711            passes, 1,
5712            "one shutdown pass on an idle draining H2 session must make exactly the \
5713             one forced writable() call, not spin until MAX_LOOP_ITERATIONS \
5714             ({MAX_LOOP_ITERATIONS}); got {passes}"
5715        );
5716        assert!(
5717            !mux.frontend.has_pending_write(),
5718            "the pass must leave nothing queued behind"
5719        );
5720    }
5721
5722    #[test]
5723    fn shutting_down_refreshes_the_snapshot_so_the_drain_budget_expires() {
5724        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5725        let (socket, _peer) = connected_socket();
5726        let deadline = Duration::from_secs(5);
5727
5728        let mut h2 = h2::H2Shell::new(
5729            Ulid::generate(),
5730            socket,
5731            Position::Server,
5732            &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5733            H2FloodConfig::default(),
5734            H2ConnectionConfig::default(),
5735            Duration::from_secs(30),
5736            Some(deadline),
5737            Duration::from_secs(30),
5738            None,
5739            Ready::READABLE | Ready::HUP | Ready::ERROR,
5740        )
5741        .expect("a pool with free buffers must yield an H2 connection");
5742
5743        // A connection that already sent its initial GOAWAY and armed the
5744        // budget well over the deadline ago. `shutting_down` takes the
5745        // already-draining branch, which never calls `graceful_goaway`.
5746        let armed_at = Instant::now() - Duration::from_secs(60);
5747        h2.core.state = H2State::Header;
5748        h2.core.drain.__test_arm_draining(armed_at);
5749        let mut frontend = Connection::H2(h2);
5750
5751        let mut context = test_context(&pool);
5752        // Freeze the snapshot at the instant the budget was armed, as a
5753        // session that has seen no event since then would carry it.
5754        context.now = armed_at;
5755        // One live stream, so `can_stop` stays false and the forced-close
5756        // deadline is the only path that can return `true`.
5757        let stream_id = context
5758            .create_stream(Ulid::generate(), 1 << 16)
5759            .expect("test context must create a stream");
5760        context.streams[stream_id].state = StreamState::Linked(Token(1));
5761        // Give the connection a live WIRE stream too: with an empty
5762        // `ConnectionH2::streams`, `finalize_write` sends the FINAL GOAWAY and
5763        // force-disconnects, so `drive_frontend_shutdown_io` would return true
5764        // and short-circuit `shutting_down` before the deadline check. The
5765        // per-stream activity maps are deliberately left empty so
5766        // `cancel_timed_out_streams` early-returns and cannot queue an RST.
5767        let Connection::H2(h2) = &mut frontend else {
5768            unreachable!("frontend was built as H2")
5769        };
5770        h2.core.__test_insert_wire_mapping_only(1, stream_id);
5771
5772        let mut mux = Mux {
5773            configured_frontend_timeout: Duration::from_secs(30),
5774            frontend_token: Token(0),
5775            frontend,
5776            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5777            context,
5778            session_ulid: Ulid::generate(),
5779            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
5780            backend_registry: BackendRegistry::default(),
5781            backends: Rc::default(),
5782        };
5783
5784        assert!(
5785            mux.frontend.is_draining(),
5786            "precondition: the frontend must already be draining, so this test \
5787             exercises the branch that never reaches graceful_goaway"
5788        );
5789
5790        assert!(
5791            mux.shutting_down(),
5792            "shutting_down must refresh the clock snapshot itself, so a silent \
5793             draining session's forced-close budget still expires"
5794        );
5795    }
5796
5797    /// sozu-proxy/sozu#1647: a CONTINUATION that `drive_frontend_shutdown_io`
5798    /// reads completes the block outside `Mux::ready`, so its stream is left
5799    /// `StreamState::Link` until the next `ready` pass links it. A draining
5800    /// H2 session must stay open for it, within budget, rather than drop a
5801    /// request the advisory GOAWAY invited.
5802    #[test]
5803    fn a_draining_h2_session_waits_for_a_stream_awaiting_its_link() {
5804        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
5805        let (socket, _peer) = connected_socket();
5806
5807        let mut h2 = h2::H2Shell::new(
5808            Ulid::generate(),
5809            socket,
5810            Position::Server,
5811            &mut PoolBufferSource::new(Rc::downgrade(&pool)),
5812            H2FloodConfig::default(),
5813            H2ConnectionConfig::default(),
5814            Duration::from_secs(30),
5815            Some(Duration::from_secs(60)),
5816            Duration::from_secs(30),
5817            None,
5818            Ready::READABLE | Ready::HUP | Ready::ERROR,
5819        )
5820        .expect("a pool with free buffers must yield an H2 connection");
5821        // Already draining, budget armed now: nothing but the stream below
5822        // can keep the session open.
5823        h2.core.state = H2State::Header;
5824        h2.core.drain.__test_arm_draining(Instant::now());
5825        let mut frontend = Connection::H2(h2);
5826
5827        let mut context = test_context(&pool);
5828        let stream_id = context
5829            .create_stream(Ulid::generate(), 1 << 16)
5830            .expect("test context must create a stream");
5831        context.streams[stream_id].state = StreamState::Link;
5832        // A live wire stream, so `finalize_write` does not send the final
5833        // GOAWAY and close through `drive_frontend_shutdown_io`.
5834        let Connection::H2(h2) = &mut frontend else {
5835            unreachable!("frontend was built as H2")
5836        };
5837        h2.core.__test_insert_wire_mapping_only(1, stream_id);
5838
5839        let mut mux = Mux {
5840            configured_frontend_timeout: Duration::from_secs(30),
5841            frontend_token: Token(0),
5842            frontend,
5843            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
5844            context,
5845            session_ulid: Ulid::generate(),
5846            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
5847            backend_registry: BackendRegistry::default(),
5848            backends: Rc::default(),
5849        };
5850        assert!(
5851            !mux.frontend.graceful_shutdown_deadline_elapsed(),
5852            "precondition: the forced-close budget has not elapsed"
5853        );
5854
5855        assert!(
5856            !mux.shutting_down(),
5857            "a draining H2 session must stay open for a stream awaiting its link"
5858        );
5859        assert_eq!(
5860            mux.context.streams[stream_id].state,
5861            StreamState::Link,
5862            "the shutdown pass must leave the stream for `Mux::ready` to link"
5863        );
5864    }
5865
5866    #[test]
5867    fn a_short_read_after_hup_keeps_readable_to_read_the_eof() {
5868        let mut readiness = Readiness {
5869            event: Ready::READABLE | Ready::HUP,
5870            interest: Ready::READABLE,
5871        };
5872
5873        let should_yield =
5874            update_readiness_after_read(17, SocketResult::WouldBlock, &mut readiness);
5875
5876        assert!(!should_yield);
5877        assert!(
5878            readiness.event.is_readable(),
5879            "no edge will announce the queued EOF: READABLE must stay"
5880        );
5881    }
5882
5883    /// The case the HUP exception exists for, on a real loopback socket
5884    /// registered edge-triggered with mio: the data and the FIN arrive before
5885    /// the worker polls, so ONE event carries both `readable` and
5886    /// `read_closed`. The short read returns the data and stops before the
5887    /// EOF, and no later event will ever announce it. READABLE must survive
5888    /// `update_readiness_after_read` so the next pass reads `(0, Closed)`.
5889    ///
5890    /// To SEE THIS RED: remove the `is_hup()` exception from
5891    /// `update_readiness_after_read`; READABLE is dropped after the short read
5892    /// and the queued EOF is never read.
5893    #[test]
5894    fn data_and_fin_in_one_edge_keep_readable_until_the_eof_is_read() {
5895        use std::io::Write as _;
5896
5897        use crate::socket::SocketHandler as _;
5898
5899        let token = mio::Token(9);
5900        let mut poll = mio::Poll::new().expect("a poll instance must open");
5901        let mut events = mio::Events::with_capacity(8);
5902        let listener = std::net::TcpListener::bind("127.0.0.1:0")
5903            .expect("test listener must bind to a loopback port");
5904        let mut client = std::net::TcpStream::connect(
5905            listener
5906                .local_addr()
5907                .expect("test listener must report its local address"),
5908        )
5909        .expect("loopback connect must complete");
5910        let (accepted, _) = listener.accept().expect("the connection must be accepted");
5911        accepted
5912            .set_nonblocking(true)
5913            .expect("mio requires a nonblocking stream");
5914        let mut server = mio::net::TcpStream::from_std(accepted);
5915
5916        // Data and FIN are both queued before the socket is registered, so
5917        // the first poll reports them in a single edge.
5918        client.write_all(b"DATA").expect("the client must write");
5919        client
5920            .shutdown(std::net::Shutdown::Write)
5921            .expect("the client must half-close");
5922        poll.registry()
5923            .register(&mut server, token, mio::Interest::READABLE)
5924            .expect("the stream must register");
5925        poll.poll(&mut events, Some(std::time::Duration::from_secs(5)))
5926            .expect("poll must succeed");
5927        let event = events
5928            .iter()
5929            .find(|event| event.token() == token)
5930            .expect("the arrival must raise an event");
5931        assert!(
5932            event.is_readable() && event.is_read_closed(),
5933            "premise: data and FIN must share one edge, got {event:?}"
5934        );
5935        let mut readiness = Readiness {
5936            event: Ready::from(event),
5937            interest: Ready::READABLE,
5938        };
5939
5940        let mut buf = vec![0u8; 16 * 1024];
5941        let (size, status) = server.socket_read(&mut buf);
5942        assert_eq!((size, status), (4, SocketResult::WouldBlock));
5943        assert!(!update_readiness_after_read(size, status, &mut readiness));
5944        assert!(
5945            readiness.event.is_readable(),
5946            "HUP was seen: READABLE must stay so the EOF is read"
5947        );
5948
5949        poll.poll(&mut events, Some(std::time::Duration::from_millis(300)))
5950            .expect("poll must succeed");
5951        assert!(
5952            events.iter().all(|event| event.token() != token),
5953            "edge-triggered: no further event announces the queued EOF"
5954        );
5955
5956        let (size, status) = server.socket_read(&mut buf);
5957        assert_eq!((size, status), (0, SocketResult::Closed));
5958        assert!(update_readiness_after_read(size, status, &mut readiness));
5959        assert!(!readiness.event.is_readable());
5960    }
5961
5962    #[test]
5963    fn a_short_read_without_hup_drops_readable() {
5964        let mut readiness = Readiness {
5965            event: Ready::READABLE,
5966            interest: Ready::READABLE,
5967        };
5968
5969        let should_yield =
5970            update_readiness_after_read(17, SocketResult::WouldBlock, &mut readiness);
5971
5972        assert!(!should_yield);
5973        assert!(!readiness.event.is_readable(), "the next edge re-arms it");
5974    }
5975
5976    #[test]
5977    fn an_empty_read_after_hup_still_drops_readable() {
5978        let mut readiness = Readiness {
5979            event: Ready::READABLE | Ready::HUP,
5980            interest: Ready::READABLE,
5981        };
5982
5983        let should_yield = update_readiness_after_read(0, SocketResult::WouldBlock, &mut readiness);
5984
5985        assert!(should_yield);
5986        assert!(
5987            !readiness.event.is_readable(),
5988            "the exception must not spin"
5989        );
5990    }
5991
5992    #[test]
5993    fn update_readiness_after_read_closed_keeps_writable() {
5994        let mut readiness = Readiness {
5995            event: Ready::READABLE | Ready::WRITABLE | Ready::HUP,
5996            interest: Ready::READABLE | Ready::WRITABLE | Ready::HUP,
5997        };
5998
5999        let should_yield = update_readiness_after_read(17, SocketResult::Closed, &mut readiness);
6000
6001        assert!(!should_yield);
6002        assert!(!readiness.event.is_readable());
6003        assert!(readiness.event.is_writable());
6004        assert!(readiness.event.is_hup());
6005    }
6006
6007    /// Build a minimal H1 frontend `Mux` with one `Idle` stream. The returned
6008    /// peer socket must be kept alive for the duration of the test, otherwise
6009    /// the loopback connection is torn down and `readable()` sees a forced
6010    /// disconnect.
6011    ///
6012    /// An H1 frontend is deliberate: the `StreamState::Idle` arm of
6013    /// `Mux::timeout` writes a 408 and stamps
6014    /// `stream.context.access_log_message = Some("client_timeout")`, which is
6015    /// the crispest available witness that the timeout BODY ran. The H2 `Idle`
6016    /// arm is silently ignored and would witness nothing.
6017    fn h1_mux_with_idle_stream(
6018        pool: &Rc<RefCell<Pool>>,
6019        frontend_timeout: Duration,
6020    ) -> (
6021        Mux<mio::net::TcpStream, test_support::TestListener>,
6022        std::net::TcpStream,
6023    ) {
6024        let (socket, peer) = connected_socket();
6025        let frontend = Connection::new_h1_server(Ulid::generate(), socket, frontend_timeout);
6026        let mut context = test_context(pool);
6027        context
6028            .create_stream(Ulid::generate(), 1 << 16)
6029            .expect("test context must create a stream");
6030        let mux = Mux {
6031            configured_frontend_timeout: frontend_timeout,
6032            frontend_token: Token(0),
6033            frontend,
6034            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
6035            context,
6036            session_ulid: Ulid::generate(),
6037            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(frontend_timeout)),
6038            backend_registry: BackendRegistry::default(),
6039            backends: Rc::default(),
6040        };
6041        (mux, peer)
6042    }
6043
6044    /// Stage the exact shape of sozu-proxy/sozu#1442: a request fully written
6045    /// onto a REUSED keep-alive upstream that then closed without answering.
6046    ///
6047    /// `front.consumed` is what `end_stream_decision` reads to mean "the
6048    /// request left the front buffer"; the armed `retry_buffer` is what
6049    /// `ConnectionH1::writable` fills, and only ever on a connection whose
6050    /// `reused_from_pool` is set.
6051    fn stale_pooled_upstream_stream(
6052        pool: &Rc<RefCell<Pool>>,
6053        method: Method,
6054    ) -> (
6055        Mux<mio::net::TcpStream, test_support::TestListener>,
6056        std::net::TcpStream,
6057    ) {
6058        let (mut mux, peer) = h1_mux_with_idle_stream(pool, Duration::from_secs(60));
6059        let stream = &mut mux.context.streams[0];
6060        // The capture is armed while the method is still `Get`, and the method
6061        // under test is set afterwards. Since sozu-proxy/sozu#1450
6062        // `arm_upstream_replay` refuses to capture a non-idempotent request at
6063        // all, so arming under `method` would leave the POST and `PATCH` cases
6064        // below with no capture — and they would then be observing the ARM
6065        // guard rather than the conjunct they exist to pin. Staging it this way
6066        // keeps `Stream::can_replay_on_fresh_upstream`'s own idempotence
6067        // conjunct as the thing under test: dropping that conjunct still
6068        // reddens them. Production cannot reach this state any more, because a
6069        // request's method does not change after its headers are parsed, which
6070        // is exactly why the conjunct is now defense in depth and why these
6071        // tests are what keep it honest.
6072        stream.context.method = Some(Method::Get);
6073        stream.arm_upstream_replay();
6074        stream
6075            .retry_buffer
6076            .as_mut()
6077            .expect("arm_upstream_replay must install a buffer")
6078            .extend_from_slice(b"GET /api HTTP/1.1\r\nHost: localhost\r\n\r\n");
6079        stream.context.method = Some(method);
6080        stream.front.consumed = true;
6081        // The upstream produced nothing: not in body phase, nothing forwarded,
6082        // nothing buffered.
6083        assert!(
6084            !stream.back.is_main_phase() && !stream.back.consumed && stream.back.storage.is_empty(),
6085            "precondition: the upstream must have produced no response at all"
6086        );
6087        (mux, peer)
6088    }
6089
6090    /// Concatenate every byte `ConnectionH1::writable` would hand the socket,
6091    /// in queue order. Mirrors its own `kawa.out` walk.
6092    fn queued_request_bytes(front: &GenericHttpStream) -> Vec<u8> {
6093        let buffer = front.storage.buffer();
6094        front
6095            .out
6096            .iter()
6097            .map(|block| match block {
6098                kawa::OutBlock::Store(store) => store.data(buffer).to_vec(),
6099                kawa::OutBlock::Delimiter => Vec::new(),
6100            })
6101            .collect::<Vec<_>>()
6102            .concat()
6103    }
6104
6105    /// Stage the partial-write shape of sozu-proxy/sozu#1442 on the front
6106    /// kawa: a real request, parsed and serialized exactly as
6107    /// `ConnectionH1::writable` serializes it, of which the socket accepted
6108    /// everything but the last `unwritten_tail` bytes.
6109    ///
6110    /// `kawa::Kawa::consume` pushes the partially consumed store back to the
6111    /// FRONT of `out` (kawa-0.7.1 `storage/repr.rs`), so the refused tail
6112    /// stays queued ahead of anything appended afterwards. That is what makes
6113    /// the queue position of the replay load-bearing.
6114    ///
6115    /// Returns the complete serialization, which a correct replay reproduces
6116    /// byte-for-byte.
6117    fn stage_partially_written_pooled_request(
6118        stream: &mut Stream,
6119        unwritten_tail: usize,
6120    ) -> Vec<u8> {
6121        let request: &[u8] = b"GET /api HTTP/1.1\r\nHost: localhost\r\nX-Tail: unwritten\r\n\r\n";
6122        stream.front.storage.space()[..request.len()].copy_from_slice(request);
6123        stream.front.storage.fill(request.len());
6124        kawa::h1::parse(&mut stream.front, &mut stream.context);
6125        assert!(
6126            stream.front.is_main_phase(),
6127            "the staged request must parse"
6128        );
6129        stream.front.prepare(&mut kawa::h1::BlockConverter);
6130        assert!(
6131            stream.front.blocks.is_empty(),
6132            "prepare must drain blocks, as it has by the time writable writes"
6133        );
6134
6135        let serialized = queued_request_bytes(&stream.front);
6136        let accepted = serialized
6137            .len()
6138            .checked_sub(unwritten_tail)
6139            .expect("the unwritten tail must fit inside the serialized request");
6140        stream.context.method = Some(Method::Get);
6141        // Through `arm_upstream_replay` rather than by assigning the field:
6142        // the capture owns a charge against `MAX_ARMED_REPLAY_CAPTURES`, so a
6143        // hand-built one would release a charge it never took.
6144        stream.arm_upstream_replay();
6145        stream
6146            .retry_buffer
6147            .as_mut()
6148            .expect("arm_upstream_replay must install a buffer")
6149            .extend_from_slice(&serialized[..accepted]);
6150        stream.front.consume(accepted);
6151        stream.front.consumed = true;
6152        serialized
6153    }
6154
6155    /// The replay must be queued AHEAD of the bytes the kernel refused.
6156    ///
6157    /// `ConnectionH1::writable` handles a partial `socket_write_vectored` by
6158    /// calling `signal_pending_write()` and returning `MuxResult::Continue`,
6159    /// which leaves the unwritten remainder queued: `kawa::Kawa::consume`
6160    /// pushes the partially consumed store back to the FRONT of `out`
6161    /// (kawa-0.7.1 `storage/repr.rs`). `Kawa::push_out` APPENDS, so queueing
6162    /// the capture with it would hand the healthy backend `[tail][head]` — a
6163    /// request mangled mid-token, on the wire, to a backend that never saw
6164    /// the first attempt.
6165    ///
6166    /// Reachable on an idempotent request large enough that the kernel
6167    /// accepts only part of one write to a pooled upstream, followed by that
6168    /// stale peer EOFing without answering.
6169    ///
6170    /// The remainder spans zero, one, three and thirteen `out` entries across
6171    /// the cases below: `consume` pushes back at most ONE partially consumed
6172    /// store and drops every entry before it, so how many survive depends on
6173    /// where the split falls. Prepending is correct for all of them because
6174    /// `VecDeque::push_front` preserves the relative order of what is already
6175    /// queued, and the capture is exactly the complementary prefix.
6176    ///
6177    /// To SEE THIS RED: in `Stream::queue_upstream_replay`, swap the
6178    /// `front.out.push_front(...)` back for `self.front.push_out(...)`.
6179    #[test]
6180    fn a_replay_is_queued_ahead_of_the_unwritten_tail() {
6181        for (unwritten_tail, queued_entries) in [(0, 0), (2, 1), (21, 3), (100, 13)] {
6182            let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6183            let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6184            let stream = &mut mux.context.streams[0];
6185            let serialized = stage_partially_written_pooled_request(stream, unwritten_tail);
6186            assert_eq!(
6187                stream.front.out.len(),
6188                queued_entries,
6189                "staging a {unwritten_tail}-byte tail must leave {queued_entries} queued entries"
6190            );
6191            assert_eq!(
6192                shared::end_stream_decision(stream),
6193                shared::EndStreamAction::ReplayOnFreshBackend,
6194                "a partially written request on a stale pooled upstream is still replayable"
6195            );
6196
6197            stream
6198                .queue_upstream_replay()
6199                .expect("the staged stream carries a capture");
6200
6201            assert_eq!(
6202                queued_request_bytes(&stream.front),
6203                serialized,
6204                "with a {unwritten_tail}-byte unwritten tail the fresh backend must \
6205                 receive the request in its original order, not the tail first"
6206            );
6207        }
6208    }
6209
6210    /// The defect of sozu-proxy/sozu#1442. A request written onto a pooled
6211    /// keep-alive upstream that the peer had already closed must be replayed
6212    /// on a fresh backend, not answered `502 Bad Gateway`: nothing was
6213    /// observed by the client, so re-issuing it is unobservable. Measured
6214    /// 2026-09-22 with only the replay decision reverted: 25 failures over
6215    /// 576 pooled `test_issue_806` trials (4.3% per trial), 25 red runs of
6216    /// 25; green on 25 of 25 with the replay in place.
6217    ///
6218    /// To SEE THIS RED: in `super::shared::end_stream_decision`, replace the
6219    /// `can_replay_on_fresh_upstream()` branch with a bare
6220    /// `EndStreamAction::SendDefault(502)` — the pre-fix shape.
6221    #[test]
6222    fn a_stale_pooled_upstream_that_never_answered_is_replayed_not_502() {
6223        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6224        let (mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6225        assert_eq!(
6226            shared::end_stream_decision(&mux.context.streams[0]),
6227            shared::EndStreamAction::ReplayOnFreshBackend,
6228        );
6229    }
6230
6231    /// The method veto. nginx keeps `non_idempotent` out of the
6232    /// `proxy_next_upstream` default so `POST, LOCK, PATCH` are "not passed
6233    /// to the next server if a request has been sent to an upstream server";
6234    /// pingora's default `error_while_proxy` calls `set_retry(false)` on
6235    /// `!method.is_idempotent()`. Replaying a POST can duplicate a write the
6236    /// origin may already have committed.
6237    ///
6238    /// To SEE THIS RED: drop the `Method::is_idempotent` conjunct from
6239    /// `Stream::can_replay_on_fresh_upstream`.
6240    #[test]
6241    fn a_non_idempotent_request_is_never_replayed_on_a_stale_upstream() {
6242        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6243        let (mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Post);
6244        assert_eq!(
6245            shared::end_stream_decision(&mux.context.streams[0]),
6246            shared::EndStreamAction::SendDefault(502),
6247        );
6248    }
6249
6250    /// An extension method sozu does not know the semantics of is treated
6251    /// like POST. nginx's list is a closed `POST, LOCK, PATCH`; sozu parses
6252    /// `PATCH` as `Method::Custom`, so refusing every `Custom` is what makes
6253    /// PATCH non-replayable here without maintaining a second list.
6254    #[test]
6255    fn an_unknown_method_is_never_replayed_on_a_stale_upstream() {
6256        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6257        let (mux, _peer) = stale_pooled_upstream_stream(&pool, Method::new(b"PATCH"));
6258        assert!(matches!(
6259            mux.context.streams[0].context.method,
6260            Some(Method::Custom(_))
6261        ));
6262        assert_eq!(
6263            shared::end_stream_decision(&mux.context.streams[0]),
6264            shared::EndStreamAction::SendDefault(502),
6265        );
6266    }
6267
6268    /// pingora's `RetryType::ReusedOnly`. The same EOF means "the pool handed
6269    /// out a socket the peer had already closed" on a reused connection and
6270    /// "the origin accepted the request and then died" on a fresh dial; only
6271    /// the first is replayable. A fresh dial never arms the capture, so the
6272    /// absent buffer IS the provenance check.
6273    ///
6274    /// To SEE THIS RED: drop the `retry_buffer.is_some()` conjunct from
6275    /// `Stream::can_replay_on_fresh_upstream` — the provenance check itself.
6276    /// NOT `ConnectionH1::writable`'s `reused_from_pool` guard: this test
6277    /// never calls `writable`. It stages the capture through
6278    /// `stale_pooled_upstream_stream` and clears it with
6279    /// `Stream::forget_upstream_replay`, so a recipe naming the write path
6280    /// cannot redden it however plausible it reads.
6281    #[test]
6282    fn a_request_written_to_a_freshly_dialled_upstream_is_not_replayed() {
6283        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6284        let (mut mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6285        mux.context.streams[0].forget_upstream_replay();
6286        assert_eq!(
6287            shared::end_stream_decision(&mux.context.streams[0]),
6288            shared::EndStreamAction::SendDefault(502),
6289        );
6290    }
6291
6292    /// The boundary itself: once a response byte exists the request is no
6293    /// longer replayable. nginx — "passing a request to the next server is
6294    /// only possible if nothing has been sent to a client yet". A byte merely
6295    /// BUFFERED is already refused here, which is stricter: a partial status
6296    /// line is HAProxy's `junk-response`, absent from every default.
6297    ///
6298    /// To SEE THIS RED: drop the `back.storage.is_empty()` conjunct from
6299    /// `Stream::can_replay_on_fresh_upstream`.
6300    #[test]
6301    fn a_buffered_response_byte_forecloses_the_replay() {
6302        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6303        let (mut mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6304        let back = &mut mux.context.streams[0].back;
6305        back.storage.space()[..4].copy_from_slice(b"HTTP");
6306        back.storage.fill(4);
6307        assert!(!back.is_main_phase(), "a bare `HTTP` is not yet a response");
6308        assert_eq!(
6309            shared::end_stream_decision(&mux.context.streams[0]),
6310            shared::EndStreamAction::SendDefault(502),
6311        );
6312    }
6313
6314    /// A response byte already FORWARDED to the client is the same refusal,
6315    /// reached through the other conjunct.
6316    ///
6317    /// To SEE THIS RED: drop the `!back.consumed` conjunct from
6318    /// `Stream::can_replay_on_fresh_upstream`.
6319    #[test]
6320    fn a_forwarded_response_byte_forecloses_the_replay() {
6321        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6322        let (mut mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6323        mux.context.streams[0].back.consumed = true;
6324        assert_eq!(
6325            shared::end_stream_decision(&mux.context.streams[0]),
6326            shared::EndStreamAction::SendDefault(502),
6327        );
6328    }
6329
6330    /// The replay re-serializes the captured bytes and nothing else: they go
6331    /// back into `front.out` as an owned store, `front.blocks` stays empty so
6332    /// `kawa::Kawa::prepare` contributes nothing, and the buffer is taken so
6333    /// one capture cannot be replayed twice. That is not a per-REQUEST bound:
6334    /// `ConnectionH1::start_stream` arms a fresh capture on the next pooled
6335    /// attempt, and `max_connection_attempts` is what bounds the request as a
6336    /// whole.
6337    ///
6338    /// To SEE THIS RED: make `Stream::queue_upstream_replay` clone the buffer
6339    /// instead of taking it.
6340    #[test]
6341    fn queueing_a_replay_moves_the_captured_bytes_into_the_front_buffer() {
6342        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
6343        let (mut mux, _peer) = stale_pooled_upstream_stream(&pool, Method::Get);
6344        let stream = &mut mux.context.streams[0];
6345        let captured = stream
6346            .retry_buffer
6347            .as_ref()
6348            .expect("the staged stream carries a capture")
6349            .to_vec();
6350
6351        assert_eq!(stream.queue_upstream_replay(), Some(captured.len()));
6352
6353        assert!(
6354            stream.retry_buffer.is_none(),
6355            "the capture must be taken, not cloned: it cannot outlive its replay"
6356        );
6357        assert!(
6358            stream.front.blocks.is_empty(),
6359            "prepare must have nothing to convert, or it would prepend a \
6360             second copy of the request line"
6361        );
6362        let buffer = stream.front.storage.buffer();
6363        let queued: Vec<u8> = stream
6364            .front
6365            .out
6366            .iter()
6367            .map(|block| match block {
6368                kawa::OutBlock::Store(store) => store.data(buffer).to_vec(),
6369                kawa::OutBlock::Delimiter => Vec::new(),
6370            })
6371            .collect::<Vec<_>>()
6372            .concat();
6373        assert_eq!(
6374            queued, captured,
6375            "the replayed request must be byte-identical to the first attempt"
6376        );
6377        assert_eq!(
6378            shared::end_stream_decision(stream),
6379            shared::EndStreamAction::SendDefault(502),
6380            "with the capture spent and nothing re-arming it, a second stale \
6381             upstream falls back to 502"
6382        );
6383    }
6384
6385    /// Park the frontend core's next deadline at an exact instant, then push it
6386    /// onto the wheel the way a real pass would.
6387    ///
6388    /// Writes `ConnectionH1::timeout_deadline` directly rather than going
6389    /// through `arm_timeout`, so a test can stage a deadline in the past ("the
6390    /// wheel is late") as easily as one in the future, without disturbing the
6391    /// configured duration the re-arm will use.
6392    fn arm_frontend_at(
6393        mux: &mut Mux<mio::net::TcpStream, test_support::TestListener>,
6394        deadline: Option<Instant>,
6395    ) {
6396        let Connection::H1(h1) = &mut mux.frontend else {
6397            unreachable!("the helper builds an H1 frontend")
6398        };
6399        h1.timeout_deadline = deadline;
6400        mux.reschedule();
6401    }
6402
6403    /// Same, for a backend connection that has not been inserted yet.
6404    fn park_backend_deadline(connection: &mut Connection<SessionTcpStream>, deadline: Instant) {
6405        let Connection::H1(h1) = connection else {
6406            unreachable!("the helper builds an H1 backend")
6407        };
6408        h1.timeout_deadline = Some(deadline);
6409    }
6410
6411    /// Fire the frontend timeout of `mux` now, on an H1 frontend whose
6412    /// lingering close may take `linger_timeout`, after `partial` request
6413    /// bytes arrived. Returns the result, the instants bracketing the call,
6414    /// and the frontend's linger state and deadline afterwards.
6415    fn fire_frontend_timeout_after(
6416        partial: &[u8],
6417        linger_timeout: Duration,
6418    ) -> (
6419        StateResult,
6420        Instant,
6421        Instant,
6422        h1::Linger,
6423        Option<Instant>,
6424        std::net::TcpStream,
6425    ) {
6426        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6427        let (mut mux, peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6428        let mut metrics = SessionMetrics::new(None);
6429        {
6430            let Connection::H1(h1) = &mut mux.frontend else {
6431                unreachable!("the helper builds an H1 frontend")
6432            };
6433            h1.linger_timeout = linger_timeout;
6434        }
6435        let storage = &mut mux.context.streams[0].front.storage;
6436        storage.space()[..partial.len()].copy_from_slice(partial);
6437        storage.fill(partial.len());
6438        arm_frontend_at(&mut mux, Some(Instant::now() - Duration::from_millis(1)));
6439
6440        let before = Instant::now();
6441        let result = mux.timeout(Token(0), &mut metrics);
6442        let after = Instant::now();
6443        let Connection::H1(h1) = &mux.frontend else {
6444            unreachable!("the helper builds an H1 frontend")
6445        };
6446        (
6447            result,
6448            before,
6449            after,
6450            h1.linger,
6451            mux.frontend.poll_timeout(),
6452            peer,
6453        )
6454    }
6455
6456    /// A frontend timeout answers a partly received request (408), and the
6457    /// write pass that sends the answer starts the lingering close. The
6458    /// re-arm `Mux::timeout_inner` runs after that pass must not replace the
6459    /// linger deadline (`request_timeout`, 5 s here) with the frontend
6460    /// timeout (60 s).
6461    ///
6462    /// TO SEE THIS RED: remove the `is_lingering()` guard of
6463    /// `ConnectionH1::arm_timeout`. The deadline is then about 60 s out.
6464    #[test]
6465    fn a_linger_started_by_a_timeout_write_keeps_its_own_deadline() {
6466        let linger = Duration::from_secs(5);
6467        let (result, before, after, state, deadline, _peer) =
6468            fire_frontend_timeout_after(b"POST /upl", linger);
6469
6470        assert_eq!(result, StateResult::Continue, "the session lingers");
6471        assert!(
6472            matches!(state, h1::Linger::Draining { .. }),
6473            "a partly received request is drained after its answer, got {state:?}"
6474        );
6475        let deadline = deadline.expect("a lingering frontend has a deadline");
6476        assert!(
6477            deadline >= before + linger && deadline <= after + linger,
6478            "the deadline is the linger's, {:?} from the call",
6479            deadline.saturating_duration_since(before)
6480        );
6481    }
6482
6483    /// A silent client — no byte of a request — gets its 408 and an
6484    /// immediate close: nothing is in flight to drain.
6485    ///
6486    /// TO SEE THIS RED: in `ConnectionH1::writable`, set the linger on
6487    /// `!request_complete` alone, without `request_started`.
6488    #[test]
6489    fn a_408_to_a_silent_client_closes_without_lingering() {
6490        let (result, _, _, state, _, mut peer) =
6491            fire_frontend_timeout_after(b"", Duration::from_secs(5));
6492
6493        assert_eq!(
6494            result,
6495            StateResult::CloseSession,
6496            "nothing was received: the 408 closes at once"
6497        );
6498        assert_eq!(state, h1::Linger::Off, "no linger for a silent client");
6499        peer.set_nonblocking(false)
6500            .expect("the peer reads blocking");
6501        peer.set_read_timeout(Some(Duration::from_secs(1)))
6502            .expect("the peer read is bounded");
6503        let mut head = [0u8; 12];
6504        std::io::Read::read_exact(&mut peer, &mut head).expect("the 408 reaches the client");
6505        assert_eq!(&head, b"HTTP/1.1 408", "the client gets its 408");
6506    }
6507
6508    /// A frontend socket standing for TLS: its output is buffered behind the
6509    /// socket (`socket_wants_write`) until the next write flushes it, and a
6510    /// close queues a `close_notify` the same way.
6511    #[derive(Debug)]
6512    struct BufferedTlsSocket {
6513        stream: mio::net::TcpStream,
6514        buffered: bool,
6515    }
6516
6517    impl SocketHandler for BufferedTlsSocket {
6518        fn socket_read(&mut self, buf: &mut [u8]) -> (usize, SocketResult) {
6519            self.stream.socket_read(buf)
6520        }
6521
6522        fn socket_write(&mut self, buf: &[u8]) -> (usize, SocketResult) {
6523            self.stream.socket_write(buf)
6524        }
6525
6526        fn socket_write_vectored(&mut self, bufs: &[std::io::IoSlice]) -> (usize, SocketResult) {
6527            // What was buffered leaves with this write.
6528            self.buffered = false;
6529            self.stream.socket_write_vectored(bufs)
6530        }
6531
6532        fn socket_wants_write(&self) -> bool {
6533            self.buffered
6534        }
6535
6536        fn socket_close(&mut self) {
6537            self.buffered = true;
6538        }
6539
6540        fn socket_ref(&self) -> &mio::net::TcpStream {
6541            &self.stream
6542        }
6543
6544        fn socket_mut(&mut self) -> &mut mio::net::TcpStream {
6545            &mut self.stream
6546        }
6547
6548        fn peer_addr(&self) -> Option<std::net::SocketAddr> {
6549            self.stream.peer_addr().ok()
6550        }
6551
6552        fn protocol(&self) -> crate::socket::TransportProtocol {
6553            crate::socket::TransportProtocol::Tcp
6554        }
6555
6556        fn read_error(&self) {}
6557
6558        fn write_error(&self) {}
6559    }
6560
6561    /// A TLS client half-closes while the answer to its partly received
6562    /// request is still buffered (a 408 from `Mux::timeout_inner`): the
6563    /// answer is flushed, then the `close_notify`, and the lingering close
6564    /// starts inside the loop (RFC 9112 §9.6, `ConnectionH1::start_linger`).
6565    /// The in-loop HUP check must leave that frontend to its drain: closing it
6566    /// with the client's last bytes unread resets the connection.
6567    ///
6568    /// What is asserted is the socket, not the reader: whichever path
6569    /// consumed the client's last bytes, the request reader or the linger
6570    /// drain, none may be left unread at the close.
6571    ///
6572    /// TO SEE THIS RED: drop `c.readiness.filter_interest().is_readable() ||`
6573    /// from `Connection::has_unread_input`, which is what keeps a lingering
6574    /// frontend (READABLE raised by `ConnectionH1::start_linger`) from the
6575    /// in-loop close: 4 bytes are left unread. Before sozu-proxy/sozu#1779 the
6576    /// `!self.frontend.is_lingering()` guard of that check did this.
6577    #[test]
6578    fn a_lingering_frontend_reads_the_clients_last_bytes_before_closing() {
6579        use std::io::{Read, Write};
6580
6581        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6582        let (socket, mut peer) = connected_socket();
6583        let frontend_timeout = Duration::from_secs(60);
6584        let mut frontend = Connection::new_h1_server(
6585            Ulid::generate(),
6586            BufferedTlsSocket {
6587                stream: socket,
6588                buffered: true,
6589            },
6590            frontend_timeout,
6591        );
6592        let mut context = test_context(&pool);
6593        context
6594            .create_stream(Ulid::generate(), 1 << 16)
6595            .expect("test context must create a stream");
6596        let partial = b"POST /upload HTTP/1.1\r\nHost: localhost\r\nContent-Length: 64\r\n\r\npart";
6597        {
6598            let Connection::H1(h1) = &mut frontend else {
6599                unreachable!("new_h1_server builds an H1 connection")
6600            };
6601            h1.stream = Some(0);
6602            let stream = &mut context.streams[0];
6603            let storage = &mut stream.front.storage;
6604            storage.space()[..partial.len()].copy_from_slice(partial);
6605            storage.fill(partial.len());
6606            let answers_rc = stream.answers.clone();
6607            let answers = answers_rc.borrow();
6608            set_default_answer(stream, &mut h1.readiness, 408, &answers);
6609        }
6610        let mut mux = Mux {
6611            configured_frontend_timeout: frontend_timeout,
6612            frontend_token: Token(0),
6613            frontend,
6614            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
6615            context,
6616            session_ulid: Ulid::generate(),
6617            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(frontend_timeout)),
6618            backend_registry: BackendRegistry::default(),
6619            backends: Rc::default(),
6620        };
6621
6622        // The client sends more of its body, then hangs up: one edge
6623        // carries both, as `EPOLLIN|EPOLLRDHUP`.
6624        peer.write_all(b"more").expect("the client peer writes");
6625        peer.shutdown(std::net::Shutdown::Write)
6626            .expect("the client peer sends its FIN");
6627        std::thread::sleep(Duration::from_millis(20));
6628        mux.frontend.readiness_mut().event = Ready::READABLE | Ready::HUP;
6629
6630        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
6631        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
6632        let mut metrics = SessionMetrics::new(None);
6633        let result = mux.ready(session, proxy, &mut metrics);
6634        let Connection::H1(h1) = &mux.frontend else {
6635            unreachable!("the frontend is H1")
6636        };
6637        // What the close must not leave behind: a byte the client sent and
6638        // sozu never read, which turns the close into a reset. Whichever
6639        // reader consumed them, the request reader or the linger drain, a
6640        // peek on the socket must meet the client's EOF, not data.
6641        let mut probe = [0u8; 16];
6642        let unread = match h1.socket.stream.peek(&mut probe) {
6643            Ok(n) => n,
6644            Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => 0,
6645            Err(e) => panic!("the frontend socket peek failed: {e}"),
6646        };
6647        assert!(
6648            matches!(result, SessionResult::Close),
6649            "the drain meets the client's EOF and closes, got {result:?}"
6650        );
6651        assert_eq!(
6652            unread, 0,
6653            "the session must read the rest of the request before it closes, {unread} bytes \
6654             left unread, {:?}",
6655            h1.linger
6656        );
6657        drop(mux);
6658
6659        peer.set_nonblocking(false)
6660            .expect("the peer reads blocking");
6661        peer.set_read_timeout(Some(Duration::from_secs(1)))
6662            .expect("the peer read is bounded");
6663        let mut received = Vec::new();
6664        let outcome = peer.read_to_end(&mut received);
6665        assert!(
6666            received.starts_with(b"HTTP/1.1 408"),
6667            "the client gets its answer, got {:?}",
6668            String::from_utf8_lossy(&received)
6669        );
6670        assert!(
6671            outcome.is_ok(),
6672            "the response ends with the FIN, not a reset: {outcome:?}"
6673        );
6674    }
6675
6676    /// A request answered before its head was whole (a 408 from
6677    /// `Mux::timeout_inner` on an `Idle` stream) is never linked to a backend
6678    /// when the rest of its head arrives afterwards: the answer is final, and
6679    /// the bytes behind it are drained after it, never forwarded.
6680    ///
6681    /// TO SEE THIS RED: drop `&& !answered` from the first-seen-request branch
6682    /// of `ConnectionH1::readable`.
6683    #[test]
6684    fn an_answered_request_is_never_linked() {
6685        use std::io::Write;
6686
6687        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6688        let (socket, mut peer) = connected_socket();
6689        let frontend_timeout = Duration::from_secs(60);
6690        let mut frontend = Connection::new_h1_server(Ulid::generate(), socket, frontend_timeout);
6691        let mut context = test_context(&pool);
6692        context
6693            .create_stream(Ulid::generate(), 1 << 16)
6694            .expect("test context must create a stream");
6695        let head_start = b"GET /slow HTTP/1.1\r\nHost: loc";
6696        {
6697            let Connection::H1(h1) = &mut frontend else {
6698                unreachable!("new_h1_server builds an H1 connection")
6699            };
6700            h1.stream = Some(0);
6701            let stream = &mut context.streams[0];
6702            let storage = &mut stream.front.storage;
6703            storage.space()[..head_start.len()].copy_from_slice(head_start);
6704            storage.fill(head_start.len());
6705            let answers_rc = stream.answers.clone();
6706            let answers = answers_rc.borrow();
6707            set_default_answer(stream, &mut h1.readiness, 408, &answers);
6708        }
6709        assert_eq!(
6710            context.streams[0].state,
6711            StreamState::Unlinked,
6712            "premise: the 408 leaves the stream answered"
6713        );
6714
6715        peer.write_all(b"alhost\r\n\r\n")
6716            .expect("the client peer writes the rest of its head");
6717        std::thread::sleep(Duration::from_millis(20));
6718        frontend.readiness_mut().event.insert(Ready::READABLE);
6719        let mut router = Router::new(Duration::from_secs(30), Duration::from_secs(30));
6720        let _ = frontend.readable(&mut context, EndpointClient(&mut router));
6721
6722        assert!(
6723            context.streams[0].front.is_main_phase(),
6724            "premise: the rest of the head was read and parsed"
6725        );
6726        assert_eq!(
6727            context.streams[0].state,
6728            StreamState::Unlinked,
6729            "an answered request must stay answered"
6730        );
6731        assert!(
6732            context.pending_links.is_empty(),
6733            "an answered request must not be queued for a backend"
6734        );
6735    }
6736
6737    /// An H1 frontend whose request was received whole and whose response
6738    /// has not started, as one waiting for its backend, with nothing queued
6739    /// for the client. `event` is the readiness the next pass sees.
6740    fn waiting_request_pass(event: Ready) -> SessionResult {
6741        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6742        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6743        {
6744            let stream = &mut mux.context.streams[0];
6745            let request = b"GET / HTTP/1.1\r\nHost: localhost\r\n\r\n";
6746            let storage = &mut stream.front.storage;
6747            storage.space()[..request.len()].copy_from_slice(request);
6748            storage.fill(request.len());
6749            kawa::h1::parse(&mut stream.front, &mut stream.context);
6750            assert!(
6751                stream.front.is_terminated(),
6752                "premise: the request is whole"
6753            );
6754            stream.state = StreamState::Unlinked;
6755        }
6756        let readiness = mux.frontend.readiness_mut();
6757        readiness.interest.remove(Ready::READABLE);
6758        readiness.event = event;
6759        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
6760        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
6761        let mut metrics = SessionMetrics::new(None);
6762        mux.ready(session, proxy, &mut metrics)
6763    }
6764
6765    /// A full hang-up whose reset a `read` or `write` already consumed comes
6766    /// as `EPOLLHUP` without `EPOLLERR`: HUP and WRITE_CLOSED, no ERROR. It
6767    /// closes the session at once, even with a request in flight and no write
6768    /// pending to fail; a half-close (HUP alone) keeps it for the response.
6769    ///
6770    /// TO SEE THIS RED: drop the `event.is_write_closed()` clause from
6771    /// `Mux::frontend_exchange_in_flight`.
6772    #[test]
6773    fn a_hang_up_without_error_closes_a_waiting_request() {
6774        assert!(
6775            matches!(
6776                waiting_request_pass(Ready::READABLE | Ready::HUP | Ready::WRITE_CLOSED),
6777                SessionResult::Close
6778            ),
6779            "a hang-up must close the session at once"
6780        );
6781        assert!(
6782            matches!(
6783                waiting_request_pass(Ready::READABLE | Ready::HUP),
6784                SessionResult::Continue
6785            ),
6786            "a half-close keeps the session for the response"
6787        );
6788    }
6789
6790    /// A write that meets the client's reset before `epoll` reports it fails
6791    /// (`ECONNRESET`/`EPIPE`): the session closes at once, instead of
6792    /// waiting for an event that only says what the write already learned.
6793    /// Any failed write closes it, since nothing more can be flushed.
6794    ///
6795    /// TO SEE THIS RED: make `ConnectionH1::client_write_failed` return
6796    /// `None`.
6797    #[test]
6798    #[cfg(target_os = "linux")]
6799    fn a_failed_write_to_the_client_closes_the_session() {
6800        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6801        let (mut mux, peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6802        {
6803            let Connection::H1(h1) = &mut mux.frontend else {
6804                unreachable!("the helper builds an H1 frontend")
6805            };
6806            let stream = &mut mux.context.streams[0];
6807            let request = b"GET / HTTP/1.1\r\nHost: localhost\r\n\r\n";
6808            let storage = &mut stream.front.storage;
6809            storage.space()[..request.len()].copy_from_slice(request);
6810            storage.fill(request.len());
6811            kawa::h1::parse(&mut stream.front, &mut stream.context);
6812            let answers_rc = stream.answers.clone();
6813            let answers = answers_rc.borrow();
6814            set_default_answer(stream, &mut h1.readiness, 503, &answers);
6815        }
6816        // The client resets: `SO_LINGER` zero, then close.
6817        let linger = libc::linger {
6818            l_onoff: 1,
6819            l_linger: 0,
6820        };
6821        // SAFETY: `linger` is a valid `libc::linger` and `peer` owns the
6822        // descriptor for the duration of the call.
6823        let rc = unsafe {
6824            libc::setsockopt(
6825                std::os::fd::AsRawFd::as_raw_fd(&peer),
6826                libc::SOL_SOCKET,
6827                libc::SO_LINGER,
6828                &linger as *const _ as *const libc::c_void,
6829                std::mem::size_of::<libc::linger>() as libc::socklen_t,
6830            )
6831        };
6832        assert_eq!(rc, 0, "SO_LINGER zero must set");
6833        drop(peer);
6834        std::thread::sleep(Duration::from_millis(20));
6835        // The pass runs on a WRITABLE edge only: the reset is learned by the
6836        // write, not from the event.
6837        mux.frontend.readiness_mut().event = Ready::WRITABLE;
6838
6839        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
6840        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
6841        let mut metrics = SessionMetrics::new(None);
6842        let result = mux.ready(session, proxy, &mut metrics);
6843        assert!(
6844            matches!(result, SessionResult::Close),
6845            "a failed write to the client must close the session, got {result:?}"
6846        );
6847    }
6848
6849    /// A client that never sends its EOF is closed at the linger deadline.
6850    #[test]
6851    fn a_silent_client_is_closed_at_the_linger_deadline() {
6852        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6853        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6854        let mut metrics = SessionMetrics::new(None);
6855        let deadline = Instant::now() - Duration::from_millis(1);
6856        {
6857            let Connection::H1(h1) = &mut mux.frontend else {
6858                unreachable!("the helper builds an H1 frontend")
6859            };
6860            h1.linger = h1::Linger::Draining {
6861                deadline,
6862                remaining: h1::LINGER_MAX_BYTES,
6863            };
6864        }
6865        arm_frontend_at(&mut mux, Some(deadline));
6866
6867        assert_eq!(
6868            mux.timeout(Token(0), &mut metrics),
6869            StateResult::CloseSession,
6870            "the linger deadline closes the session"
6871        );
6872        assert_eq!(
6873            mux.context.streams[0].context.access_log_message, None,
6874            "no per-stream timeout answer for a lingering frontend"
6875        );
6876    }
6877
6878    /// The timer wheel rounds a delay to the NEAREST tick and `Timer::poll`
6879    /// fires everything whose tick has come, so an entry armed for deadline `D`
6880    /// is handed over from `tick * round(D / tick) - tick/2` onwards — up to
6881    /// 99 ms early with the default 100 ms tick. An early delivery is not an
6882    /// expiry: `Mux::timeout` must leave the session alone.
6883    ///
6884    /// To SEE THIS RED: make `Mux::consume_timer_entry` return `true`
6885    /// unconditionally (drop its `core_deadline.is_some_and(...)` early
6886    /// return). The body then runs on an early delivery and this fails with
6887    /// `access_log_message == Some("client_timeout")` — a session closed up to
6888    /// 99 ms before its configured `front_timeout`.
6889    #[test]
6890    fn an_early_wheel_delivery_does_not_run_the_timeout_body() {
6891        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6892        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6893        let mut metrics = SessionMetrics::new(None);
6894
6895        // A minute out, then delivered immediately: early, exaggerated so the
6896        // test cannot be flaky.
6897        arm_frontend_at(&mut mux, Some(Instant::now() + Duration::from_secs(60)));
6898
6899        let result = mux.timeout(Token(0), &mut metrics);
6900
6901        assert_eq!(
6902            result,
6903            StateResult::Continue,
6904            "an early delivery must leave the session running"
6905        );
6906        assert_eq!(
6907            mux.context.streams[0].context.access_log_message, None,
6908            "the timeout body must not run for a delivery before the deadline"
6909        );
6910    }
6911
6912    /// **The ordering test.** This is the one that proves why the
6913    /// `poll_timeout()` split (stored deadline first, memoized reschedule
6914    /// second) must land in that order and not the other way round.
6915    ///
6916    /// `Mux::sync_timeout` memoizes: it touches the wheel only when what the
6917    /// handle holds differs from what the core wants. On an early delivery the
6918    /// core's deadline has NOT moved — it still wants `D` — so a naive memo
6919    /// says "nothing to do" and emits nothing. But the wheel entry is already
6920    /// gone: the delivery consumed it. The session is then left with no timer
6921    /// at all and nothing to arm one. That is the LOST WAKEUP, and it is
6922    /// exactly the bug fixed for the UDP shell in `UdpManager::handle_timeout`,
6923    /// where `reschedule` emitted `ArmTimer` only when the minimum deadline had
6924    /// MOVED.
6925    ///
6926    /// What prevents it here is consume-then-reschedule:
6927    /// `Mux::consume_timer_entry` calls `TimeoutContainer::triggered`, which
6928    /// clears the handle's deadline, so the memo correctly reads "the wheel
6929    /// holds nothing" and re-arms at the same instant.
6930    ///
6931    /// To SEE THIS RED — this is the C2-without-C1 tree:
6932    /// 1. delete `self.deadline = None;` from `TimeoutContainer::triggered`
6933    ///    (`timer.rs`), and
6934    /// 2. delete the `&& container.is_armed() == next.is_some()` half of the
6935    ///    memo in `Mux::sync_timeout`.
6936    ///
6937    /// `sync_timeout` then sees `deadline() == Some(D) == next` and emits
6938    /// nothing, leaving the session alive with no wheel entry. In a DEBUG build
6939    /// `Mux::debug_assert_timer_coherence` catches it first, inside the
6940    /// `reschedule` on the way out of `timeout`, with `timer coherence for
6941    /// Token(0): armed=false but the core wants Some(..)`; the `is_armed()`
6942    /// assertion below is what catches it with debug assertions off. Either
6943    /// mutation alone is enough; both together are the shape the refactor would
6944    /// have had if the memoized reschedule had landed before the stored
6945    /// deadline.
6946    #[test]
6947    fn an_early_wheel_delivery_leaves_a_live_entry_at_the_same_deadline() {
6948        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6949        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6950        let mut metrics = SessionMetrics::new(None);
6951
6952        let deadline = Instant::now() + Duration::from_secs(60);
6953        arm_frontend_at(&mut mux, Some(deadline));
6954
6955        let result = mux.timeout(Token(0), &mut metrics);
6956        assert_eq!(result, StateResult::Continue);
6957
6958        let container = &mux.timeouts.frontend;
6959        assert!(
6960            container.is_armed(),
6961            "the wheel entry was consumed by the delivery; if the memoized \
6962             reschedule does not put one back the session has no timer at all \
6963             — a lost wakeup"
6964        );
6965        assert_eq!(
6966            container.deadline(),
6967            Some(deadline),
6968            "the entry must go back at the SAME instant: a later one is a \
6969             silently extended timeout, a different one is a moved deadline"
6970        );
6971        assert_eq!(
6972            mux.frontend.poll_timeout(),
6973            Some(deadline),
6974            "an early delivery must not disturb what the core asked for"
6975        );
6976    }
6977
6978    /// The mirror: a delivery at or past the deadline is a real expiry and must
6979    /// run the timeout body. Without this, "treat everything as early" would
6980    /// pass the two tests above and never time anything out.
6981    ///
6982    /// To SEE THIS RED: make `Mux::consume_timer_entry` return `false`
6983    /// unconditionally.
6984    #[test]
6985    fn a_delivery_past_the_deadline_runs_the_timeout_body() {
6986        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
6987        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
6988        let mut metrics = SessionMetrics::new(None);
6989
6990        arm_frontend_at(&mut mux, Some(Instant::now() - Duration::from_secs(1)));
6991
6992        let _ = mux.timeout(Token(0), &mut metrics);
6993
6994        assert_eq!(
6995            mux.context.streams[0].context.access_log_message,
6996            Some("client_timeout"),
6997            "a delivery at or past the deadline is a real expiry and must run \
6998             the timeout body"
6999        );
7000    }
7001
7002    /// A firing for a core that wants no timer at all must fail OPEN — be
7003    /// reported due — rather than being re-validated away. Failing closed there
7004    /// would make such a session immortal, which is strictly worse than the
7005    /// up-to-99 ms-early close the re-validation exists to prevent.
7006    ///
7007    /// To SEE THIS RED: change `Mux::consume_timer_entry` to return `false`
7008    /// when `core_deadline` is `None`.
7009    #[test]
7010    fn a_firing_with_no_deadline_is_treated_as_due() {
7011        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7012        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7013        let mut metrics = SessionMetrics::new(None);
7014
7015        arm_frontend_at(&mut mux, None);
7016        assert_eq!(mux.frontend.poll_timeout(), None, "precondition: no timer");
7017
7018        let _ = mux.timeout(Token(0), &mut metrics);
7019
7020        assert_eq!(
7021            mux.context.streams[0].context.access_log_message,
7022            Some("client_timeout"),
7023            "a firing with no deadline must fail open and run the body"
7024        );
7025    }
7026
7027    /// A real expiry must CLEAR the core's elapsed deadline rather than leave
7028    /// it in place for `reschedule` to re-arm.
7029    ///
7030    /// The core asked to be called back at `D` and has been. Leaving `D` there
7031    /// makes the memoized reschedule arm an already-elapsed instant, which the
7032    /// wheel re-delivers on the very next tick, and the session spins — the
7033    /// canonical sans-io busy loop that `UdpManager::handle_timeout`'s
7034    /// strict-advance `debug_assert!` exists to catch. Every branch of
7035    /// `Mux::timeout` that keeps the session alive re-arms explicitly with
7036    /// `arm_timeout`, which is what the old `triggered()` + `set(token)` pair
7037    /// did.
7038    ///
7039    /// Staged on a BACKEND token deliberately. A frontend firing runs
7040    /// `ConnectionH1::writable`, whose first act is `arm_timeout`, so the
7041    /// frontend re-arms as a side effect and cannot witness the difference. The
7042    /// backend `should_close` path re-arms nothing — it is the one place the
7043    /// elapsed deadline would survive.
7044    ///
7045    /// To SEE THIS RED: delete the `clear_timeout()` block from
7046    /// `Mux::consume_timer_entry`. The strict-advance `debug_assert!` in
7047    /// `SessionState::timeout` fires first (which is the point of it); with
7048    /// debug assertions off, these assertions catch it.
7049    #[test]
7050    fn a_real_expiry_clears_the_elapsed_deadline_instead_of_re_arming_it() {
7051        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7052        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7053        let mut metrics = SessionMetrics::new(None);
7054        let backend_token = Token(1);
7055
7056        let (mut connection, _backend_peer) =
7057            test_backend_connection(&mut mux, Duration::from_secs(30));
7058        let Connection::H1(h1) = &mut connection else {
7059            unreachable!("new_h1_client builds an H1 connection")
7060        };
7061        h1.stream = Some(0);
7062        let fired_at = Instant::now();
7063        park_backend_deadline(&mut connection, fired_at - Duration::from_secs(1));
7064        mux.router.backends.insert(backend_token, connection);
7065        mux.reschedule();
7066
7067        // One linked stream whose response is already terminated and fully
7068        // proxied: the arm that leaves `should_close` true, so nothing re-arms
7069        // the backend.
7070        mux.context.link_stream(0, backend_token);
7071        mux.context.streams[0].back.consumed = true;
7072        mux.context.streams[0].back.parsing_phase = kawa::ParsingPhase::Terminated;
7073
7074        let _ = mux.timeout(backend_token, &mut metrics);
7075
7076        let core = mux
7077            .router
7078            .backends
7079            .get(&backend_token)
7080            .and_then(Connection::poll_timeout);
7081        assert!(
7082            core.is_none_or(|next| next > fired_at),
7083            "after a real expiry the core must want no timer or a strictly \
7084             later one, never the instant that just fired ({core:?})"
7085        );
7086        assert!(
7087            mux.timeouts
7088                .backends
7089                .get(&backend_token)
7090                .and_then(TimeoutContainer::deadline)
7091                .is_none_or(|next| next > fired_at),
7092            "and the wheel must not be holding the elapsed instant either"
7093        );
7094    }
7095
7096    /// A backend that leaves `router.backends` must take its wheel handle with
7097    /// it. Nothing cancels it explicitly any more — `reschedule`'s `retain`
7098    /// drops the handle and `TimeoutContainer::drop` cancels the entry — so the
7099    /// eviction has to be pinned somewhere.
7100    ///
7101    /// To SEE THIS RED: delete the `self.timeouts.backends.retain(...)` call from
7102    /// `Mux::reschedule`. The handle then outlives its connection and
7103    /// `debug_assert_timer_coherence` panics with "timer handle for Token(1)
7104    /// outlived its connection".
7105    #[test]
7106    fn a_departed_backend_releases_its_wheel_handle() {
7107        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7108        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7109        let backend_token = Token(1);
7110        let (connection, _backend_peer) =
7111            test_backend_connection(&mut mux, Duration::from_secs(30));
7112        mux.router.backends.insert(backend_token, connection);
7113
7114        mux.reschedule();
7115        assert!(
7116            mux.timeouts.backends.contains_key(&backend_token),
7117            "a live backend must hold a wheel handle"
7118        );
7119
7120        mux.router.backends.remove(&backend_token);
7121        mux.reschedule();
7122
7123        assert!(
7124            !mux.timeouts.backends.contains_key(&backend_token),
7125            "a departed backend must not leave a wheel handle behind"
7126        );
7127    }
7128
7129    // ── Endpoint::peer_rtt is keyed by token ────────────────────────────
7130    //
7131    // `EndpointClient` keys backends by token; `EndpointServer` has a single
7132    // frontend and ignores the token. The pair matters because the two sides
7133    // populate DIFFERENT access-log cells — `snapshot_rtts` reads the local
7134    // socket for `client_rtt` and this trait method for `server_rtt` — so a
7135    // lookup that returned the wrong connection's RTT would mislabel the
7136    // value rather than lose it, which no downstream assertion would catch.
7137
7138    /// An unknown token yields `None`, not some other backend's RTT.
7139    ///
7140    /// The premise is asserted first: on a live loopback socket
7141    /// `getsockopt(TCP_INFO)` succeeds, so a KNOWN token really does return
7142    /// `Some`. Without that half, this test would pass against a
7143    /// `peer_rtt` that always answered `None`.
7144    ///
7145    /// TO SEE THIS RED: in `EndpointClient::peer_rtt` (`connection.rs`),
7146    /// replace `.get(&token)` with `.values().next()`. The unknown token then
7147    /// resolves to the only backend in the map and the final assertion fails
7148    /// with `an unknown token must not resolve to another backend's RTT`.
7149    #[test]
7150    fn endpoint_client_peer_rtt_is_keyed_by_token() {
7151        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7152        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(30));
7153        let backend_token = Token(1);
7154        let (connection, _backend_peer) =
7155            test_backend_connection(&mut mux, Duration::from_secs(30));
7156        mux.router.backends.insert(backend_token, connection);
7157
7158        let endpoint = EndpointClient(&mut mux.router);
7159
7160        assert!(
7161            endpoint.peer_rtt(backend_token).is_some(),
7162            "premise: TCP_INFO answers on a live loopback backend, so a known \
7163             token returns Some — otherwise the assertion below proves nothing"
7164        );
7165        assert!(
7166            endpoint.peer_rtt(Token(99)).is_none(),
7167            "an unknown token must not resolve to another backend's RTT"
7168        );
7169    }
7170
7171    /// `EndpointServer` ignores the token: it holds one frontend, and every
7172    /// token must report that frontend's RTT rather than `None`.
7173    ///
7174    /// TO SEE THIS RED: in `EndpointServer::peer_rtt` (`connection.rs`),
7175    /// return `None` instead of `self.0.rtt()`. It fails on
7176    /// the first loop iteration with `EndpointServer must report its single
7177    /// frontend's RTT for any token` — the second never runs, which is why
7178    /// the loop is two tokens rather than an assertion about "any".
7179    #[test]
7180    fn endpoint_server_peer_rtt_ignores_the_token() {
7181        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7182        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(30));
7183
7184        let endpoint = EndpointServer(&mut mux.frontend);
7185
7186        for token in [Token(0), Token(7)] {
7187            assert!(
7188                endpoint.peer_rtt(token).is_some(),
7189                "EndpointServer must report its single frontend's RTT for any token"
7190            );
7191        }
7192    }
7193
7194    /// A `Mux` pass keeps the connection's frontend-RTT sample, and takes
7195    /// none when nothing in the pass is logged.
7196    ///
7197    /// Until the per-connection sample, every `Mux` entry point expired the
7198    /// frontend's sample so the next pass read a fresh one: twenty keep-alive
7199    /// requests cost twenty `getsockopt(TCP_INFO)` on the frontend alone. The
7200    /// sample is now the connection's (`memoized_rtt`), so a pass must leave
7201    /// it exactly where the previous one put it.
7202    ///
7203    /// Entered through [`SessionState::timeout`] rather than
7204    /// [`SessionState::ready`]: `ready` takes `Rc<RefCell<dyn ProxySession>>`
7205    /// and `Rc<RefCell<dyn L7Proxy>>` and this crate has no test
7206    /// implementation of either — `L7Proxy::sessions` alone would require a
7207    /// live `SessionManager`. `timeout` needs no mock and is not a lesser
7208    /// path: it is where `cancel_timed_out_streams` reaps a silent peer's
7209    /// streams and emits their access logs, and it was one of the three entry
7210    /// points that used to expire the sample.
7211    ///
7212    /// TO SEE THIS RED: at the top of [`Mux::timeout_inner`], below the
7213    /// early-delivery gate, clear the frontend's `local_rtt` again
7214    /// (`if let Connection::H2(h2) = &self.frontend { h2.local_rtt.set(None) }`),
7215    /// which is what `Mux::expire_client_rtt` did. The first assertion then
7216    /// fails with `a Mux pass must keep the connection's frontend RTT`,
7217    /// `left: None`.
7218    #[test]
7219    fn a_mux_pass_keeps_the_connection_sample_and_reads_none_without_a_log() {
7220        /// A value no loopback SRTT can take, standing for the sample an
7221        /// earlier pass of the connection took.
7222        const CONNECTION_SAMPLE: Duration = Duration::from_secs(4321);
7223
7224        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7225        let (socket, _peer) = connected_socket();
7226        let h2 = h2::H2Shell::new(
7227            Ulid::generate(),
7228            socket,
7229            Position::Server,
7230            &mut PoolBufferSource::new(Rc::downgrade(&pool)),
7231            H2FloodConfig::default(),
7232            H2ConnectionConfig::default(),
7233            Duration::from_secs(30),
7234            None,
7235            Duration::from_secs(30),
7236            None,
7237            Ready::READABLE | Ready::HUP | Ready::ERROR,
7238        )
7239        .expect("a pool with free buffers must yield an H2 connection");
7240
7241        let mut frontend = Connection::H2(h2);
7242        let Connection::H2(h2) = &mut frontend else {
7243            unreachable!("frontend was built as H2")
7244        };
7245        h2.local_rtt.set(Some(Some(CONNECTION_SAMPLE)));
7246        // A deadline already in the past, so `Mux::consume_timer_entry`
7247        // accepts the firing as a real expiry and the body below it runs.
7248        // Without this the connection carries the 30 s deadline it armed at
7249        // construction, the firing re-validates as an early wheel delivery,
7250        // and `timeout_inner` returns before reaching anything this test is
7251        // about.
7252        h2.core.timeout_deadline = Some(Instant::now() - Duration::from_secs(1));
7253
7254        let mut mux = Mux {
7255            configured_frontend_timeout: Duration::from_secs(30),
7256            frontend_token: Token(0),
7257            frontend,
7258            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
7259            context: test_context(&pool),
7260            session_ulid: Ulid::generate(),
7261            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(30))),
7262            backend_registry: BackendRegistry::default(),
7263            backends: Rc::default(),
7264        };
7265
7266        let mut metrics = SessionMetrics::new(None);
7267        let before = tcp_info_reads();
7268        let _ = mux.timeout(Token(0), &mut metrics);
7269
7270        let Connection::H2(h2) = &mux.frontend else {
7271            unreachable!("frontend was built as H2")
7272        };
7273        assert_eq!(
7274            h2.local_rtt.get(),
7275            Some(Some(CONNECTION_SAMPLE)),
7276            "a Mux pass must keep the connection's frontend RTT"
7277        );
7278        assert_eq!(
7279            tcp_info_reads() - before,
7280            0,
7281            "a Mux pass that logs no stream must not read TCP_INFO"
7282        );
7283    }
7284
7285    /// An `L7Proxy` for [`Mux::close`] on a session that holds no backend:
7286    /// the teardown only borrows the proxy per backend, so no method runs.
7287    struct NoBackendProxy;
7288
7289    impl L7Proxy for NoBackendProxy {
7290        fn kind(&self) -> sozu_command::proto::command::ListenerType {
7291            unreachable!("Mux::close without a backend never asks the proxy")
7292        }
7293        fn register_socket(
7294            &self,
7295            _socket: &mut TcpStream,
7296            _token: Token,
7297            _interest: Interest,
7298        ) -> Result<(), std::io::Error> {
7299            unreachable!("Mux::close without a backend never asks the proxy")
7300        }
7301        fn add_session(&self, _session: Rc<RefCell<dyn ProxySession>>) -> Token {
7302            unreachable!("Mux::close without a backend never asks the proxy")
7303        }
7304        fn remove_session(&self, _token: Token) -> bool {
7305            unreachable!("Mux::close without a backend never asks the proxy")
7306        }
7307        fn clusters(
7308            &self,
7309        ) -> &HashMap<sozu_command::state::ClusterId, sozu_command::proto::command::Cluster>
7310        {
7311            unreachable!("Mux::close without a backend never asks the proxy")
7312        }
7313        fn sessions(&self) -> Rc<RefCell<crate::server::SessionManager>> {
7314            unreachable!("Mux::close without a backend never asks the proxy")
7315        }
7316    }
7317
7318    /// Close a mux whose only stream is in `state`, still owing its access log
7319    /// or not, and return the `getsockopt(TCP_INFO)` reads the close made.
7320    ///
7321    /// `SessionMetrics::reset` is what every access-log site runs right after
7322    /// the line goes out, so a reset stream is one whose log was written.
7323    fn tcp_info_reads_of_a_close(state: StreamState, owes_a_log: bool) -> usize {
7324        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
7325        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(30));
7326        mux.context.streams[0].state = state;
7327        if !owes_a_log {
7328            mux.context.streams[0].metrics.reset();
7329        }
7330        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(NoBackendProxy));
7331        let mut metrics = SessionMetrics::new(None);
7332        let before = tcp_info_reads();
7333        mux.close(proxy, &mut metrics);
7334        tcp_info_reads() - before
7335    }
7336
7337    /// `Mux::close` reads the frontend RTT only when a stream still owes an
7338    /// access log — the ordinary close, where every stream was logged when it
7339    /// finished, reads nothing (issue #1590).
7340    ///
7341    /// TO SEE THIS RED: in `Mux::close`, hoist the sample above the loop
7342    /// (`let client_rtt = self.frontend.rtt();`). The first assertion then
7343    /// fails with `left: 1`, `right: 0`.
7344    #[test]
7345    fn mux_close_reads_the_frontend_rtt_only_for_a_stream_it_logs() {
7346        assert_eq!(
7347            tcp_info_reads_of_a_close(StreamState::Link, false),
7348            0,
7349            "a close with every stream already logged must not read TCP_INFO"
7350        );
7351        assert_eq!(
7352            tcp_info_reads_of_a_close(StreamState::Recycle, true),
7353            0,
7354            "a recycled stream is not logged at close, so it costs no read"
7355        );
7356        assert_eq!(
7357            tcp_info_reads_of_a_close(StreamState::Link, true),
7358            1,
7359            "an open stream logged at close costs exactly the frontend read: \
7360             it has no backend, so no server_rtt read"
7361        );
7362    }
7363
7364    /// #1564: naming an already-interned backend for another dial allocates
7365    /// nothing. `BackendRegistry::id_for` runs once per backend connection a
7366    /// session opens; the first dial of a backend pays for its slot, every
7367    /// later one must only bump reference counts. Two backends, so a cache
7368    /// that held a single id instead of one per slot fails too.
7369    #[test]
7370    fn a_redial_of_an_interned_backend_allocates_nothing() {
7371        use std::hint::black_box;
7372
7373        use crate::test_allocations::allocations;
7374
7375        const DIALS: usize = 64;
7376        let new_backend = |id: &str, port: u16| {
7377            Rc::new(RefCell::new(Backend::new(
7378                id,
7379                SocketAddr::from(([127, 0, 0, 1], port)),
7380                None,
7381                None,
7382                None,
7383            )))
7384        };
7385        let first = new_backend("redial-first", 7001);
7386        let second = new_backend("redial-second", 7002);
7387
7388        let mut registry = BackendRegistry::default();
7389        // Warm-up: the first dial of each backend interns it.
7390        let warm = [registry.id_for(&first), registry.id_for(&second)];
7391
7392        let before = allocations();
7393        for _ in 0..DIALS {
7394            black_box(registry.id_for(black_box(&first)));
7395            black_box(registry.id_for(black_box(&second)));
7396        }
7397        let allocated = allocations() - before;
7398        assert_eq!(
7399            allocated, 0,
7400            "{DIALS} redials of two interned backends made {allocated} heap \
7401             allocations, expected none"
7402        );
7403
7404        // The redial still names the right slot, id and address.
7405        let again = registry.id_for(&second);
7406        assert_eq!(again, warm[1]);
7407        assert_eq!(&*again.backend_id, "redial-second");
7408        assert_eq!(again.address, SocketAddr::from(([127, 0, 0, 1], 7002)));
7409        assert_ne!(warm[0].slot(), warm[1].slot());
7410    }
7411
7412    /// #1610: a session's first interned backend sizes the registry for one
7413    /// entry, not the four a first `Vec::push` reserves.
7414    ///
7415    /// The interning pays for the entry and for the `Rc<str>` copy of its id,
7416    /// and for nothing else; the id copy is measured on its own, so the
7417    /// assertion carries no size literal.
7418    ///
7419    /// TO SEE THIS RED: drop the `reserve_exact(1)` in
7420    /// `BackendRegistry::intern`. The first entry then allocates four
7421    /// `RegistryEntry` slots.
7422    #[test]
7423    fn the_first_interned_backend_sizes_the_registry_for_one() {
7424        use crate::test_allocations::bytes;
7425
7426        let backend = Rc::new(RefCell::new(Backend::new(
7427            "interned-first",
7428            SocketAddr::from(([127, 0, 0, 1], 7003)),
7429            None,
7430            None,
7431            None,
7432        )));
7433        let before = bytes();
7434        let id_copy: Rc<str> = Rc::from(backend.borrow().backend_id.as_str());
7435        let id_bytes = bytes() - before;
7436        drop(id_copy);
7437
7438        let mut registry = BackendRegistry::default();
7439        let before = bytes();
7440        let id = registry.id_for(&backend);
7441        let interned = bytes() - before;
7442
7443        assert_eq!(
7444            interned,
7445            std::mem::size_of::<RegistryEntry>() + id_bytes,
7446            "the first interned backend must size the registry for one entry"
7447        );
7448        assert_eq!(&*id.backend_id, "interned-first");
7449    }
7450
7451    /// #1613: the first backend entry of a session's reverse index holds its
7452    /// one stream inline, so linking it allocates nothing. The `HashMap` of
7453    /// `Vec`s it replaces allocated its table and the entry's vector here —
7454    /// two allocations, 156 bytes — once per session, so once per request
7455    /// on a one-request connection. (#1610 had already cut that vector from
7456    /// four slots to one.)
7457    ///
7458    /// TO SEE THIS RED: in `LinkedStreams::push`, spill into the `Vec` on
7459    /// the first link as well (1 allocation); or in
7460    /// `InlineTokenMap::get_or_insert_with`, send every token to `rest`
7461    /// (the `BTreeMap` leaf is allocated).
7462    #[test]
7463    fn a_new_backend_entry_in_the_reverse_index_is_sized_for_one_stream() {
7464        use crate::test_allocations::{allocations, bytes};
7465
7466        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7467        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7468
7469        let (before, before_bytes) = (allocations(), bytes());
7470        mux.context.link_stream(0, Token(1));
7471        let linked = (allocations() - before, bytes() - before_bytes);
7472
7473        assert_eq!(
7474            linked,
7475            (0, 0),
7476            "the first reverse-index entry must hold its one stream without allocating"
7477        );
7478        assert_eq!(
7479            mux.context
7480                .backend_streams
7481                .get(&Token(1))
7482                .map(|ids| ids.to_vec()),
7483            Some(vec![0])
7484        );
7485    }
7486
7487    /// The other side of the inline slot: a second stream linked to the same
7488    /// backend (an H2 backend multiplexing) moves both into one `Vec` sized
7489    /// for four, in link order — the single 32-byte allocation the `Vec` the
7490    /// slot replaced made at the same point — and a later link on that
7491    /// connection reuses it.
7492    #[test]
7493    fn a_second_stream_on_one_backend_spills_the_reverse_index_entry_once() {
7494        use crate::test_allocations::{allocations, bytes};
7495
7496        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7497        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7498        let second = mux
7499            .context
7500            .create_stream(Ulid::generate(), 1 << 16)
7501            .expect("the test pool must hand out a second stream");
7502        mux.context.link_stream(0, Token(1));
7503
7504        let (before, before_bytes) = (allocations(), bytes());
7505        mux.context.link_stream(second, Token(1));
7506        let spilled = (allocations() - before, bytes() - before_bytes);
7507        assert_eq!(
7508            spilled,
7509            (1, 4 * std::mem::size_of::<GlobalStreamId>()),
7510            "the second stream must spill both into one four-slot vector"
7511        );
7512        assert_eq!(
7513            mux.context
7514                .backend_streams
7515                .get(&Token(1))
7516                .map(|ids| ids.to_vec()),
7517            Some(vec![0, second]),
7518            "the spill must keep the link order"
7519        );
7520
7521        mux.context.unlink_stream(0);
7522        let before = allocations();
7523        mux.context.link_stream(0, Token(1));
7524        assert_eq!(
7525            allocations() - before,
7526            0,
7527            "a spilled entry keeps its capacity for the next link (#1583)"
7528        );
7529    }
7530
7531    /// #1613: the wheel handles of the usual session — the frontend's and one
7532    /// backend's — cost no allocation, because the frontend's is a field of
7533    /// `MuxTimeouts` and the first backend's is held inline. The
7534    /// `HashMap<Token, TimeoutContainer>` they replace allocated its table
7535    /// (one allocation, 340 bytes) once per session.
7536    ///
7537    /// Neither core wants a deadline here, so `reschedule` never reaches the
7538    /// timer wheel, and what is measured is the handles' storage alone.
7539    ///
7540    /// TO SEE THIS RED: in `InlineTokenMap::get_or_insert_with`, send every
7541    /// token to `rest` (the `BTreeMap` leaf is allocated).
7542    #[test]
7543    fn a_session_holds_its_frontend_and_first_backend_wheel_handles_without_allocating() {
7544        use crate::test_allocations::{allocations, bytes};
7545
7546        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7547        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7548        let backend_token = Token(1);
7549        let (mut connection, _backend_peer) =
7550            test_backend_connection(&mut mux, Duration::from_secs(30));
7551        connection.clear_timeout();
7552        mux.frontend.clear_timeout();
7553
7554        let (before, before_bytes) = (allocations(), bytes());
7555        mux.router.backends.insert(backend_token, connection);
7556        mux.reschedule();
7557        let held = (allocations() - before, bytes() - before_bytes);
7558
7559        assert_eq!(
7560            held,
7561            (0, 0),
7562            "the frontend and first backend handles must not allocate"
7563        );
7564        assert!(
7565            mux.timeouts.backends.contains_key(&backend_token),
7566            "the live backend must hold a wheel handle"
7567        );
7568    }
7569
7570    /// Move a client connection built `Connecting` to `Connected`, as the
7571    /// WRITABLE edge of an established dial does in `Mux::ready_inner`.
7572    fn mark_connected(h1: &mut ConnectionH1<SessionTcpStream>) {
7573        let Position::Client(_, _, status) = &mut h1.position else {
7574            unreachable!("a backend connection is in client position")
7575        };
7576        *status = BackendStatus::Connected;
7577    }
7578
7579    /// A backend timer that fires while the dial is still `Connecting` is a
7580    /// connect timeout (sozu-proxy/sozu#1800): no byte reached the backend,
7581    /// so the request is neither answered 504 nor unlinked here. The
7582    /// connection is flagged dead as a refused dial is, its streams stay
7583    /// linked for the `ready` pass that counts the failure and re-links them,
7584    /// and the session asks for that pass.
7585    ///
7586    /// To SEE THIS RED: delete the `Connecting` early return from the
7587    /// backend-token branch of `Mux::timeout_inner`.
7588    #[test]
7589    fn a_connect_timeout_flags_the_dial_dead_instead_of_answering_504() {
7590        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7591        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7592        let mut metrics = SessionMetrics::new(None);
7593        let backend_token = Token(1);
7594
7595        let (mut connection, _backend_peer) =
7596            test_backend_connection(&mut mux, Duration::from_secs(3));
7597        let Connection::H1(h1) = &mut connection else {
7598            unreachable!("new_h1_client builds an H1 connection")
7599        };
7600        h1.stream = Some(0);
7601        assert!(
7602            matches!(
7603                h1.position,
7604                Position::Client(_, _, BackendStatus::Connecting(_))
7605            ),
7606            "precondition: the dial has not completed"
7607        );
7608        park_backend_deadline(&mut connection, Instant::now() - Duration::from_secs(1));
7609        mux.router.backends.insert(backend_token, connection);
7610        mux.reschedule();
7611        mux.context.link_stream(0, backend_token);
7612        assert!(!mux.has_pending_backend_failure());
7613
7614        let result = mux.timeout(backend_token, &mut metrics);
7615
7616        assert_eq!(result, StateResult::Continue, "the session stays open");
7617        assert!(
7618            mux.has_pending_backend_failure(),
7619            "the dial is flagged dead for the ready pass"
7620        );
7621        assert_eq!(
7622            mux.context.streams[0].state,
7623            StreamState::Linked(backend_token),
7624            "the stream stays linked until the ready pass re-links it"
7625        );
7626        assert_eq!(
7627            mux.context.streams[0].context.access_log_message, None,
7628            "no backend_timeout answer"
7629        );
7630        assert!(
7631            mux.context.streams[0].back.blocks.is_empty()
7632                && mux.context.streams[0].back.out.is_empty(),
7633            "no default answer was written"
7634        );
7635    }
7636
7637    /// An H1 backend connection on a live loopback socket, plus the peer the
7638    /// caller must keep alive.
7639    fn test_backend_connection(
7640        mux: &mut Mux<mio::net::TcpStream, test_support::TestListener>,
7641        duration: Duration,
7642    ) -> (Connection<SessionTcpStream>, std::net::TcpStream) {
7643        let (backend_socket, backend_peer) = connected_socket();
7644        let backend_address = "127.0.0.1:2".parse().expect("backend address must parse");
7645        let backend = Rc::new(RefCell::new(Backend::new(
7646            "test-backend",
7647            backend_address,
7648            None,
7649            None,
7650            None,
7651        )));
7652        // Through the embedder's table, exactly as a real dial does: the
7653        // handle stops here and the connection carries the opaque id.
7654        let backend = mux.backend_registry.id_for(&backend);
7655        let connection = Connection::new_h1_client(
7656            Ulid::generate(),
7657            SessionTcpStream::new(backend_socket, mux.session_ulid, Some(backend_address)),
7658            "test-cluster".into(),
7659            backend,
7660            duration,
7661        );
7662        (connection, backend_peer)
7663    }
7664
7665    /// The loose end flagged as UNVERIFIED in the `poll_timeout` series: in the
7666    /// backend-token branch of `Mux::timeout`, the backend's timer is consumed
7667    /// unconditionally but re-armed only when `!should_close`, while the shared
7668    /// tail can still return `Continue`. If a backend could reach that tail
7669    /// still holding linked streams, those streams would be left with no
7670    /// backend timer at all — a lost wakeup.
7671    ///
7672    /// It cannot, and this pins the reason: the branch calls
7673    /// `Connection::end_stream` for every id in `context.backend_streams`.
7674    /// `ConnectionH2::end_stream` starts with `Context::unlink_stream`;
7675    /// `ConnectionH1::end_stream` guards `self.stream != Some(stream)` and
7676    /// returns early first, which is safe only because an H1 connection carries
7677    /// at most one linked stream and it equals `self.stream` (the comment at
7678    /// the re-arm site has the full chain). This test sets
7679    /// `h1.stream = Some(0)` by hand, so it exercises the MATCHED path only —
7680    /// the mismatch arm is unreached here. It is covered by that invariant plus
7681    /// the `debug_assert_h1_owns_stream` tripwire that runs at every
7682    /// `end_stream` call in `Mux::timeout`, not by a test that drives the arm
7683    /// itself. It drives the exact shape the issue asked for — a backend
7684    /// stalled after a PARTIAL response (`back.consumed`, not terminated),
7685    /// which is the arm that sets `should_write` while leaving `should_close`
7686    /// true, i.e. the arm that skips the re-arm.
7687    ///
7688    /// To SEE THIS RED: delete the `self.context.unlink_stream(stream_id);`
7689    /// from the "forcefully terminate it" arm AND the
7690    /// `backend.end_stream(stream_id, &mut self.context);` at the bottom of
7691    /// that loop — this arm evicts through both, so either alone still clears
7692    /// the index. The companion test
7693    /// `a_completed_backend_response_timeout_unlinks_through_end_stream_alone`
7694    /// covers the arm where `end_stream` IS the sole eviction.
7695    #[test]
7696    fn a_backend_timeout_leaves_no_linked_stream_behind_on_the_close_path() {
7697        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7698        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7699        let mut metrics = SessionMetrics::new(None);
7700        let backend_token = Token(1);
7701
7702        let (mut connection, _backend_peer) =
7703            test_backend_connection(&mut mux, Duration::from_secs(30));
7704        let Connection::H1(h1) = &mut connection else {
7705            unreachable!("new_h1_client builds an H1 connection")
7706        };
7707        h1.stream = Some(0);
7708        // A backend that has sent part of a response is connected. Left
7709        // `Connecting`, as `new_h1_client` builds it, the timer would be a
7710        // connect timeout, which fails the request over instead
7711        // (sozu-proxy/sozu#1800).
7712        mark_connected(h1);
7713        // Due, not early: the entry really elapsed.
7714        park_backend_deadline(&mut connection, Instant::now() - Duration::from_secs(1));
7715        mux.router.backends.insert(backend_token, connection);
7716        mux.reschedule();
7717
7718        mux.context.link_stream(0, backend_token);
7719        // A response that started but never finished.
7720        mux.context.streams[0].back.consumed = true;
7721        assert_eq!(
7722            mux.context
7723                .backend_streams
7724                .get(&backend_token)
7725                .map(|ids| ids.len()),
7726            Some(1),
7727            "precondition: the backend carries exactly one linked stream"
7728        );
7729
7730        let _ = mux.timeout(backend_token, &mut metrics);
7731
7732        assert!(
7733            mux.context
7734                .backend_streams
7735                .get(&backend_token)
7736                .is_none_or(|ids| ids.is_empty()),
7737            "every stream linked to a timed-out backend must be unlinked before \
7738             the shared tail runs, because the tail may return Continue with \
7739             only the frontend timer re-armed"
7740        );
7741        assert_eq!(
7742            mux.context.streams[0].context.access_log_message,
7743            Some("backend_response_timeout"),
7744            "precondition: this is the partial-response arm, the one that \
7745             leaves should_close true and therefore skips the backend re-arm"
7746        );
7747    }
7748
7749    /// A frontend whose stream 0 is linked to a connected H1 backend, with a
7750    /// response in progress (`back.consumed`, not terminated). `queued`
7751    /// leaves response bytes waiting for the client; the two deadlines are
7752    /// staged exactly. Shared by the sozu-proxy/sozu#1906 timeout tests.
7753    fn mux_with_response_in_progress(
7754        pool: &Rc<RefCell<Pool>>,
7755        queued: bool,
7756        frontend_deadline: Instant,
7757        backend_deadline: Instant,
7758    ) -> (
7759        Mux<mio::net::TcpStream, test_support::TestListener>,
7760        Token,
7761        std::net::TcpStream,
7762        std::net::TcpStream,
7763    ) {
7764        let (mut mux, peer) = h1_mux_with_idle_stream(pool, Duration::from_secs(60));
7765        let backend_token = Token(1);
7766        let (mut connection, backend_peer) =
7767            test_backend_connection(&mut mux, Duration::from_secs(30));
7768        let Connection::H1(h1) = &mut connection else {
7769            unreachable!("new_h1_client builds an H1 connection")
7770        };
7771        h1.stream = Some(0);
7772        mark_connected(h1);
7773        park_backend_deadline(&mut connection, backend_deadline);
7774        mux.router.backends.insert(backend_token, connection);
7775        mux.context.link_stream(0, backend_token);
7776        mux.context.streams[0].back.consumed = true;
7777        if queued {
7778            mux.context.streams[0]
7779                .back
7780                .out
7781                .push_back(kawa::OutBlock::Store(kawa::Store::Static(b"pending")));
7782        }
7783        arm_frontend_at(&mut mux, Some(frontend_deadline));
7784        (mux, backend_token, peer, backend_peer)
7785    }
7786
7787    /// A backend timeout on a response whose bytes wait for a slow client
7788    /// does not cut it: Sōzu stopped reading that backend itself, so the
7789    /// backend is not inactive. The stream stays linked, the backend is
7790    /// re-armed, and the FRONTEND deadline is left exactly where it was, so
7791    /// a client that stopped reading still meets `front_timeout`
7792    /// (sozu-proxy/sozu#1906).
7793    ///
7794    /// To SEE THIS RED: drop the `response_waits_on_client` arm of the
7795    /// backend-timeout branch of `Mux::timeout_inner` — the stream is cut
7796    /// with `backend_response_timeout` — or drop its early
7797    /// `return StateResult::Continue`, and the frontend deadline moves.
7798    #[test]
7799    fn a_backend_timeout_defers_to_the_client_while_response_bytes_wait_for_it() {
7800        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7801        let frontend_deadline = Instant::now() + Duration::from_secs(45);
7802        let (mut mux, backend_token, _peer, _backend_peer) = mux_with_response_in_progress(
7803            &pool,
7804            true,
7805            frontend_deadline,
7806            Instant::now() - Duration::from_secs(1),
7807        );
7808        mux.reschedule();
7809        let mut metrics = SessionMetrics::new(None);
7810
7811        let before = Instant::now();
7812        let result = mux.timeout(backend_token, &mut metrics);
7813
7814        assert_eq!(result, StateResult::Continue);
7815        assert_eq!(
7816            mux.context.streams[0].state,
7817            StreamState::Linked(backend_token),
7818            "a response waiting on the client must stay linked to its backend"
7819        );
7820        assert_eq!(mux.context.streams[0].context.access_log_message, None);
7821        assert!(
7822            mux.router
7823                .backends
7824                .get(&backend_token)
7825                .and_then(|backend| backend.poll_timeout())
7826                .is_some_and(|deadline| deadline > before),
7827            "the deferred backend timer must be re-armed"
7828        );
7829        assert_eq!(
7830            mux.frontend.poll_timeout(),
7831            Some(frontend_deadline),
7832            "a deferral must not push the frontend deadline out"
7833        );
7834    }
7835
7836    /// A frontend timeout on a response the client has fully taken does not
7837    /// cut it: the response waits on its backend, and `back_timeout` bounds
7838    /// that wait. The stream stays linked to its backend and the frontend is
7839    /// re-armed (sozu-proxy/sozu#1906).
7840    ///
7841    /// To SEE THIS RED: drop the `!response_waits_on_client` arm of the
7842    /// frontend-timeout branch of `Mux::timeout_inner`.
7843    #[test]
7844    fn a_frontend_timeout_defers_to_the_backend_while_the_client_waits_on_it() {
7845        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7846        let (mut mux, backend_token, _peer, _backend_peer) = mux_with_response_in_progress(
7847            &pool,
7848            false,
7849            Instant::now() - Duration::from_secs(1),
7850            Instant::now() + Duration::from_secs(20),
7851        );
7852        let mut metrics = SessionMetrics::new(None);
7853
7854        let before = Instant::now();
7855        let result = mux.timeout(Token(0), &mut metrics);
7856
7857        assert_eq!(result, StateResult::Continue);
7858        assert_eq!(
7859            mux.context.streams[0].state,
7860            StreamState::Linked(backend_token),
7861            "a response waiting on its backend must stay linked"
7862        );
7863        assert_eq!(mux.context.streams[0].context.access_log_message, None);
7864        assert!(
7865            mux.frontend
7866                .poll_timeout()
7867                .is_some_and(|deadline| deadline > before),
7868            "the deferred frontend timer must be re-armed"
7869        );
7870    }
7871
7872    /// The complement: a frontend timeout on a response whose bytes wait for
7873    /// the client still cuts it, so a client that stopped reading cannot
7874    /// hold the session through the backend-side deferral
7875    /// (sozu-proxy/sozu#1906).
7876    #[test]
7877    fn a_frontend_timeout_still_cuts_a_response_waiting_on_the_client() {
7878        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7879        let (mut mux, backend_token, _peer, _backend_peer) = mux_with_response_in_progress(
7880            &pool,
7881            true,
7882            Instant::now() - Duration::from_secs(1),
7883            Instant::now() + Duration::from_secs(20),
7884        );
7885        let mut metrics = SessionMetrics::new(None);
7886
7887        let _ = mux.timeout(Token(0), &mut metrics);
7888
7889        assert_eq!(
7890            mux.context.streams[0].context.access_log_message,
7891            Some("client_timeout_during_response")
7892        );
7893        assert_ne!(
7894            mux.context.streams[0].state,
7895            StreamState::Linked(backend_token),
7896            "the cut stream must be unlinked"
7897        );
7898    }
7899
7900    /// The other half of the loose-end proof. In the "response terminated and
7901    /// fully proxied" arm of the backend-timeout branch there is no
7902    /// `unlink_stream` call of its own: the only thing that evicts the stream
7903    /// from `context.backend_streams` on this arm is the unconditional
7904    /// `backend.end_stream(...)` at the bottom of the loop. As above, this test
7905    /// sets `h1.stream = Some(0)` so `ConnectionH1::end_stream` takes its
7906    /// matched path; the mismatch arm is unreached, and guarded by
7907    /// `debug_assert_h1_owns_stream` rather than by a test. That arm also
7908    /// leaves `should_close` true and `should_write` false, which is exactly
7909    /// the `delay_close_for_frontend_flush("timeout")` path the issue named as
7910    /// the one that can return `Continue` with only the frontend re-armed.
7911    ///
7912    /// To SEE THIS RED: delete `backend.end_stream(stream_id, &mut
7913    /// self.context);` from the bottom of that loop.
7914    #[test]
7915    fn a_completed_backend_response_timeout_unlinks_through_end_stream_alone() {
7916        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
7917        let (mut mux, _peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
7918        let mut metrics = SessionMetrics::new(None);
7919        let backend_token = Token(1);
7920
7921        let (mut connection, _backend_peer) =
7922            test_backend_connection(&mut mux, Duration::from_secs(30));
7923        let Connection::H1(h1) = &mut connection else {
7924            unreachable!("new_h1_client builds an H1 connection")
7925        };
7926        h1.stream = Some(0);
7927        // A backend whose response was fully proxied is connected; see
7928        // `a_backend_timeout_leaves_no_linked_stream_behind_on_the_close_path`.
7929        mark_connected(h1);
7930        park_backend_deadline(&mut connection, Instant::now() - Duration::from_secs(1));
7931        mux.router.backends.insert(backend_token, connection);
7932        mux.reschedule();
7933
7934        mux.context.link_stream(0, backend_token);
7935        // Terminated AND fully proxied: the arm that does nothing but fall
7936        // through to `end_stream`.
7937        mux.context.streams[0].back.consumed = true;
7938        mux.context.streams[0].back.parsing_phase = kawa::ParsingPhase::Terminated;
7939        assert!(
7940            mux.context.streams[0].back.is_terminated()
7941                && mux.context.streams[0].back.is_completed(),
7942            "precondition: this test needs the terminated-and-completed arm"
7943        );
7944
7945        let _ = mux.timeout(backend_token, &mut metrics);
7946
7947        assert!(
7948            mux.context
7949                .backend_streams
7950                .get(&backend_token)
7951                .is_none_or(|ids| ids.is_empty()),
7952            "end_stream is the sole eviction point on this arm; without it the \
7953             backend reaches the shared tail still holding a linked stream and \
7954             with no timer of its own"
7955        );
7956        assert_eq!(
7957            mux.context.streams[0].context.access_log_message, None,
7958            "precondition: a fully proxied response is not a timeout outcome, \
7959             so this arm leaves should_write false and reaches \
7960             delay_close_for_frontend_flush"
7961        );
7962    }
7963
7964    /// Q7 of sozu-proxy/sozu#1340: a stream runs on the listener
7965    /// configuration that was in force when its request arrived, and the next
7966    /// request on the same connection picks up a reload that landed in
7967    /// between. The staleness window is therefore exactly one request, and
7968    /// this test is where that window is pinned — see `LIFECYCLE.md` §2.5.
7969    ///
7970    /// Both halves matter and they fail for opposite reasons:
7971    ///
7972    /// - freezing the listener at [`Context::new`] would satisfy the first
7973    ///   half and break the second — the reload would never reach the
7974    ///   connection at all, which is the construction-time snapshot the
7975    ///   maintainer rejected because hot reconfiguration is the point of this
7976    ///   proxy;
7977    /// - borrowing the listener live on the datapath satisfies the second and
7978    ///   breaks the first — a request already in flight would answer under a
7979    ///   cookie name, an `X-Real-IP` policy or an error page installed after
7980    ///   it started.
7981    ///
7982    /// To SEE THIS RED: give `Context` back an `elide_x_real_ip: bool` field
7983    /// set in [`Context::new`] from `listener.borrow().get_elide_x_real_ip()`,
7984    /// and make [`Context::create_stream`] pass `self.elide_x_real_ip` to
7985    /// `HttpContext::new` instead of `listener.get_elide_x_real_ip()`. The
7986    /// second stream then still carries the pre-reload value.
7987    #[test]
7988    fn a_listener_reload_between_two_streams_moves_only_the_second() {
7989        setup_test_logger!();
7990        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 10, 16_384)));
7991        let mut context = test_context(&pool);
7992
7993        let before = context.listener.borrow().get_answers().clone();
7994        let first = context
7995            .create_stream(Ulid::generate(), 1 << 16)
7996            .expect("the first stream must check out its buffers");
7997
7998        // The operator reloads the listener while the first request is still
7999        // in flight. `reload` publishes a new answer registry under a new
8000        // `Rc`, exactly as `HttpListener::update_config` does.
8001        let after = context.listener.borrow_mut().reload("RELOADEDID", true);
8002        assert!(
8003            !Rc::ptr_eq(&before, &after),
8004            "precondition: the reload must PUBLISH a registry, not rewrite the \
8005             captured one — otherwise this test cannot tell a snapshot from a \
8006             shared handle"
8007        );
8008
8009        let second = context
8010            .create_stream(Ulid::generate(), 1 << 16)
8011            .expect("the second stream must check out its buffers");
8012        assert_ne!(
8013            first, second,
8014            "precondition: the second request must land in its own slot, not \
8015             recycle the first one's"
8016        );
8017
8018        // The next request picks the reload up.
8019        assert_eq!(
8020            context.streams[second].context.sticky_name, "RELOADEDID",
8021            "a request arriving after the reload must use the new sticky name"
8022        );
8023        assert!(
8024            context.streams[second].context.elide_x_real_ip,
8025            "a request arriving after the reload must use the new X-Real-IP \
8026             policy: a listener frozen at Context::new never sees a reload on \
8027             a connection that is already open"
8028        );
8029        assert!(
8030            Rc::ptr_eq(&context.streams[second].answers, &after),
8031            "a request arriving after the reload must render from the newly \
8032             published answer registry"
8033        );
8034
8035        // The request already in flight does not.
8036        assert_eq!(
8037            context.streams[first].context.sticky_name, TEST_STICKY_NAME,
8038            "a request in flight keeps the sticky name it started under"
8039        );
8040        assert!(
8041            !context.streams[first].context.elide_x_real_ip,
8042            "a request in flight keeps the X-Real-IP policy it started under"
8043        );
8044        assert!(
8045            Rc::ptr_eq(&context.streams[first].answers, &before),
8046            "a request in flight keeps the answer registry it captured: a live \
8047             listener borrow on the write, reset or access-log path would hand \
8048             it a page installed after it started"
8049        );
8050    }
8051
8052    // ── peer-address snapshot in the MUX log envelope ────────────────────
8053    //
8054    // `log_context!` / `log_context_lite!` render `peer=` from
8055    // `Connection::peer_address` (`lib/src/protocol/mux/connection.rs`), the
8056    // snapshot the connection took at construction — not a live
8057    // `getpeername(2)`. Both tests below make the two answers differ on
8058    // purpose: the socket is genuinely connected to a loopback listener, so a
8059    // live lookup succeeds and would print `127.0.0.1:<port>`; only a macro
8060    // reading the snapshot can print `10.0.0.42:12345`.
8061    //
8062    // That gap is the PROXY-protocol symptom in miniature — the frontend's real
8063    // peer is the load balancer while the snapshot holds the advertised client
8064    // — and the same read is what keeps the slot populated after the peer's
8065    // RST, when `getpeername(2)` answers ENOTCONN and a live lookup collapses
8066    // to `None` on exactly the error lines an operator is reading. Until this
8067    // change the `MUX` envelope printed one peer while the `MUX-H1`, `MUX-H2`,
8068    // `SOCKET` and `HTTPS` lines of the same ULID printed another.
8069    //
8070    // Mirrors `h2::tests::log_context_renders_the_cached_peer_not_a_live_lookup`
8071    // and `h1::tests::log_context_renders_the_proxy_advertised_peer_not_the_load_balancer`,
8072    // which pin the same slot one layer down in the per-protocol envelopes.
8073    //
8074    // A counting test would not discriminate here, unlike
8075    // `h1::tests::log_context_reads_the_peer_address_once_per_connection`:
8076    // before this change the macros never reached `SocketHandler::peer_addr` at
8077    // all — they went through `Connection::socket()` to mio's inherent method —
8078    // so a trait-call counter reads zero on both sides. The assertions are
8079    // therefore on the rendered VALUE.
8080
8081    /// A live, established loopback connection plus the address
8082    /// `getpeername(2)` reports for it. The listener is returned so the
8083    /// connection stays up for the whole test: these tests assert the snapshot
8084    /// wins even while the live lookup is perfectly healthy, so a half-dead
8085    /// socket would weaken them rather than strengthen them.
8086    ///
8087    /// Connects with a blocking `std` socket and converts afterwards rather
8088    /// than reusing `connected_socket`, because `mio::net::TcpStream::connect`
8089    /// returns before the handshake completes and the live lookup below is a
8090    /// load-bearing premise, not a convenience.
8091    fn connected_loopback_stream() -> (std::net::TcpListener, TcpStream, SocketAddr) {
8092        let listener = std::net::TcpListener::bind("127.0.0.1:0")
8093            .expect("test listener must bind to a loopback port");
8094        let live_peer = listener
8095            .local_addr()
8096            .expect("test listener must report its local address");
8097        let stream =
8098            std::net::TcpStream::connect(live_peer).expect("loopback connect must complete");
8099        stream
8100            .set_nonblocking(true)
8101            .expect("mio requires a nonblocking stream");
8102        (listener, TcpStream::from_std(stream), live_peer)
8103    }
8104
8105    /// Address the frontend handler snapshots. Deliberately non-loopback so it
8106    /// cannot collide with whatever ephemeral port the live lookup reports.
8107    const SNAPSHOT_PEER: &str = "10.0.0.42:12345";
8108
8109    /// An H1 frontend `Mux` whose connection snapshotted [`SNAPSHOT_PEER`]
8110    /// while its socket is really connected to `live_peer`.
8111    /// `Connection::new_h1_server` takes that snapshot itself, from the
8112    /// handler's `SocketHandler::peer_addr`, so nothing here writes the field
8113    /// behind the production path's back.
8114    ///
8115    /// An H1 frontend only because it is the cheaper of the two to build: the
8116    /// accessor under test resolves both `Connection` arms to the same
8117    /// `peer_address` field through `forward!`, and the two macros are defined
8118    /// once for every frontend.
8119    fn mux_with_snapshotted_peer(
8120        pool: &Rc<RefCell<Pool>>,
8121        stream: TcpStream,
8122    ) -> Mux<SessionTcpStream, test_support::TestListener> {
8123        let session_ulid = Ulid::generate();
8124        let socket = SessionTcpStream::new(
8125            stream,
8126            session_ulid,
8127            Some(
8128                SNAPSHOT_PEER
8129                    .parse()
8130                    .expect("the snapshotted peer literal must parse"),
8131            ),
8132        );
8133        Mux {
8134            configured_frontend_timeout: Duration::from_secs(60),
8135            frontend_token: Token(0),
8136            frontend: Connection::new_h1_server(session_ulid, socket, Duration::from_secs(60)),
8137            router: Router::new(Duration::from_secs(30), Duration::from_secs(30)),
8138            context: test_context(pool),
8139            session_ulid,
8140            timeouts: MuxTimeouts::new(TimeoutContainer::new_empty(Duration::from_secs(60))),
8141            backend_registry: BackendRegistry::default(),
8142            backends: Rc::default(),
8143        }
8144    }
8145
8146    /// To SEE THIS RED: in `log_context!` (mod.rs), put
8147    /// `peer = $self.frontend.socket().peer_addr().ok(),` back in place of
8148    /// `peer = $self.frontend.peer_address(),`. `Connection::socket()` hands
8149    /// back a `&mio::net::TcpStream`, so the slot resolves to mio's *inherent*
8150    /// `peer_addr` — a live `getpeername(2)` — and the rendered line carries
8151    /// the loopback address the socket is really connected to.
8152    #[test]
8153    fn log_context_renders_the_snapshotted_peer_not_a_live_lookup() {
8154        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
8155        let (_listener, stream, live_peer) = connected_loopback_stream();
8156        let mux = mux_with_snapshotted_peer(&pool, stream);
8157
8158        // Premise of the test: the live lookup is healthy and disagrees with the
8159        // snapshot. Without it the assertions below could pass for the wrong
8160        // reason (both answers happening to be the same address).
8161        assert_eq!(
8162            mux.frontend.socket().peer_addr().ok(),
8163            Some(live_peer),
8164            "the test socket must be genuinely connected, so a live lookup succeeds"
8165        );
8166        assert_ne!(
8167            SNAPSHOT_PEER,
8168            live_peer.to_string(),
8169            "the snapshotted and live addresses must differ for this test to discriminate"
8170        );
8171
8172        let rendered = log_context!(mux);
8173
8174        assert!(
8175            rendered.contains(&format!("peer=Some({SNAPSHOT_PEER})")),
8176            "the MUX peer= slot must render the snapshotted address; rendered: {rendered}"
8177        );
8178        assert!(
8179            !rendered.contains(&live_peer.to_string()),
8180            "the MUX peer= slot must not fall back to a live getpeername(2); rendered: {rendered}"
8181        );
8182    }
8183
8184    /// The lighter envelope reads the same slot. Separate from its full
8185    /// sibling because the two macros carry their own copy of the `peer` line:
8186    /// fixing one and not the other would render two different peers for one
8187    /// session depending only on which borrow the callsite happened to hold.
8188    ///
8189    /// To SEE THIS RED: in `log_context_lite!` (mod.rs), put
8190    /// `peer = $self.frontend.socket().peer_addr().ok(),` back in place of
8191    /// `peer = $self.frontend.peer_address(),`.
8192    #[test]
8193    fn log_context_lite_renders_the_snapshotted_peer_not_a_live_lookup() {
8194        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
8195        let (_listener, stream, live_peer) = connected_loopback_stream();
8196        let mux = mux_with_snapshotted_peer(&pool, stream);
8197
8198        assert_eq!(
8199            mux.frontend.socket().peer_addr().ok(),
8200            Some(live_peer),
8201            "the test socket must be genuinely connected, so a live lookup succeeds"
8202        );
8203
8204        let rendered = log_context_lite!(mux);
8205
8206        assert!(
8207            rendered.contains(&format!("peer=Some({SNAPSHOT_PEER})")),
8208            "the lite MUX peer= slot must render the snapshotted address; rendered: {rendered}"
8209        );
8210        assert!(
8211            !rendered.contains(&live_peer.to_string()),
8212            "the lite MUX peer= slot must not fall back to a live getpeername(2); \
8213             rendered: {rendered}"
8214        );
8215    }
8216
8217    /// `Mux::close` issues no `EPOLL_CTL_DEL` for its backend sockets: each
8218    /// stays in the epoll set until the `Mux` drops, and its last close is what
8219    /// removes it. The second assertion is the half that makes skipping the
8220    /// deregister safe — the socket is not duplicated anywhere, so dropping the
8221    /// `Mux` really does take it out of the epoll set.
8222    ///
8223    /// To SEE THIS RED: make `Mux::close`'s backend loop issue an
8224    /// `EPOLL_CTL_DEL` after `let socket = client.socket_mut();`. `L7Proxy`
8225    /// no longer carries `deregister_socket` (sozu#1615), so restore it for
8226    /// the experiment — its `HttpProxy` body was
8227    /// `self.registry.deregister(tcp_stream)` — and call
8228    /// `let _ = proxy_borrow.deregister_socket(socket);` there. The first
8229    /// assertion then fails.
8230    #[cfg(target_os = "linux")]
8231    #[test]
8232    fn close_leaves_backend_sockets_to_their_last_close() {
8233        use std::os::fd::AsRawFd;
8234
8235        use mio::Interest;
8236
8237        use super::test_support::epoll_watches;
8238        use crate::{backends::Backend, http::HttpProxy, socket::SessionTcpStream};
8239
8240        let pool = Rc::new(RefCell::new(Pool::with_capacity(2, 4, 16384)));
8241        let (mut mux, _front_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(30));
8242        let parts =
8243            crate::testing::prebuild_server(4, 16_384, false).expect("test server must build");
8244        let registry = parts
8245            .registry
8246            .try_clone()
8247            .expect("the test registry must clone");
8248        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(HttpProxy::new(
8249            parts.registry,
8250            parts.sessions,
8251            parts.pool,
8252            parts.backends,
8253        )));
8254
8255        let (socket, _back_peer) = connected_socket();
8256        let backend = Rc::new(RefCell::new(Backend::new(
8257            "test-backend",
8258            "127.0.0.1:2".parse().expect("backend address must parse"),
8259            None,
8260            None,
8261            None,
8262        )));
8263        let backend_id = mux.backend_registry.id_for(&backend);
8264        let session_ulid = mux.session_ulid;
8265        let mut connection = Connection::new_h1_client(
8266            session_ulid,
8267            SessionTcpStream::new(socket, session_ulid, None),
8268            "test-cluster".into(),
8269            backend_id,
8270            Duration::from_secs(30),
8271        );
8272        let back_fd = connection.socket().as_raw_fd();
8273        registry
8274            .register(
8275                connection.socket_mut(),
8276                Token(7),
8277                Interest::READABLE | Interest::WRITABLE,
8278            )
8279            .expect("the backend socket must register");
8280        mux.router.backends.insert(Token(7), connection);
8281        assert!(
8282            epoll_watches(&registry, back_fd),
8283            "precondition: the backend socket is in the epoll set"
8284        );
8285
8286        mux.close(proxy, &mut SessionMetrics::new(None));
8287        assert!(
8288            epoll_watches(&registry, back_fd),
8289            "Mux::close must not deregister a backend socket it is about to \
8290             close: the close removes it, the EPOLL_CTL_DEL is a wasted syscall"
8291        );
8292
8293        drop(mux);
8294        assert!(
8295            !epoll_watches(&registry, back_fd),
8296            "dropping the Mux closes the backend socket's only descriptor, \
8297             which must take it out of the epoll set"
8298        );
8299        drop(parts.event_loop);
8300    }
8301
8302    // ── a read that meets the EOF records HUP ──────────────────────────
8303
8304    /// An H1 backend read that meets the EOF records HUP; the same read on
8305    /// the frontend leaves HUP to the event loop.
8306    ///
8307    /// Both peers write bytes and their FIN before the read. Since
8308    /// sozu-proxy/sozu#1606 a read stops on a short read, as HAProxy does
8309    /// (`src/raw_sock.c`, `fd_cant_recv` when `ret < try`): the first
8310    /// `readable` delivers the bytes and stops before the EOF, and the EOF is
8311    /// read by the next `readable`, which the poller's `EPOLLRDHUP` edge (or a
8312    /// HUP already in the same event) triggers. That second read is the one
8313    /// that meets the EOF and must record HUP.
8314    ///
8315    /// TO SEE THIS RED: delete the HUP insertion from `ConnectionH1::readable`
8316    /// (`h1.rs`). The backend assertion fails with `a backend read that met the
8317    /// EOF must record HUP`.
8318    #[test]
8319    fn an_h1_backend_eof_read_records_hup_and_a_frontend_one_does_not() {
8320        use std::io::Write;
8321
8322        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8323        let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8324        let (mut backend, mut backend_peer) =
8325            test_backend_connection(&mut mux, Duration::from_secs(30));
8326        let Connection::H1(h1) = &mut backend else {
8327            unreachable!("new_h1_client builds an H1 connection")
8328        };
8329        h1.stream = Some(0);
8330        mux.context.link_stream(0, Token(1));
8331
8332        backend_peer
8333            .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello")
8334            .expect("the backend peer must write its response");
8335        backend_peer
8336            .shutdown(std::net::Shutdown::Write)
8337            .expect("the backend peer must send its FIN");
8338        std::thread::sleep(Duration::from_millis(20));
8339        backend.readiness_mut().event = Ready::READABLE;
8340        let _ = backend.readable(&mut mux.context, EndpointServer(&mut mux.frontend));
8341        assert!(
8342            mux.context.streams[0].back.is_terminated(),
8343            "the bytes read before the EOF must be parsed"
8344        );
8345        assert!(
8346            !backend.readiness().event.is_hup(),
8347            "the short read stops before the EOF, so it has not met it (#1606)"
8348        );
8349        let _ = backend.readable(&mut mux.context, EndpointServer(&mut mux.frontend));
8350        assert!(
8351            backend.readiness().event.is_hup(),
8352            "a backend read that met the EOF must record HUP"
8353        );
8354
8355        // The same stream carries the frontend's request.
8356        mux.context.streams[0].state = StreamState::Idle;
8357        frontend_peer
8358            .write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\n\r\n")
8359            .expect("the client must write its request");
8360        frontend_peer
8361            .shutdown(std::net::Shutdown::Write)
8362            .expect("the client must send its FIN");
8363        std::thread::sleep(Duration::from_millis(20));
8364        mux.frontend.readiness_mut().event = Ready::READABLE;
8365        // The first read takes the request and stops short (#1606); the
8366        // second meets the EOF.
8367        let _ = mux
8368            .frontend
8369            .readable(&mut mux.context, EndpointClient(&mut mux.router));
8370        let _ = mux
8371            .frontend
8372            .readable(&mut mux.context, EndpointClient(&mut mux.router));
8373        assert!(
8374            !mux.frontend.readiness().event.is_readable(),
8375            "precondition: the frontend read reached the EOF"
8376        );
8377        assert!(
8378            !mux.frontend.readiness().event.is_hup(),
8379            "a frontend read must leave HUP to the event loop"
8380        );
8381    }
8382
8383    /// A `ProxySession` for a `Mux::ready` pass that links no stream: only
8384    /// `Mux::dial_backend` reads the session, and nothing in these tests
8385    /// dials.
8386    struct NoDialSession;
8387
8388    impl ProxySession for NoDialSession {
8389        fn protocol(&self) -> Protocol {
8390            unreachable!("a Mux pass that dials nothing never asks the session")
8391        }
8392        fn ready(&mut self, _session: Rc<RefCell<dyn ProxySession>>) -> SessionIsToBeClosed {
8393            unreachable!("a Mux pass that dials nothing never asks the session")
8394        }
8395        fn update_readiness(&mut self, _token: Token, _events: Ready) {
8396            unreachable!("a Mux pass that dials nothing never asks the session")
8397        }
8398        fn close(&mut self) {
8399            unreachable!("a Mux pass that dials nothing never asks the session")
8400        }
8401        fn timeout(&mut self, _t: Token) -> SessionIsToBeClosed {
8402            unreachable!("a Mux pass that dials nothing never asks the session")
8403        }
8404        fn last_event(&self) -> Instant {
8405            unreachable!("a Mux pass that dials nothing never asks the session")
8406        }
8407        fn print_session(&self) {
8408            unreachable!("a Mux pass that dials nothing never asks the session")
8409        }
8410        fn frontend_token(&self) -> Token {
8411            unreachable!("a Mux pass that dials nothing never asks the session")
8412        }
8413        fn shutting_down(&mut self) -> SessionIsToBeClosed {
8414            unreachable!("a Mux pass that dials nothing never asks the session")
8415        }
8416    }
8417
8418    /// An `L7Proxy` for a session that only ever removes backend sessions:
8419    /// the dead-backend sweep of `Mux::ready_inner` and `Mux::close` call
8420    /// `remove_session` once per backend and nothing else.
8421    struct RemoveOnlyProxy;
8422
8423    impl L7Proxy for RemoveOnlyProxy {
8424        fn kind(&self) -> sozu_command::proto::command::ListenerType {
8425            unreachable!("closing a backend only removes its session")
8426        }
8427        fn register_socket(
8428            &self,
8429            _socket: &mut TcpStream,
8430            _token: Token,
8431            _interest: Interest,
8432        ) -> Result<(), std::io::Error> {
8433            unreachable!("closing a backend only removes its session")
8434        }
8435        fn add_session(&self, _session: Rc<RefCell<dyn ProxySession>>) -> Token {
8436            unreachable!("closing a backend only removes its session")
8437        }
8438        fn remove_session(&self, _token: Token) -> bool {
8439            true
8440        }
8441        fn clusters(
8442            &self,
8443        ) -> &HashMap<sozu_command::state::ClusterId, sozu_command::proto::command::Cluster>
8444        {
8445            unreachable!("closing a backend only removes its session")
8446        }
8447        fn sessions(&self) -> Rc<RefCell<crate::server::SessionManager>> {
8448            unreachable!("closing a backend only removes its session")
8449        }
8450    }
8451
8452    /// An `L7Proxy` for a pass that must have released every stream before
8453    /// the `pending_links` pass of `Mux::ready_inner` runs: removing a
8454    /// backend session is allowed, and reaching the routing view or
8455    /// registering a backend names the regression.
8456    struct NoRoutingProxy;
8457
8458    impl L7Proxy for NoRoutingProxy {
8459        fn kind(&self) -> sozu_command::proto::command::ListenerType {
8460            unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8461        }
8462        fn register_socket(
8463            &self,
8464            _socket: &mut TcpStream,
8465            _token: Token,
8466            _interest: Interest,
8467        ) -> Result<(), std::io::Error> {
8468            unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8469        }
8470        fn add_session(&self, _session: Rc<RefCell<dyn ProxySession>>) -> Token {
8471            unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8472        }
8473        fn remove_session(&self, _token: Token) -> bool {
8474            true
8475        }
8476        fn clusters(
8477            &self,
8478        ) -> &HashMap<sozu_command::state::ClusterId, sozu_command::proto::command::Cluster>
8479        {
8480            unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8481        }
8482        fn sessions(&self) -> Rc<RefCell<crate::server::SessionManager>> {
8483            unreachable!("pending_links routed a stream of an error-GOAWAY linger")
8484        }
8485    }
8486
8487    /// Read what the peer has been sent until it sees the FIN, or until the
8488    /// bytes read end with a non-empty `until`, and report whether the FIN was
8489    /// seen. Bounded: a peer that sees neither within the deadline reports
8490    /// what it has.
8491    fn drain_peer(peer: &mut std::net::TcpStream, until: &[u8]) -> (Vec<u8>, bool) {
8492        use std::io::Read;
8493        let deadline = Instant::now() + Duration::from_secs(2);
8494        let mut bytes = Vec::new();
8495        let mut chunk = [0u8; 4096];
8496        loop {
8497            match peer.read(&mut chunk) {
8498                Ok(0) => return (bytes, true),
8499                Ok(n) => {
8500                    bytes.extend_from_slice(&chunk[..n]);
8501                    if !until.is_empty() && bytes.ends_with(until) {
8502                        return (bytes, false);
8503                    }
8504                }
8505                Err(e) if e.kind() == ErrorKind::WouldBlock => {
8506                    if Instant::now() >= deadline {
8507                        return (bytes, false);
8508                    }
8509                    std::thread::sleep(Duration::from_millis(5));
8510                }
8511                Err(e) => panic!("the test peer failed to read: {e}"),
8512            }
8513        }
8514    }
8515
8516    /// #1610: a session whose backend connections die one pass after another
8517    /// sizes its dead-backend list once, then reuses it.
8518    ///
8519    /// Each pass closes one idle keep-alive backend whose event carries HUP,
8520    /// the shape a backend that closes after every response leaves on a
8521    /// keep-alive frontend. Past the first such pass, a pass allocates
8522    /// nothing at all, so the list must not either.
8523    ///
8524    /// TO SEE THIS RED: in `Mux::ready_inner`, collect the dead tokens in a
8525    /// local `Vec::new()` again instead of `self.router.dead_backends`. Every
8526    /// pass that closes a backend then allocates the list.
8527    #[test]
8528    fn successive_dead_backends_reuse_one_dead_backend_list() {
8529        use crate::test_allocations::allocations;
8530
8531        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8532        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8533        mux.frontend.readiness_mut().event = Ready::EMPTY;
8534        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
8535        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
8536        let mut metrics = SessionMetrics::new(None);
8537
8538        let mut close_one = |mux: &mut Mux<_, _>, token: Token| {
8539            let (mut connection, backend_peer) =
8540                test_backend_connection(mux, Duration::from_secs(30));
8541            let Connection::H1(h1) = &mut connection else {
8542                unreachable!("new_h1_client builds an H1 connection")
8543            };
8544            if let Position::Client(_, _, status) = &mut h1.position {
8545                *status = BackendStatus::KeepAlive;
8546            }
8547            h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8548            h1.readiness.event = Ready::HUP;
8549            mux.router.backends.insert(token, connection);
8550            drop(backend_peer);
8551
8552            let before = allocations();
8553            let _ = mux.ready(session.clone(), proxy.clone(), &mut metrics);
8554            let allocated = allocations() - before;
8555            assert!(
8556                !mux.router.backends.contains_key(&token),
8557                "premise: the HUP must close the backend in this pass"
8558            );
8559            allocated
8560        };
8561
8562        // Warm-up: the session's first dead backend sizes the list, and the
8563        // thread's first pass initialises state that outlives the session.
8564        close_one(&mut mux, Token(1));
8565        for token in [Token(2), Token(3)] {
8566            assert_eq!(
8567                close_one(&mut mux, token),
8568                0,
8569                "a later pass that closes a backend must not allocate"
8570            );
8571        }
8572    }
8573
8574    /// Issues #1603 and #1606: a backend whose last bytes and FIN reached the
8575    /// kernel before `epoll_wait` returned is closed in that same pass, after
8576    /// its bytes were parsed and forwarded.
8577    ///
8578    /// The event carries READABLE and HUP together, as
8579    /// `Ready::from(&mio::event::Event)` builds it for an `EPOLLIN|EPOLLRDHUP`
8580    /// edge (`data_and_fin_in_one_edge_keep_readable_until_the_eof_is_read`
8581    /// shows a real socket reporting both in one edge). The first read stops
8582    /// short before the EOF (#1606, as HAProxy does in `src/raw_sock.c`), and
8583    /// the dead-backend check of `Mux::ready_inner`, which reads the HUP the
8584    /// edge already carried, closes the backend on the pass's next inner
8585    /// iteration. The bytes read before the EOF are forwarded first. (When the
8586    /// stream still needs the EOF, as a close-delimited body does,
8587    /// `update_readiness_after_read` keeps READABLE so it is read first:
8588    /// `data_and_fin_in_one_edge_keep_readable_until_the_eof_is_read`.)
8589    ///
8590    /// TO SEE THIS RED: drop `readiness.event.is_hup() ||` from `dead` in
8591    /// `Mux::ready_inner`. The backend then survives the pass: `the backend
8592    /// that read its EOF must be closed in the same pass`.
8593    #[test]
8594    fn a_backend_eof_read_with_its_last_bytes_closes_the_backend_in_the_same_pass() {
8595        use std::io::Write;
8596
8597        const RESPONSE: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello";
8598
8599        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8600        let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8601        let backend_token = Token(1);
8602        let (mut connection, mut backend_peer) =
8603            test_backend_connection(&mut mux, Duration::from_secs(30));
8604        let Connection::H1(h1) = &mut connection else {
8605            unreachable!("new_h1_client builds an H1 connection")
8606        };
8607        h1.stream = Some(0);
8608        if let Position::Client(_, _, status) = &mut h1.position {
8609            *status = BackendStatus::Connected;
8610        }
8611        // The request is already written; the backend waits to read. Data and
8612        // FIN share one edge.
8613        h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8614        h1.readiness.event = Ready::READABLE | Ready::HUP;
8615        mux.router.backends.insert(backend_token, connection);
8616        mux.context.link_stream(0, backend_token);
8617        mux.frontend.readiness_mut().event = Ready::EMPTY;
8618
8619        backend_peer
8620            .write_all(RESPONSE)
8621            .expect("the backend peer must write its response");
8622        backend_peer
8623            .shutdown(std::net::Shutdown::Write)
8624            .expect("the backend peer must send its FIN");
8625        // Loopback delivers both before `write`/`shutdown` return; the margin
8626        // only guards a scheduler that defers the softirq.
8627        std::thread::sleep(Duration::from_millis(20));
8628
8629        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
8630        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
8631        let mut metrics = SessionMetrics::new(None);
8632        let _ = mux.ready(session, proxy, &mut metrics);
8633
8634        assert!(
8635            !mux.router.backends.contains_key(&backend_token),
8636            "the backend that read its EOF must be closed in the same pass"
8637        );
8638        let (forwarded, _) = drain_peer(&mut frontend_peer, b"hello");
8639        assert!(
8640            forwarded.ends_with(b"hello"),
8641            "the bytes read together with the EOF must still reach the client, \
8642             got {:?}",
8643            String::from_utf8_lossy(&forwarded)
8644        );
8645    }
8646
8647    /// Issues #1603 and #1606: the FIN lands AFTER the `epoll_wait` that
8648    /// woke the pass, so the event carries READABLE without HUP. The pass
8649    /// reads the response in one short read and stops before the EOF (#1606):
8650    /// it does not issue the `recv` that would find the FIN, exactly as
8651    /// HAProxy stops on a short read and lets the poller report the EOF
8652    /// (`src/raw_sock.c`, `fd_cant_recv` when `ret < try`). The backend is
8653    /// therefore closed one `epoll_wait` round later, on the `EPOLLRDHUP` edge
8654    /// the FIN queued — and not later: that edge is already pending when the
8655    /// pass ends, so a zero-timeout poll returns it.
8656    ///
8657    /// That round is the price of not issuing a `recv` after every short
8658    /// read, which answers EAGAIN whenever no FIN follows (the common case).
8659    /// Real loopback socket, registered edge-triggered with mio.
8660    #[test]
8661    fn a_fin_after_epoll_wait_closes_the_backend_on_the_next_round_at_the_latest() {
8662        use std::io::Write;
8663
8664        const RESPONSE: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello";
8665
8666        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8667        let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8668        let backend_token = Token(1);
8669        let (mut connection, mut backend_peer) =
8670            test_backend_connection(&mut mux, Duration::from_secs(30));
8671        let mut poll = mio::Poll::new().expect("a poll instance must open");
8672        let mut events = mio::Events::with_capacity(8);
8673        let Connection::H1(h1) = &mut connection else {
8674            unreachable!("new_h1_client builds an H1 connection")
8675        };
8676        poll.registry()
8677            .register(
8678                &mut h1.socket.stream,
8679                backend_token,
8680                mio::Interest::READABLE,
8681            )
8682            .expect("the backend socket must register");
8683        h1.stream = Some(0);
8684        if let Position::Client(_, _, status) = &mut h1.position {
8685            *status = BackendStatus::Connected;
8686        }
8687        h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8688
8689        // The response arrives; `epoll_wait` reports it, without a FIN.
8690        backend_peer
8691            .write_all(RESPONSE)
8692            .expect("the backend peer must write its response");
8693        poll.poll(&mut events, Some(Duration::from_secs(5)))
8694            .expect("poll must succeed");
8695        let event = events
8696            .iter()
8697            .find(|event| event.token() == backend_token)
8698            .expect("the response must raise an edge");
8699        assert!(
8700            event.is_readable() && !event.is_read_closed(),
8701            "premise: no FIN yet in the edge that wakes the pass"
8702        );
8703        h1.readiness.event = Ready::from(event);
8704        mux.router.backends.insert(backend_token, connection);
8705        mux.context.link_stream(0, backend_token);
8706        // A request received whole, from a client that asked to close: the
8707        // response closes the frontend without a lingering drain (which
8708        // would end this backend's stream at once).
8709        mux.context.streams[0].front.parsing_phase = kawa::ParsingPhase::Terminated;
8710        mux.context.streams[0].context.keep_alive_frontend = false;
8711        mux.frontend.readiness_mut().event = Ready::EMPTY;
8712
8713        // The FIN lands after `epoll_wait` returned, before the pass reads.
8714        backend_peer
8715            .shutdown(std::net::Shutdown::Write)
8716            .expect("the backend peer must send its FIN");
8717        std::thread::sleep(Duration::from_millis(20));
8718
8719        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
8720        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
8721        let mut metrics = SessionMetrics::new(None);
8722        let _ = mux.ready(session.clone(), proxy.clone(), &mut metrics);
8723        let (forwarded, _) = drain_peer(&mut frontend_peer, b"hello");
8724        assert!(
8725            forwarded.ends_with(b"hello"),
8726            "the response must reach the client in the first pass, got {:?}",
8727            String::from_utf8_lossy(&forwarded)
8728        );
8729        assert!(
8730            mux.router.backends.contains_key(&backend_token),
8731            "the short read stopped before the EOF: the close waits one round (#1606)"
8732        );
8733
8734        // One round: the RDHUP edge is already queued, a zero-timeout poll
8735        // returns it.
8736        poll.poll(&mut events, Some(Duration::ZERO))
8737            .expect("poll must succeed");
8738        let event = events
8739            .iter()
8740            .find(|event| event.token() == backend_token)
8741            .expect("the FIN's edge must already be pending when the pass ends");
8742        assert!(event.is_read_closed(), "the pending edge is the FIN");
8743        mux.update_readiness(backend_token, Ready::from(event));
8744        let _ = mux.ready(session, proxy, &mut metrics);
8745        assert!(
8746            !mux.router.backends.contains_key(&backend_token),
8747            "the backend must be closed on that round, not later"
8748        );
8749    }
8750
8751    // ── a backend that closed with its response still buffered ─────────
8752
8753    /// Drive one H1 exchange whose backend wrote its whole response and
8754    /// closed before a slow client read it, and return the number of body
8755    /// bytes the client received, the body length, the `SessionResult` of
8756    /// the first `Mux::ready` pass, and how many times the exchange reached
8757    /// the loop limit (`LOOP_LIMIT_HITS`).
8758    ///
8759    /// The backend's last bytes and its FIN wait in the kernel while the
8760    /// stream's response buffer is full and the client's socket would block:
8761    /// the dead-backend check keeps the backend for its buffered bytes, and
8762    /// nothing in the pass can progress until the client reads. With
8763    /// `client_half_closed`, the client also shut its write side after its
8764    /// request, as `test_plain_client_half_close_after_request_receives_large_response`
8765    /// does.
8766    fn slow_client_after_backend_close(
8767        client_half_closed: bool,
8768    ) -> (usize, usize, SessionResult, u32) {
8769        use std::io::{Read, Write};
8770
8771        const BODY: usize = 160 * 1024;
8772        let head = format!("HTTP/1.1 200 OK\r\nContent-Length: {BODY}\r\n\r\n");
8773        let length = head.len() + BODY;
8774
8775        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
8776        let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8777        // A slow client: small buffers on both ends of the frontend, so its
8778        // socket would block long before the response is out.
8779        socket2::SockRef::from(&frontend_peer)
8780            .set_recv_buffer_size(16 * 1024)
8781            .expect("the client receive buffer must be settable");
8782        socket2::SockRef::from(mux.frontend.socket())
8783            .set_send_buffer_size(4096)
8784            .expect("the frontend send buffer must be settable");
8785
8786        let backend_token = Token(1);
8787        let (mut connection, mut backend_peer) =
8788            test_backend_connection(&mut mux, Duration::from_secs(30));
8789        // Room in the kernel for the whole response, so the backend can
8790        // write all of it and close before sozu reads it.
8791        socket2::SockRef::from(connection.socket())
8792            .set_recv_buffer_size(1 << 20)
8793            .expect("the backend receive buffer must be settable");
8794        socket2::SockRef::from(&backend_peer)
8795            .set_send_buffer_size(1 << 20)
8796            .expect("the backend peer send buffer must be settable");
8797        let Connection::H1(h1) = &mut connection else {
8798            unreachable!("new_h1_client builds an H1 connection")
8799        };
8800        h1.stream = Some(0);
8801        if let Position::Client(_, _, status) = &mut h1.position {
8802            *status = BackendStatus::Connected;
8803        }
8804        h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8805        h1.readiness.event = Ready::READABLE | Ready::HUP;
8806        mux.router.backends.insert(backend_token, connection);
8807        mux.context.link_stream(0, backend_token);
8808        // The request was received whole and forwarded.
8809        mux.context.streams[0].front.parsing_phase = kawa::ParsingPhase::Terminated;
8810        mux.frontend.readiness_mut().event = if client_half_closed {
8811            Ready::HUP
8812        } else {
8813            Ready::EMPTY
8814        };
8815
8816        let mut response = head.into_bytes();
8817        response.resize(length, b'b');
8818        let mut written = 0;
8819        let deadline = Instant::now() + Duration::from_secs(5);
8820        while written < response.len() {
8821            match backend_peer.write(&response[written..]) {
8822                Ok(size) => written += size,
8823                Err(e) if e.kind() == ErrorKind::WouldBlock => {
8824                    assert!(
8825                        Instant::now() < deadline,
8826                        "premise: the kernel must hold the whole response, wrote {written}"
8827                    );
8828                    std::thread::sleep(Duration::from_millis(1));
8829                }
8830                Err(e) => panic!("the backend peer failed to write: {e}"),
8831            }
8832        }
8833        backend_peer
8834            .shutdown(std::net::Shutdown::Write)
8835            .expect("the backend peer must send its FIN");
8836        std::thread::sleep(Duration::from_millis(20));
8837
8838        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
8839        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
8840        let mut metrics = SessionMetrics::new(None);
8841
8842        LOOP_LIMIT_HITS.with(|hits| hits.set(0));
8843        let first = mux.ready(session.clone(), proxy.clone(), &mut metrics);
8844        let mut result = first;
8845        let mut received = Vec::new();
8846        let mut chunk = [0u8; 4096];
8847        let deadline = Instant::now() + Duration::from_secs(10);
8848        // sozu adds its own response headers: the body is what follows the
8849        // head the client received.
8850        let body_received = |received: &[u8]| {
8851            received
8852                .windows(4)
8853                .position(|window| window == b"\r\n\r\n")
8854                .map_or(0, |end| received.len() - end - 4)
8855        };
8856        loop {
8857            // The client reads what it can, then the poller reports the
8858            // frontend writable again, as epoll does once the client drains.
8859            loop {
8860                match frontend_peer.read(&mut chunk) {
8861                    Ok(0) => break,
8862                    Ok(size) => received.extend_from_slice(&chunk[..size]),
8863                    Err(e) if e.kind() == ErrorKind::WouldBlock => break,
8864                    Err(e) => panic!("the client failed to read: {e}"),
8865                }
8866            }
8867            if body_received(&received) >= BODY
8868                || result == SessionResult::Close
8869                || Instant::now() >= deadline
8870            {
8871                break;
8872            }
8873            mux.update_readiness(Token(0), Ready::WRITABLE);
8874            result = mux.ready(session.clone(), proxy.clone(), &mut metrics);
8875        }
8876        if body_received(&received) < BODY {
8877            // Whatever the last pass queued in the kernel reaches the client.
8878            let (rest, _) = drain_peer(&mut frontend_peer, &[]);
8879            received.extend_from_slice(&rest);
8880        }
8881        let hits = LOOP_LIMIT_HITS.with(|hits| hits.get());
8882        assert!(
8883            mux.context.client_bytes_out >= received.len(),
8884            "every byte the client received was counted as written to it: {} < {}",
8885            mux.context.client_bytes_out,
8886            received.len()
8887        );
8888        (body_received(&received), BODY, first, hits)
8889    }
8890
8891    /// A slow client downloading a response whose backend has already
8892    /// written all of it and closed must receive the whole response.
8893    ///
8894    /// The kept backend's HUP stays in its `filter_interest()`, which is
8895    /// never empty while HUP is in its interest: the inner loop of
8896    /// `Mux::ready_inner` used to count that HUP as work, spin to
8897    /// `MAX_LOOP_ITERATIONS` with nothing to progress, and close the session
8898    /// with the end of the body still in the stream's buffer.
8899    ///
8900    /// TO SEE THIS RED: make `Connection::has_loop_work` return
8901    /// `!readiness.filter_interest().is_empty()` again. The first pass then
8902    /// runs to `MAX_LOOP_ITERATIONS`. With `Mux::wait_for_client_at_loop_limit`
8903    /// also back to checking `frontend.has_pending_write()` alone, that pass
8904    /// returns `SessionResult::Close` and the client is short of bytes.
8905    #[test]
8906    fn a_slow_client_receives_the_whole_response_of_a_backend_that_closed() {
8907        let (received, expected, first, hits) = slow_client_after_backend_close(false);
8908        assert_eq!(
8909            hits, 0,
8910            "every pass must stop once nothing can progress, not at the loop limit"
8911        );
8912        assert_ne!(
8913            first,
8914            SessionResult::Close,
8915            "the first pass must wait for the client, not close"
8916        );
8917        assert_eq!(received, expected, "the client must receive the whole body");
8918    }
8919
8920    /// The loop budget yields only on real progress: at least one stream byte
8921    /// per budgeted iteration. A spin that trickled a few bytes, which an
8922    /// unconditional "any byte moved" rule would let yield forever, is left
8923    /// to the spin path (sozu-proxy/sozu#1906 hardening).
8924    #[test]
8925    fn only_a_byte_per_budgeted_iteration_counts_as_progress_at_the_loop_budget() {
8926        let budget = MAX_LOOP_ITERATIONS as usize;
8927        assert!(!progressed_through_loop_budget(100, 100), "nothing moved");
8928        assert!(
8929            !progressed_through_loop_budget(100, 101),
8930            "one byte is a trickle"
8931        );
8932        assert!(!progressed_through_loop_budget(0, budget - 1));
8933        assert!(progressed_through_loop_budget(0, budget));
8934        assert!(progressed_through_loop_budget(0, 512 * budget));
8935        // `Context::stream_bytes_out` wraps.
8936        assert!(progressed_through_loop_budget(usize::MAX - 10, budget));
8937        assert!(!progressed_through_loop_budget(usize::MAX, 3));
8938    }
8939
8940    /// A transfer that forwards bytes on every iteration of the inner loop of
8941    /// `Mux::ready_inner` is not closed at `MAX_LOOP_ITERATIONS`: the pass
8942    /// yields to the event loop with its readiness intact, and the next pass,
8943    /// run with no new event, carries on (sozu-proxy/sozu#1906).
8944    ///
8945    /// Both peers keep up: the kernel already holds the backend's whole
8946    /// response, and the client reads everything it is sent. Each iteration
8947    /// reads one 512-byte buffer from the backend and writes it to the
8948    /// client, so the response needs more iterations than the budget before
8949    /// any socket answers `WouldBlock`. In production the same pass needs
8950    /// both peers to keep up for about 160 MiB with 16 KiB buffers: a fast
8951    /// backend and a fast client were cut at a random offset, with a clean
8952    /// close and, on 2.2.1, no log at the default level.
8953    ///
8954    /// The passes after the first are driven exactly as
8955    /// `Server::resume_yielded_sessions` drives them: `ready` again, with no
8956    /// `update_readiness` call. Re-arming the readiness by hand would hide a
8957    /// pass that yields with an event consumed, which edge-triggered epoll
8958    /// would never deliver again.
8959    ///
8960    /// TO SEE THIS RED: make the yield arm of the loop budget in
8961    /// `Mux::ready_inner` `return SessionResult::Close` instead. The first
8962    /// pass then closes the session with the body unsent.
8963    #[test]
8964    fn a_transfer_progressing_at_the_loop_limit_yields_and_resumes_without_a_new_event() {
8965        use std::io::{Read, Write};
8966
8967        const BUFFER: usize = 512;
8968        const BODY: usize = (MAX_LOOP_ITERATIONS as usize + 1000) * BUFFER;
8969        let head = format!("HTTP/1.1 200 OK\r\nContent-Length: {BODY}\r\n\r\n");
8970        let length = head.len() + BODY;
8971
8972        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, BUFFER)));
8973        let (mut mux, mut frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
8974        // Room for the whole response between sozu and the client.
8975        socket2::SockRef::from(&frontend_peer)
8976            .set_recv_buffer_size(4 << 20)
8977            .expect("the client receive buffer must be settable");
8978        socket2::SockRef::from(mux.frontend.socket())
8979            .set_send_buffer_size(4 << 20)
8980            .expect("the frontend send buffer must be settable");
8981
8982        let backend_token = Token(1);
8983        let (mut connection, mut backend_peer) =
8984            test_backend_connection(&mut mux, Duration::from_secs(30));
8985        socket2::SockRef::from(connection.socket())
8986            .set_recv_buffer_size(4 << 20)
8987            .expect("the backend receive buffer must be settable");
8988        socket2::SockRef::from(&backend_peer)
8989            .set_send_buffer_size(4 << 20)
8990            .expect("the backend peer send buffer must be settable");
8991        let Connection::H1(h1) = &mut connection else {
8992            unreachable!("new_h1_client builds an H1 connection")
8993        };
8994        h1.stream = Some(0);
8995        if let Position::Client(_, _, status) = &mut h1.position {
8996            *status = BackendStatus::Connected;
8997        }
8998        h1.readiness.interest = Ready::READABLE | Ready::HUP | Ready::ERROR;
8999        h1.readiness.event = Ready::READABLE;
9000        mux.router.backends.insert(backend_token, connection);
9001        mux.context.link_stream(0, backend_token);
9002        mux.context.streams[0].front.parsing_phase = kawa::ParsingPhase::Terminated;
9003
9004        // The backend writes its whole response and keeps its connection
9005        // open, as a keep-alive backend does.
9006        let mut response = head.into_bytes();
9007        response.resize(length, b'b');
9008        let mut written = 0;
9009        let deadline = Instant::now() + Duration::from_secs(5);
9010        while written < response.len() {
9011            match backend_peer.write(&response[written..]) {
9012                Ok(size) => written += size,
9013                Err(e) if e.kind() == ErrorKind::WouldBlock => {
9014                    assert!(
9015                        Instant::now() < deadline,
9016                        "premise: the kernel must hold the whole response, wrote {written}"
9017                    );
9018                    std::thread::sleep(Duration::from_millis(1));
9019                }
9020                Err(e) => panic!("the backend peer failed to write: {e}"),
9021            }
9022        }
9023        std::thread::sleep(Duration::from_millis(20));
9024
9025        let session: Rc<RefCell<dyn ProxySession>> = Rc::new(RefCell::new(NoDialSession));
9026        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
9027        let mut metrics = SessionMetrics::new(None);
9028
9029        LOOP_LIMIT_HITS.with(|hits| hits.set(0));
9030        let first = mux.ready(session.clone(), proxy.clone(), &mut metrics);
9031        assert_ne!(
9032            first,
9033            SessionResult::Close,
9034            "the transfer must not be closed while it progresses"
9035        );
9036        assert!(
9037            mux.take_loop_budget_yield(),
9038            "the first pass must yield at the budget: it forwarded {} of {length} bytes",
9039            mux.context.stream_bytes_out
9040        );
9041        assert!(
9042            mux.context.stream_bytes_out < length,
9043            "premise: the budget must stop the first pass before the end of the response"
9044        );
9045
9046        let mut result = first;
9047        let mut passes = 1;
9048        let mut received = Vec::new();
9049        let mut chunk = vec![0u8; 1 << 16];
9050        let body_received = |received: &[u8]| {
9051            received
9052                .windows(4)
9053                .position(|window| window == b"\r\n\r\n")
9054                .map_or(0, |end| received.len() - end - 4)
9055        };
9056        let mut yielded = true;
9057        loop {
9058            loop {
9059                match frontend_peer.read(&mut chunk) {
9060                    Ok(0) => break,
9061                    Ok(size) => received.extend_from_slice(&chunk[..size]),
9062                    Err(e) if e.kind() == ErrorKind::WouldBlock => break,
9063                    Err(e) => panic!("the client failed to read: {e}"),
9064                }
9065            }
9066            if body_received(&received) >= BODY || result == SessionResult::Close || !yielded {
9067                break;
9068            }
9069            assert!(passes < 10, "the transfer must end in a few passes");
9070            // What `Server::resume_yielded_sessions` does: no new event.
9071            result = mux.ready(session.clone(), proxy.clone(), &mut metrics);
9072            yielded = mux.take_loop_budget_yield();
9073            passes += 1;
9074        }
9075        assert!(passes >= 2, "the body must have needed a resumed pass");
9076        assert_eq!(
9077            LOOP_LIMIT_HITS.with(|hits| hits.get()),
9078            0,
9079            "a progressing pass must yield, never take the infinite-loop path"
9080        );
9081        assert_ne!(
9082            result,
9083            SessionResult::Close,
9084            "no pass may close the transfer"
9085        );
9086        assert_eq!(
9087            body_received(&received),
9088            BODY,
9089            "the resumed passes must deliver the whole body with no new event"
9090        );
9091        assert!(
9092            mux.context.stream_bytes_out >= received.len(),
9093            "every byte the client received was counted as forwarded: {} < {}",
9094            mux.context.stream_bytes_out,
9095            received.len()
9096        );
9097    }
9098
9099    /// The same exchange from a client that shut its write side after its
9100    /// request.
9101    #[test]
9102    fn a_half_closed_slow_client_receives_the_whole_response_of_a_backend_that_closed() {
9103        let (received, expected, first, hits) = slow_client_after_backend_close(true);
9104        assert_eq!(
9105            hits, 0,
9106            "every pass must stop once nothing can progress, not at the loop limit"
9107        );
9108        assert_ne!(
9109            first,
9110            SessionResult::Close,
9111            "the first pass must wait for the client, not close"
9112        );
9113        assert_eq!(received, expected, "the client must receive the whole body");
9114    }
9115
9116    /// At the loop limit, a response still queued in an H1 frontend's stream
9117    /// buffer is not dropped: the session waits for the client's next
9118    /// writable event instead of closing. A plain-TCP H1 frontend's
9119    /// connection-level `has_pending_write()` is false whatever its stream
9120    /// holds, which is what the limit used to check (sozu-proxy/sozu#1819).
9121    ///
9122    /// TO SEE THIS RED: make `Mux::wait_for_client_at_loop_limit` compute
9123    /// `output_queued` from `self.frontend.has_pending_write()` alone.
9124    #[test]
9125    fn the_loop_limit_waits_for_the_client_while_a_stream_holds_its_response() {
9126        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9127        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9128        mux.context.streams[0].state = StreamState::Linked(Token(1));
9129        mux.context.streams[0]
9130            .back
9131            .push_out(kawa::Store::Static(b"the end of the body"));
9132        mux.frontend.readiness_mut().event = Ready::WRITABLE;
9133        assert!(
9134            !mux.frontend.has_pending_write(),
9135            "premise: the connection itself has nothing pending"
9136        );
9137
9138        assert!(
9139            mux.wait_for_client_at_loop_limit(),
9140            "a queued response must make the session wait, not close"
9141        );
9142        let readiness = mux.frontend.readiness();
9143        assert!(
9144            readiness.interest.is_writable() && !readiness.event.is_writable(),
9145            "the session waits for the next writable event, got {readiness:?}"
9146        );
9147    }
9148
9149    /// Waiting at the loop limit is bounded: reaching it again with no byte
9150    /// written to the client since closes the session, while a client that
9151    /// drained some of the response in between is waited for again.
9152    ///
9153    /// TO SEE THIS RED: drop `progressed &&` from `output_queued` in
9154    /// `Mux::wait_for_client_at_loop_limit`. The second hit then waits again.
9155    #[test]
9156    fn a_second_loop_limit_without_progress_closes_the_session() {
9157        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9158        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9159        mux.context.streams[0].state = StreamState::Linked(Token(1));
9160        mux.context.streams[0]
9161            .back
9162            .push_out(kawa::Store::Static(b"the end of the body"));
9163
9164        assert!(
9165            mux.wait_for_client_at_loop_limit(),
9166            "the first hit with output queued waits for the client"
9167        );
9168        mux.context.client_bytes_out += 100;
9169        assert!(
9170            mux.wait_for_client_at_loop_limit(),
9171            "the client drained bytes since: the session waits again"
9172        );
9173        assert!(
9174            !mux.wait_for_client_at_loop_limit(),
9175            "no byte reached the client since the last hit: the session must close"
9176        );
9177    }
9178
9179    /// Progress at the loop limit is read from the session's own count of
9180    /// bytes written to the client, not from the per-stream metrics, which
9181    /// `SessionMetrics::reset` zeroes when a stream ends and its slot is
9182    /// reused.
9183    ///
9184    /// TO SEE THIS RED: compute `bytes_out` in
9185    /// `Mux::wait_for_client_at_loop_limit` as the sum of every stream's
9186    /// `metrics.bout` again. A slot reused between two hits then reads as
9187    /// progress when nothing was written, and as no progress when the new
9188    /// stream wrote exactly what the old one had.
9189    #[test]
9190    fn a_stream_slot_reused_between_two_loop_limits_does_not_fake_progress() {
9191        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9192        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9193        mux.context.streams[0].state = StreamState::Linked(Token(1));
9194        mux.context.streams[0]
9195            .back
9196            .push_out(kawa::Store::Static(b"the end of the body"));
9197        mux.context.streams[0].metrics.bout = 100;
9198        mux.context.client_bytes_out = 100;
9199        assert!(
9200            mux.wait_for_client_at_loop_limit(),
9201            "the first hit with output queued waits for the client"
9202        );
9203
9204        // The stream ends and its slot is reused before anything is written.
9205        mux.context.streams[0].metrics.reset();
9206        assert!(
9207            !mux.wait_for_client_at_loop_limit(),
9208            "a reset metric is not progress: the session must close"
9209        );
9210
9211        // A fresh session state: the new stream writes as much as the old one
9212        // had, which is progress.
9213        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9214        mux.context.streams[0].state = StreamState::Linked(Token(1));
9215        mux.context.streams[0]
9216            .back
9217            .push_out(kawa::Store::Static(b"the end of the body"));
9218        mux.context.streams[0].metrics.bout = 100;
9219        mux.context.client_bytes_out = 100;
9220        assert!(mux.wait_for_client_at_loop_limit(), "the first hit waits");
9221        mux.context.streams[0].metrics.reset();
9222        mux.context.streams[0].metrics.bout = 100;
9223        mux.context.client_bytes_out += 100;
9224        assert!(
9225            mux.wait_for_client_at_loop_limit(),
9226            "the reused slot wrote 100 bytes since: the session waits again"
9227        );
9228    }
9229
9230    /// At the loop limit, a session with nothing queued for the client is
9231    /// closed: the limit stays a safety bound. Bytes left in a stream slot
9232    /// that is no longer open are not owed to the client.
9233    #[test]
9234    fn the_loop_limit_closes_a_session_with_nothing_queued() {
9235        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9236        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9237        mux.context.streams[0].state = StreamState::Recycle;
9238        mux.context.streams[0]
9239            .back
9240            .push_out(kawa::Store::Static(b"a recycled slot's leftover"));
9241        assert!(
9242            !mux.wait_for_client_at_loop_limit(),
9243            "nothing queued: the session must close"
9244        );
9245    }
9246
9247    // ── no write-side shutdown once the peer has closed ────────────────
9248
9249    /// Close a mux holding one backend whose readiness is `event` and whose
9250    /// status is `status`, and return the `shutdown(SHUT_WR)` calls the close
9251    /// issued together with what the backend peer then observed.
9252    fn backend_shutdowns_of_a_close(
9253        event: Ready,
9254        status: BackendStatus,
9255        peer_sends_fin: bool,
9256    ) -> (usize, bool) {
9257        let pool = Rc::new(RefCell::new(Pool::with_capacity(4, 8, 16384)));
9258        let (mut mux, _frontend_peer) = h1_mux_with_idle_stream(&pool, Duration::from_secs(60));
9259        mux.context.streams[0].metrics.reset();
9260        let (mut connection, mut backend_peer) =
9261            test_backend_connection(&mut mux, Duration::from_secs(30));
9262        if peer_sends_fin {
9263            backend_peer
9264                .shutdown(std::net::Shutdown::Write)
9265                .expect("the backend peer must send its FIN");
9266        }
9267        *connection.position_mut() = match connection.position() {
9268            Position::Client(cluster_id, backend, _) => {
9269                Position::Client(cluster_id.clone(), backend.clone(), status)
9270            }
9271            Position::Server => unreachable!("new_h1_client builds a client"),
9272        };
9273        connection.readiness_mut().event = event;
9274        mux.router.backends.insert(Token(1), connection);
9275
9276        let proxy: Rc<RefCell<dyn L7Proxy>> = Rc::new(RefCell::new(RemoveOnlyProxy));
9277        let mut metrics = SessionMetrics::new(None);
9278        let before = shutdown_writes();
9279        mux.close(proxy, &mut metrics);
9280        let shutdowns = shutdown_writes() - before;
9281        // The descriptor closes when the mux drops, as it does with its
9282        // session in production.
9283        drop(mux);
9284        let (_, saw_fin) = drain_peer(&mut backend_peer, &[]);
9285        (shutdowns, saw_fin)
9286    }
9287
9288    /// Issue #1603: `Mux::close` skips `shutdown(SHUT_WR)` on a backend that
9289    /// has already closed, and the backend still gets its FIN from the close.
9290    ///
9291    /// The peer in the first case half-closed and is still reading: that is
9292    /// the peer the skipped shutdown could have mattered to, and the FIN it
9293    /// observes comes from `close(2)` alone.
9294    ///
9295    /// TO SEE THIS RED: make `shutdown_write` ignore `peer_closed`. The first
9296    /// assertion then fails with `left: 1`, `right: 0`.
9297    #[test]
9298    fn mux_close_skips_the_backend_shutdown_once_the_backend_closed() {
9299        let (shutdowns, saw_fin) =
9300            backend_shutdowns_of_a_close(Ready::HUP, BackendStatus::Connected, true);
9301        assert_eq!(
9302            shutdowns, 0,
9303            "a backend whose FIN was seen must not be shut down before its close"
9304        );
9305        assert!(
9306            saw_fin,
9307            "the close alone must still deliver the FIN to a half-closed peer"
9308        );
9309    }
9310
9311    /// The other half of #1603: a backend that has NOT closed keeps its
9312    /// `shutdown(SHUT_WR)` and observes the FIN — including one sozu itself
9313    /// is dropping, whose readiness `force_disconnect` overwrote with HUP.
9314    #[test]
9315    fn mux_close_keeps_the_backend_shutdown_while_the_backend_is_there() {
9316        for (event, status, why) in [
9317            (
9318                Ready::READABLE | Ready::WRITABLE,
9319                BackendStatus::Connected,
9320                "a live backend",
9321            ),
9322            (
9323                Ready::HUP,
9324                BackendStatus::Disconnecting,
9325                "a backend force_disconnect marked with HUP",
9326            ),
9327        ] {
9328            let (shutdowns, saw_fin) = backend_shutdowns_of_a_close(event, status, false);
9329            assert_eq!(shutdowns, 1, "{why} must keep its shutdown(SHUT_WR)");
9330            assert!(saw_fin, "{why} must observe the FIN");
9331        }
9332    }
9333}