Skip to main content

actix_http/h1/
dispatcher.rs

1use std::{
2    cmp,
3    collections::VecDeque,
4    fmt,
5    future::Future,
6    io, mem, net,
7    pin::Pin,
8    rc::Rc,
9    task::{Context, Poll},
10};
11
12use actix_codec::{Framed, FramedParts};
13use actix_service::Service;
14use bitflags::bitflags;
15use bytes::{Buf, Bytes, BytesMut};
16use futures_core::ready;
17use pin_project_lite::pin_project;
18use tokio::{
19    io::{AsyncRead, AsyncWrite},
20    time::sleep_until,
21};
22use tokio_util::codec::{Decoder as _, Encoder as _};
23use tracing::{error, trace};
24
25use super::{
26    codec::Codec,
27    decoder::MAX_BUFFER_SIZE,
28    payload::{Payload, PayloadSender, PayloadStatus},
29    timer::TimerState,
30    Message, MessageType,
31};
32use crate::{
33    body::{BodySize, BoxBody, MessageBody},
34    config::ServiceConfig,
35    error::{DispatchError, ParseError, PayloadError},
36    service::HttpFlow,
37    ConnectionType, Error, Extensions, HttpMessage, OnConnectData, Request, Response, StatusCode,
38};
39
40const LW_BUFFER_SIZE: usize = 1024;
41const HW_BUFFER_SIZE: usize = 1024 * 8;
42const MAX_PIPELINED_MESSAGES: usize = 16;
43
44/// Write buffer capacity at which we reallocate it after flushing.
45const WRITE_BUFFER_RETENTION_LIMIT: usize = 1024 * 128;
46
47bitflags! {
48    #[derive(Debug, Clone, Copy)]
49    pub struct Flags: u8 {
50        /// Set when stream is read for first time.
51        const STARTED          = 0b0000_0001;
52
53        /// Set when full request-response cycle has occurred.
54        const FINISHED         = 0b0000_0010;
55
56        /// Set if connection is in keep-alive (inactive) state.
57        const KEEP_ALIVE       = 0b0000_0100;
58
59        /// Set if in shutdown procedure.
60        const SHUTDOWN         = 0b0000_1000;
61
62        /// Set if read-half is disconnected.
63        const READ_DISCONNECT  = 0b0001_0000;
64
65        /// Set if write-half is disconnected.
66        const WRITE_DISCONNECT = 0b0010_0000;
67
68        /// Set while gracefully closing a connection after an early response.
69        const LINGER           = 0b0100_0000;
70
71        /// Set when the server is draining this connection during graceful shutdown.
72        ///
73        /// Unlike [`SHUTDOWN`](Self::SHUTDOWN), this state continues polling the current request
74        /// and response. It prevents queued requests from starting and transitions to `SHUTDOWN`
75        /// after the current response finishes. [`LINGER`](Self::LINGER) remains separate and is
76        /// used only to close cleanly after a response to an unread request payload.
77        const DRAINING         = 0b1000_0000;
78    }
79}
80
81// there's 2 versions of Dispatcher state because of:
82// https://github.com/taiki-e/pin-project-lite/issues/3
83//
84// tl;dr: pin-project-lite doesn't play well with other attribute macros
85
86#[cfg(not(test))]
87pin_project! {
88    /// Dispatcher for HTTP/1.1 protocol
89    pub struct Dispatcher<T, S, B, X, U>
90    where
91        S: Service<Request>,
92        S::Error: Into<Response<BoxBody>>,
93
94        B: MessageBody,
95
96        X: Service<Request, Response = Request>,
97        X::Error: Into<Response<BoxBody>>,
98
99        U: Service<(Request, Framed<T, Codec>), Response = ()>,
100        U::Error: fmt::Display,
101    {
102        #[pin]
103        inner: DispatcherState<T, S, B, X, U>,
104    }
105}
106
107#[cfg(test)]
108pin_project! {
109    /// Dispatcher for HTTP/1.1 protocol
110    pub struct Dispatcher<T, S, B, X, U>
111    where
112        S: Service<Request>,
113        S::Error: Into<Response<BoxBody>>,
114
115        B: MessageBody,
116
117        X: Service<Request, Response = Request>,
118        X::Error: Into<Response<BoxBody>>,
119
120        U: Service<(Request, Framed<T, Codec>), Response = ()>,
121        U::Error: fmt::Display,
122    {
123        #[pin]
124        pub(super) inner: DispatcherState<T, S, B, X, U>,
125
126        // used in tests
127        pub(super) poll_count: u64,
128    }
129}
130
131pin_project! {
132    #[project = DispatcherStateProj]
133    pub(super) enum DispatcherState<T, S, B, X, U>
134    where
135        S: Service<Request>,
136        S::Error: Into<Response<BoxBody>>,
137
138        B: MessageBody,
139
140        X: Service<Request, Response = Request>,
141        X::Error: Into<Response<BoxBody>>,
142
143        U: Service<(Request, Framed<T, Codec>), Response = ()>,
144        U::Error: fmt::Display,
145    {
146        Normal { #[pin] inner: InnerDispatcher<T, S, B, X, U> },
147        Upgrade { #[pin] fut: U::Future },
148    }
149}
150
151pin_project! {
152    #[project = InnerDispatcherProj]
153    pub(super) struct InnerDispatcher<T, S, B, X, U>
154    where
155        S: Service<Request>,
156        S::Error: Into<Response<BoxBody>>,
157
158        B: MessageBody,
159
160        X: Service<Request, Response = Request>,
161        X::Error: Into<Response<BoxBody>>,
162
163        U: Service<(Request, Framed<T, Codec>), Response = ()>,
164        U::Error: fmt::Display,
165    {
166        flow: Rc<HttpFlow<S, X, U>>,
167        pub(super) flags: Flags,
168        peer_addr: Option<net::SocketAddr>,
169        conn_data: Option<Rc<Extensions>>,
170        config: ServiceConfig,
171        error: Option<DispatchError>,
172
173        #[pin]
174        pub(super) state: State<S, B, X>,
175        // when Some(_) dispatcher is in state of receiving request payload
176        payload: Option<PayloadSender>,
177        // true when current request uses chunked transfer encoding (drainable when payload is dropped)
178        payload_drainable: bool,
179        messages: VecDeque<DispatcherMessage>,
180
181        head_timer: TimerState,
182        ka_timer: TimerState,
183        shutdown_timer: TimerState,
184        graceful_shutdown: Option<crate::config::GracefulShutdownFuture>,
185
186        pub(super) io: Option<T>,
187        read_buf: BytesMut,
188
189        // Encoded response heads and buffered body chunks, in wire order.
190        //
191        // Includes small fixed-length chunks and chunks from chunked or close-delimited bodies.
192        // Encoding a chunk can grow this buffer beyond `h1_write_buffer_size`.
193        // These bytes are written before any pending bytes in `write_body`.
194        write_buf: BytesMut,
195
196        // Unwritten bytes of one large fixed-length body chunk, sent after `write_buf`.
197        //
198        // Chunks at least `h1_write_buffer_size` bytes long can be stored here without copying them
199        // into `write_buf` if the connection supports vectored writes. Uniquely owned chunks with
200        // spare capacity are copied into `write_buf` instead (see `prepare_chunk`).
201        //
202        // This can be the entire body or one chunk from a stream with a known total length. The
203        // slice is limited to the remaining declared body length.
204        //
205        // Partial writes advance the slice. The next body chunk is not polled while this field is
206        // `Some`. Continue responses and upgrades copy any remaining bytes into `write_buf` to
207        // preserve write order.
208        write_body: Option<Bytes>,
209
210        // Number of bytes to busy-buffer before flushing.
211        h1_write_buffer_size: usize,
212
213        // True if backing allocation of `write_buf` needs to be released.
214        write_buf_needs_reset: bool,
215
216        codec: Codec,
217    }
218}
219
220impl<T, S, B, X, U> InnerDispatcherProj<'_, T, S, B, X, U>
221where
222    S: Service<Request>,
223    S::Error: Into<Response<BoxBody>>,
224
225    B: MessageBody,
226
227    X: Service<Request, Response = Request>,
228    X::Error: Into<Response<BoxBody>>,
229
230    U: Service<(Request, Framed<T, Codec>), Response = ()>,
231    U::Error: fmt::Display,
232{
233    /// Copies any pending body bytes into `write_buf` and clears `write_body`.
234    fn ensure_body_buffered(&mut self) {
235        if let Some(body) = self.write_body.take() {
236            self.write_buf.extend_from_slice(&body);
237        }
238    }
239}
240
241enum DispatcherMessage {
242    Item(Request),
243    Upgrade(Request),
244    Error(Response<()>),
245}
246
247pin_project! {
248    #[project = StateProj]
249    pub(super) enum State<S, B, X>
250    where
251        S: Service<Request>,
252        X: Service<Request, Response = Request>,
253        B: MessageBody,
254    {
255        None,
256        ExpectCall { #[pin] fut: X::Future },
257        ServiceCall { #[pin] fut: S::Future },
258        SendPayload { #[pin] body: B },
259        SendErrorPayload { #[pin] body: BoxBody },
260    }
261}
262
263impl<S, B, X> State<S, B, X>
264where
265    S: Service<Request>,
266    X: Service<Request, Response = Request>,
267    B: MessageBody,
268{
269    pub(super) fn is_none(&self) -> bool {
270        matches!(self, State::None)
271    }
272}
273
274impl<S, B, X> fmt::Debug for State<S, B, X>
275where
276    S: Service<Request>,
277    X: Service<Request, Response = Request>,
278    B: MessageBody,
279{
280    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
281        match self {
282            Self::None => write!(f, "State::None"),
283            Self::ExpectCall { .. } => f.debug_struct("State::ExpectCall").finish_non_exhaustive(),
284            Self::ServiceCall { .. } => {
285                f.debug_struct("State::ServiceCall").finish_non_exhaustive()
286            }
287            Self::SendPayload { .. } => {
288                f.debug_struct("State::SendPayload").finish_non_exhaustive()
289            }
290            Self::SendErrorPayload { .. } => f
291                .debug_struct("State::SendErrorPayload")
292                .finish_non_exhaustive(),
293        }
294    }
295}
296
297#[derive(Debug)]
298enum PollResponse {
299    Upgrade(Request),
300    DoNothing,
301    DrainWriteBuf,
302}
303
304impl<T, S, B, X, U> Dispatcher<T, S, B, X, U>
305where
306    T: AsyncRead + AsyncWrite + Unpin,
307
308    S: Service<Request>,
309    S::Error: Into<Response<BoxBody>>,
310    S::Response: Into<Response<B>>,
311
312    B: MessageBody,
313
314    X: Service<Request, Response = Request>,
315    X::Error: Into<Response<BoxBody>>,
316
317    U: Service<(Request, Framed<T, Codec>), Response = ()>,
318    U::Error: fmt::Display,
319{
320    /// Create HTTP/1 dispatcher.
321    pub(crate) fn new(
322        io: T,
323        flow: Rc<HttpFlow<S, X, U>>,
324        config: ServiceConfig,
325        peer_addr: Option<net::SocketAddr>,
326        conn_data: OnConnectData,
327    ) -> Self {
328        Dispatcher {
329            inner: DispatcherState::Normal {
330                inner: InnerDispatcher {
331                    flow,
332                    flags: Flags::empty(),
333                    peer_addr,
334                    conn_data: conn_data.0.map(Rc::new),
335                    config: config.clone(),
336                    error: None,
337
338                    state: State::None,
339                    payload: None,
340                    payload_drainable: false,
341                    messages: VecDeque::new(),
342
343                    head_timer: TimerState::new(config.client_request_deadline().is_some()),
344                    ka_timer: TimerState::new(config.keep_alive().enabled()),
345                    shutdown_timer: TimerState::new(config.client_disconnect_deadline().is_some()),
346                    graceful_shutdown: config.graceful_shutdown(),
347
348                    io: Some(io),
349                    read_buf: BytesMut::with_capacity(HW_BUFFER_SIZE),
350                    write_buf: BytesMut::with_capacity(HW_BUFFER_SIZE),
351                    write_body: None,
352                    h1_write_buffer_size: config.h1_write_buffer_size(),
353                    write_buf_needs_reset: false,
354                    codec: Codec::new(config),
355                },
356            },
357
358            #[cfg(test)]
359            poll_count: 0,
360        }
361    }
362}
363
364impl<T, S, B, X, U> InnerDispatcher<T, S, B, X, U>
365where
366    T: AsyncRead + AsyncWrite + Unpin,
367
368    S: Service<Request>,
369    S::Error: Into<Response<BoxBody>>,
370    S::Response: Into<Response<B>>,
371
372    B: MessageBody,
373
374    X: Service<Request, Response = Request>,
375    X::Error: Into<Response<BoxBody>>,
376
377    U: Service<(Request, Framed<T, Codec>), Response = ()>,
378    U::Error: fmt::Display,
379{
380    fn can_read(&self, cx: &mut Context<'_>) -> bool {
381        if self.flags.contains(Flags::READ_DISCONNECT) {
382            false
383        } else if let Some(ref info) = self.payload {
384            matches!(
385                info.need_read(cx),
386                PayloadStatus::Read | PayloadStatus::Dropped
387            )
388        } else {
389            true
390        }
391    }
392
393    fn client_disconnected(self: Pin<&mut Self>) {
394        let this = self.project();
395
396        this.flags
397            .insert(Flags::READ_DISCONNECT | Flags::WRITE_DISCONNECT);
398
399        if let Some(mut payload) = this.payload.take() {
400            payload.set_error(PayloadError::Incomplete(None));
401        }
402    }
403
404    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
405        let InnerDispatcherProj {
406            io,
407            write_buf,
408            write_body,
409            h1_write_buffer_size,
410            write_buf_needs_reset,
411            ..
412        } = self.project();
413
414        let mut io = Pin::new(io.as_mut().unwrap());
415
416        // Allow normal buffer growth, but do not retain allocations caused by large response
417        // chunks on idle connections.
418        let retention_limit = (*h1_write_buffer_size).max(WRITE_BUFFER_RETENTION_LIMIT);
419
420        if write_buf.capacity() > retention_limit {
421            // Needs stored flag because `write_buf.advance(n)` reduces reported capacity without
422            // releasing the backing allocation.
423            *write_buf_needs_reset = true;
424        }
425
426        let write_buf_len = write_buf.len();
427        let mut write_buf_written = 0;
428
429        loop {
430            let head = &write_buf[write_buf_written..];
431            let head_len = head.len();
432
433            let polled = match (head, write_body.as_deref()) {
434                // Both segments are drained. Avoid an empty write that could return zero.
435                ([], None | Some([])) => break,
436
437                // The buffered prefix is drained, so write the remaining body bytes.
438                ([], Some(body)) => io.as_mut().poll_write(cx, body)?,
439
440                // Write the buffered prefix and body in order with one vectored write.
441                (head, Some(body @ [_, ..])) if io.is_write_vectored() => io
442                    .as_mut()
443                    .poll_write_vectored(cx, &[io::IoSlice::new(head), io::IoSlice::new(body)])?,
444
445                // Keep buffered bytes ahead of the body when writing one slice at a time.
446                (head, _) => io.as_mut().poll_write(cx, head)?,
447            };
448
449            match polled {
450                Poll::Ready(0) => {
451                    error!("write zero; closing");
452                    return Poll::Ready(Err(io::Error::new(io::ErrorKind::WriteZero, "")));
453                }
454
455                Poll::Ready(n) => {
456                    // Advance cursors of write_buf and write_body buffers according to how much was
457                    // written to the socket for each ready for next loop iteration.
458
459                    let from_head = cmp::min(n, head_len);
460                    write_buf_written += from_head;
461
462                    if let Some(body) = write_body.as_mut() {
463                        let from_body = n - from_head;
464                        body.advance(from_body);
465                    }
466                }
467
468                Poll::Pending => {
469                    // `write_buf_written` counts only buffered bytes and cannot exceed
470                    // `write_buf.len()`
471                    write_buf.advance(write_buf_written);
472
473                    return Poll::Pending;
474                }
475            }
476        }
477
478        debug_assert_eq!(
479            write_buf_written, write_buf_len,
480            "All buffered bytes should be written before leaving the write loop"
481        );
482
483        // All bytes have been accepted by the I/O stream, so the buffer can be reused or released.
484        if *write_buf_needs_reset {
485            *write_buf = BytesMut::with_capacity(HW_BUFFER_SIZE);
486            *write_buf_needs_reset = false;
487        } else {
488            write_buf.clear();
489        }
490
491        // `write_body` is either absent or fully consumed, so clear it.
492        *write_body = None;
493
494        // flush the I/O and check if get blocked
495        io.poll_flush(cx)
496    }
497
498    fn enter_linger(flags: &mut Flags) {
499        flags.remove(Flags::KEEP_ALIVE);
500        flags.insert(Flags::LINGER | Flags::FINISHED);
501    }
502
503    fn ensure_linger_timer(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> bool {
504        let this = self.as_mut().project();
505
506        if matches!(this.shutdown_timer, TimerState::Active { .. }) {
507            return true;
508        }
509
510        if let Some(deadline) = this.config.client_disconnect_deadline() {
511            this.shutdown_timer
512                .set_and_init(cx, sleep_until(deadline.into()), line!());
513            true
514        } else {
515            false
516        }
517    }
518
519    fn poll_linger(
520        mut self: Pin<&mut Self>,
521        cx: &mut Context<'_>,
522    ) -> Result<Poll<()>, DispatchError> {
523        if self.as_mut().poll_flush(cx)?.is_pending() {
524            return Ok(Poll::Pending);
525        }
526
527        if !self.as_mut().ensure_linger_timer(cx) {
528            let this = self.as_mut().project();
529            this.flags.remove(Flags::LINGER);
530            this.flags.insert(Flags::SHUTDOWN);
531            return Ok(Poll::Ready(()));
532        }
533
534        loop {
535            let should_disconnect = self.as_mut().read_available(cx)?;
536            let this = self.as_mut().project();
537            let mut progressed = false;
538
539            if !this.read_buf.is_empty() {
540                this.read_buf.clear();
541                progressed = true;
542            }
543
544            if should_disconnect {
545                this.flags.remove(Flags::LINGER);
546                this.flags.insert(Flags::READ_DISCONNECT | Flags::SHUTDOWN);
547                return Ok(Poll::Ready(()));
548            }
549
550            if !progressed {
551                return Ok(Poll::Pending);
552            }
553        }
554    }
555
556    fn send_response_inner(
557        self: Pin<&mut Self>,
558        res: Response<()>,
559        body: &impl MessageBody,
560    ) -> Result<BodySize, DispatchError> {
561        let this = self.project();
562
563        let size = body.size();
564
565        this.codec
566            .encode(Message::Item((res, size)), this.write_buf)
567            .map_err(|err| {
568                if let Some(mut payload) = this.payload.take() {
569                    payload.set_error(PayloadError::Incomplete(None));
570                }
571
572                DispatchError::Io(err)
573            })?;
574
575        Ok(size)
576    }
577
578    fn send_response(
579        mut self: Pin<&mut Self>,
580        mut res: Response<()>,
581        body: B,
582    ) -> Result<(), DispatchError> {
583        let is_upgrade = res.upgrade();
584        let (draining, close_for_unread_payload) = {
585            let this = self.as_mut().project();
586            (
587                this.flags.contains(Flags::DRAINING),
588                !is_upgrade
589                    && should_close_for_unread_payload(
590                        this.payload.as_ref(),
591                        *this.payload_drainable,
592                    ),
593            )
594        };
595        let close_after_response = (!is_upgrade && draining) || close_for_unread_payload;
596
597        if close_after_response {
598            res.head_mut().set_connection_type(ConnectionType::Close);
599        }
600
601        let size = self.as_mut().send_response_inner(res, &body)?;
602        match size {
603            BodySize::None | BodySize::Sized(0) => {
604                let mut this = self.as_mut().project();
605
606                if close_for_unread_payload {
607                    if this.config.client_disconnect_deadline().is_some() {
608                        Self::enter_linger(this.flags);
609                    } else {
610                        this.flags.insert(Flags::SHUTDOWN | Flags::FINISHED);
611                    }
612                } else {
613                    this.flags.insert(Flags::FINISHED);
614                }
615
616                this.state.set(State::None);
617            }
618            _ => self
619                .as_mut()
620                .project()
621                .state
622                .set(State::SendPayload { body }),
623        }
624
625        Ok(())
626    }
627
628    fn send_error_response(
629        mut self: Pin<&mut Self>,
630        mut res: Response<()>,
631        body: BoxBody,
632    ) -> Result<(), DispatchError> {
633        let is_upgrade = res.upgrade();
634        let (draining, close_for_unread_payload) = {
635            let this = self.as_mut().project();
636            (
637                this.flags.contains(Flags::DRAINING),
638                !is_upgrade
639                    && should_close_for_unread_payload(
640                        this.payload.as_ref(),
641                        *this.payload_drainable,
642                    ),
643            )
644        };
645        let close_after_response = (!is_upgrade && draining) || close_for_unread_payload;
646
647        if close_after_response {
648            res.head_mut().set_connection_type(ConnectionType::Close);
649        }
650
651        let size = self.as_mut().send_response_inner(res, &body)?;
652        match size {
653            BodySize::None | BodySize::Sized(0) => {
654                let mut this = self.as_mut().project();
655
656                if close_for_unread_payload {
657                    if this.config.client_disconnect_deadline().is_some() {
658                        Self::enter_linger(this.flags);
659                    } else {
660                        this.flags.insert(Flags::SHUTDOWN | Flags::FINISHED);
661                    }
662                } else {
663                    this.flags.insert(Flags::FINISHED);
664                }
665
666                this.state.set(State::None);
667            }
668            _ => self
669                .as_mut()
670                .project()
671                .state
672                .set(State::SendErrorPayload { body }),
673        }
674
675        Ok(())
676    }
677
678    fn send_continue(self: Pin<&mut Self>) {
679        let mut this = self.project();
680
681        this.ensure_body_buffered();
682
683        this.write_buf
684            .extend_from_slice(b"HTTP/1.1 100 Continue\r\n\r\n");
685    }
686
687    fn poll_response(
688        mut self: Pin<&mut Self>,
689        cx: &mut Context<'_>,
690    ) -> Result<PollResponse, DispatchError> {
691        'res: loop {
692            let mut this = self.as_mut().project();
693            match this.state.as_mut().project() {
694                StateProj::None if this.flags.contains(Flags::DRAINING) => {
695                    this.messages.clear();
696                    this.flags.remove(Flags::KEEP_ALIVE);
697
698                    if !this.flags.contains(Flags::LINGER) {
699                        this.flags.insert(Flags::SHUTDOWN);
700                    }
701
702                    return Ok(PollResponse::DoNothing);
703                }
704
705                // no future is in InnerDispatcher state; pop next message
706                StateProj::None => match this.messages.pop_front() {
707                    // handle request message
708                    Some(DispatcherMessage::Item(req)) => {
709                        // Handle `EXPECT: 100-Continue` header
710                        if req.head().expect() {
711                            // set InnerDispatcher state and continue loop to poll it
712                            let fut = this.flow.expect.call(req);
713                            this.state.set(State::ExpectCall { fut });
714                        } else {
715                            // set InnerDispatcher state and continue loop to poll it
716                            let fut = this.flow.service.call(req);
717                            this.state.set(State::ServiceCall { fut });
718                        };
719                    }
720
721                    // handle error message
722                    Some(DispatcherMessage::Error(res)) => {
723                        // send_response would update InnerDispatcher state to SendPayload or None
724                        // (If response body is empty)
725                        // continue loop to poll it
726                        self.as_mut().send_error_response(res, BoxBody::new(()))?;
727                    }
728
729                    // return with upgrade request and poll it exclusively
730                    Some(DispatcherMessage::Upgrade(req)) => return Ok(PollResponse::Upgrade(req)),
731
732                    // all messages are dealt with
733                    None => {
734                        // start keep-alive only if request payload is fully read/drained
735                        this.flags.set(
736                            Flags::KEEP_ALIVE,
737                            this.payload.is_none() && this.codec.keep_alive(),
738                        );
739
740                        return Ok(PollResponse::DoNothing);
741                    }
742                },
743
744                StateProj::ServiceCall { fut } => {
745                    match fut.poll(cx) {
746                        // service call resolved. send response.
747                        Poll::Ready(Ok(res)) => {
748                            let (res, body) = res.into().replace_body(());
749                            self.as_mut().send_response(res, body)?;
750                        }
751
752                        // send service call error as response
753                        Poll::Ready(Err(err)) => {
754                            let res: Response<BoxBody> = err.into();
755                            let (res, body) = res.replace_body(());
756                            self.as_mut().send_error_response(res, body)?;
757                        }
758
759                        // service call pending and could be waiting for more chunk messages
760                        // (pipeline message limit and/or payload can_read limit)
761                        Poll::Pending => {
762                            // no new message is decoded and no new payload is fed
763                            // nothing to do except waiting for new incoming data from client
764                            if !self.as_mut().poll_request(cx)? {
765                                return Ok(PollResponse::DoNothing);
766                            }
767                            // else loop
768                        }
769                    }
770                }
771
772                StateProj::SendPayload { mut body } => {
773                    // keep populate writer buffer until buffer size limit hit,
774                    // get blocked or finished.
775                    while this.write_buf.len() < *this.h1_write_buffer_size
776                        && this.write_body.is_none()
777                    {
778                        match body.as_mut().poll_next(cx) {
779                            Poll::Ready(Some(Ok(chunk))) => {
780                                let write_vectored =
781                                    this.io.as_ref().is_some_and(|io| io.is_write_vectored());
782
783                                if let ChunkDisposition::NeedsBuffering(item) = prepare_chunk(
784                                    this.codec,
785                                    this.write_body,
786                                    write_vectored,
787                                    *this.h1_write_buffer_size,
788                                    chunk,
789                                ) {
790                                    this.codec
791                                        .encode(Message::Chunk(Some(item)), this.write_buf)?;
792                                }
793                            }
794
795                            Poll::Ready(None) => {
796                                this.codec.encode(Message::Chunk(None), this.write_buf)?;
797
798                                // if we have not yet pipelined to the next request, then
799                                // this.payload was the payload for the request we just finished
800                                // responding to. We can check to see if we finished reading it
801                                // yet, and if not, shutdown the connection.
802                                let close_for_unread_payload = should_close_for_unread_payload(
803                                    this.payload.as_ref(),
804                                    *this.payload_drainable,
805                                );
806                                let not_pipelined = this.messages.is_empty();
807
808                                // payload stream finished.
809                                // set state to None and handle next message
810                                this.state.set(State::None);
811
812                                if not_pipelined && close_for_unread_payload {
813                                    if this.config.client_disconnect_deadline().is_some() {
814                                        Self::enter_linger(this.flags);
815                                    } else {
816                                        this.flags.insert(Flags::SHUTDOWN | Flags::FINISHED);
817                                    }
818                                } else {
819                                    this.flags.insert(Flags::FINISHED);
820                                }
821
822                                continue 'res;
823                            }
824
825                            Poll::Ready(Some(Err(err))) => {
826                                let err = err.into();
827                                tracing::error!("Response payload stream error: {err:?}");
828                                this.flags.insert(Flags::FINISHED);
829                                return Err(DispatchError::Body(err));
830                            }
831
832                            Poll::Pending => return Ok(PollResponse::DoNothing),
833                        }
834                    }
835
836                    // buffer is beyond max size
837                    // return and try to write the whole buffer to I/O stream.
838                    return Ok(PollResponse::DrainWriteBuf);
839                }
840
841                StateProj::SendErrorPayload { mut body } => {
842                    // TODO: de-dupe impl with SendPayload
843
844                    // keep populate writer buffer until buffer size limit hit,
845                    // get blocked or finished.
846                    while this.write_buf.len() < *this.h1_write_buffer_size
847                        && this.write_body.is_none()
848                    {
849                        match body.as_mut().poll_next(cx) {
850                            Poll::Ready(Some(Ok(chunk))) => {
851                                let write_vectored =
852                                    this.io.as_ref().is_some_and(|io| io.is_write_vectored());
853
854                                if let ChunkDisposition::NeedsBuffering(item) = prepare_chunk(
855                                    this.codec,
856                                    this.write_body,
857                                    write_vectored,
858                                    *this.h1_write_buffer_size,
859                                    chunk,
860                                ) {
861                                    this.codec
862                                        .encode(Message::Chunk(Some(item)), this.write_buf)?;
863                                }
864                            }
865
866                            Poll::Ready(None) => {
867                                this.codec.encode(Message::Chunk(None), this.write_buf)?;
868
869                                // if we have not yet pipelined to the next request, then
870                                // this.payload was the payload for the request we just finished
871                                // responding to. We can check to see if we finished reading it
872                                // yet, and if not, shutdown the connection.
873                                let close_for_unread_payload = should_close_for_unread_payload(
874                                    this.payload.as_ref(),
875                                    *this.payload_drainable,
876                                );
877                                let not_pipelined = this.messages.is_empty();
878
879                                // payload stream finished.
880                                // set state to None and handle next message
881                                this.state.set(State::None);
882
883                                if not_pipelined && close_for_unread_payload {
884                                    if this.config.client_disconnect_deadline().is_some() {
885                                        Self::enter_linger(this.flags);
886                                    } else {
887                                        this.flags.insert(Flags::SHUTDOWN | Flags::FINISHED);
888                                    }
889                                } else {
890                                    this.flags.insert(Flags::FINISHED);
891                                }
892
893                                continue 'res;
894                            }
895
896                            Poll::Ready(Some(Err(err))) => {
897                                tracing::error!("Response payload stream error: {err:?}");
898                                this.flags.insert(Flags::FINISHED);
899                                return Err(DispatchError::Body(
900                                    Error::new_body().with_cause(err).into(),
901                                ));
902                            }
903
904                            Poll::Pending => return Ok(PollResponse::DoNothing),
905                        }
906                    }
907
908                    // buffer is beyond max size
909                    // return and try to write the whole buffer to stream
910                    return Ok(PollResponse::DrainWriteBuf);
911                }
912
913                StateProj::ExpectCall { fut } => {
914                    trace!("  calling expect service");
915
916                    match fut.poll(cx) {
917                        // expect resolved. write continue to buffer and set InnerDispatcher state
918                        // to service call.
919                        Poll::Ready(Ok(req)) => {
920                            this.ensure_body_buffered();
921
922                            this.write_buf
923                                .extend_from_slice(b"HTTP/1.1 100 Continue\r\n\r\n");
924
925                            let fut = this.flow.service.call(req);
926                            this.state.set(State::ServiceCall { fut });
927                        }
928
929                        // send expect error as response
930                        Poll::Ready(Err(err)) => {
931                            let res: Response<BoxBody> = err.into();
932                            let (res, body) = res.replace_body(());
933                            self.as_mut().send_error_response(res, body)?;
934                        }
935
936                        // expect must be solved before progress can be made.
937                        Poll::Pending => return Ok(PollResponse::DoNothing),
938                    }
939                }
940            }
941        }
942    }
943
944    fn handle_request(
945        mut self: Pin<&mut Self>,
946        req: Request,
947        cx: &mut Context<'_>,
948    ) -> Result<(), DispatchError> {
949        // initialize dispatcher state
950        {
951            let mut this = self.as_mut().project();
952
953            // Handle `EXPECT: 100-Continue` header
954            if req.head().expect() {
955                // set dispatcher state to call expect handler
956                let fut = this.flow.expect.call(req);
957                this.state.set(State::ExpectCall { fut });
958            } else {
959                // set dispatcher state to call service handler
960                let fut = this.flow.service.call(req);
961                this.state.set(State::ServiceCall { fut });
962            };
963        };
964
965        // eagerly poll the future once (or twice if expect is resolved immediately).
966        loop {
967            match self.as_mut().project().state.project() {
968                StateProj::ExpectCall { fut } => {
969                    match fut.poll(cx) {
970                        // expect is resolved; continue loop and poll the service call branch.
971                        Poll::Ready(Ok(req)) => {
972                            self.as_mut().send_continue();
973
974                            let mut this = self.as_mut().project();
975                            let fut = this.flow.service.call(req);
976                            this.state.set(State::ServiceCall { fut });
977
978                            continue;
979                        }
980
981                        // future is error; send response and return a result
982                        // on success to notify the dispatcher a new state is set and the outer loop
983                        // should be continued
984                        Poll::Ready(Err(err)) => {
985                            let res: Response<BoxBody> = err.into();
986                            let (res, body) = res.replace_body(());
987                            return self.send_error_response(res, body);
988                        }
989
990                        // future is pending; return Ok(()) to notify that a new state is
991                        // set and the outer loop should be continue.
992                        Poll::Pending => return Ok(()),
993                    }
994                }
995
996                StateProj::ServiceCall { fut } => {
997                    // return no matter the service call future's result.
998                    return match fut.poll(cx) {
999                        // Future is resolved. Send response and return a result. On success
1000                        // to notify the dispatcher a new state is set and the outer loop
1001                        // should be continue.
1002                        Poll::Ready(Ok(res)) => {
1003                            let (res, body) = res.into().replace_body(());
1004                            self.as_mut().send_response(res, body)
1005                        }
1006
1007                        // see the comment on ExpectCall state branch's Pending
1008                        Poll::Pending => Ok(()),
1009
1010                        // see the comment on ExpectCall state branch's Ready(Err(_))
1011                        Poll::Ready(Err(err)) => {
1012                            let res: Response<BoxBody> = err.into();
1013                            let (res, body) = res.replace_body(());
1014                            self.as_mut().send_error_response(res, body)
1015                        }
1016                    };
1017                }
1018
1019                _ => {
1020                    unreachable!("State must be set to ServiceCall or ExceptCall in handle_request")
1021                }
1022            }
1023        }
1024    }
1025
1026    /// Process one incoming request.
1027    ///
1028    /// Returns true if any meaningful work was done.
1029    fn poll_request(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Result<bool, DispatchError> {
1030        if self.flags.contains(Flags::DRAINING) && self.state.is_none() {
1031            return Ok(false);
1032        }
1033
1034        let pipeline_queue_full = self.messages.len() >= MAX_PIPELINED_MESSAGES;
1035        let can_not_read = !self.can_read(cx);
1036
1037        // limit amount of non-processed requests
1038        if pipeline_queue_full || can_not_read {
1039            return Ok(false);
1040        }
1041
1042        let mut this = self.as_mut().project();
1043
1044        let mut updated = false;
1045
1046        // decode from read buf as many full requests as possible
1047        loop {
1048            match this.codec.decode(this.read_buf) {
1049                Ok(Some(msg)) => {
1050                    updated = true;
1051
1052                    match msg {
1053                        Message::Item(mut req) => {
1054                            // Stop the head timer before reading the request payload.
1055                            this.head_timer.clear(line!());
1056
1057                            req.head_mut().peer_addr = *this.peer_addr;
1058
1059                            req.conn_data.clone_from(this.conn_data);
1060
1061                            match this.codec.message_type() {
1062                                // request has no payload
1063                                MessageType::None => *this.payload_drainable = false,
1064
1065                                // Request is upgradable. Add upgrade message and break.
1066                                // Everything remaining in read buffer will be handed to
1067                                // upgraded Request.
1068                                MessageType::Stream if this.flow.upgrade.is_some() => {
1069                                    *this.payload_drainable = false;
1070                                    this.messages.push_back(DispatcherMessage::Upgrade(req));
1071                                    break;
1072                                }
1073
1074                                // request is not upgradable
1075                                MessageType::Payload | MessageType::Stream => {
1076                                    // PayloadSender and Payload are smart pointers share the
1077                                    // same state. PayloadSender is attached to dispatcher and used
1078                                    // to sink new chunked request data to state. Payload is
1079                                    // attached to Request and passed to Service::call where the
1080                                    // state can be collected and consumed.
1081                                    let (sender, payload) = Payload::create(false);
1082                                    *req.payload() = crate::Payload::H1 { payload };
1083                                    *this.payload = Some(sender);
1084                                    *this.payload_drainable = req.chunked().unwrap_or(false);
1085                                }
1086                            }
1087
1088                            // handle request early when no future in InnerDispatcher state.
1089                            if this.state.is_none() {
1090                                self.as_mut().handle_request(req, cx)?;
1091                                this = self.as_mut().project();
1092                            } else {
1093                                this.messages.push_back(DispatcherMessage::Item(req));
1094                            }
1095                        }
1096
1097                        Message::Chunk(Some(chunk)) => {
1098                            if let Some(ref mut payload) = this.payload {
1099                                payload.feed_data(chunk);
1100                            } else {
1101                                error!("Internal server error: unexpected payload chunk");
1102                                this.flags.insert(Flags::READ_DISCONNECT);
1103                                this.messages.push_back(DispatcherMessage::Error(
1104                                    Response::internal_server_error().drop_body(),
1105                                ));
1106                                *this.error = Some(DispatchError::InternalError);
1107                                break;
1108                            }
1109                        }
1110
1111                        Message::Chunk(None) => {
1112                            if let Some(mut payload) = this.payload.take() {
1113                                payload.feed_eof();
1114                                *this.payload_drainable = false;
1115                            } else {
1116                                error!("Internal server error: unexpected eof");
1117                                this.flags.insert(Flags::READ_DISCONNECT);
1118                                this.messages.push_back(DispatcherMessage::Error(
1119                                    Response::internal_server_error().drop_body(),
1120                                ));
1121                                *this.error = Some(DispatchError::InternalError);
1122                                break;
1123                            }
1124                        }
1125                    }
1126                }
1127
1128                // Decode is partial.
1129                Ok(None) => {
1130                    if this.payload.is_none()
1131                        && this.codec.keep_alive()
1132                        && !this.read_buf.is_empty()
1133                        && !matches!(this.head_timer, TimerState::Active { .. })
1134                    {
1135                        // Subsequent requests on this connection get client deadlines too.
1136                        if let Some(deadline) = this.config.client_request_deadline() {
1137                            this.head_timer
1138                                .set_and_init(cx, sleep_until(deadline.into()), line!());
1139                        }
1140                    }
1141
1142                    // Needs more bytes read for full head.
1143                    break;
1144                }
1145
1146                Err(ParseError::Io(err)) => {
1147                    trace!("I/O error: {}", &err);
1148                    self.as_mut().client_disconnected();
1149                    this = self.as_mut().project();
1150                    *this.error = Some(DispatchError::Io(err));
1151                    break;
1152                }
1153
1154                Err(ParseError::TooLarge) => {
1155                    trace!("request head was too big; returning 431 response");
1156
1157                    if let Some(mut payload) = this.payload.take() {
1158                        payload.set_error(PayloadError::Overflow);
1159                    }
1160
1161                    // request heads that overflow buffer size return a 431 error
1162                    this.messages
1163                        .push_back(DispatcherMessage::Error(Response::with_body(
1164                            StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE,
1165                            (),
1166                        )));
1167
1168                    this.flags.insert(Flags::READ_DISCONNECT);
1169                    *this.error = Some(ParseError::TooLarge.into());
1170
1171                    break;
1172                }
1173
1174                Err(err) => {
1175                    trace!("parse error {}", &err);
1176
1177                    if let Some(mut payload) = this.payload.take() {
1178                        payload.set_error(PayloadError::EncodingCorrupted);
1179                    }
1180
1181                    // malformed requests should be responded with 400
1182                    this.messages.push_back(DispatcherMessage::Error(
1183                        Response::bad_request().drop_body(),
1184                    ));
1185
1186                    this.flags.insert(Flags::READ_DISCONNECT);
1187                    *this.error = Some(err.into());
1188                    break;
1189                }
1190            }
1191        }
1192
1193        Ok(updated)
1194    }
1195
1196    fn poll_head_timer(
1197        mut self: Pin<&mut Self>,
1198        cx: &mut Context<'_>,
1199    ) -> Result<(), DispatchError> {
1200        let this = self.as_mut().project();
1201
1202        if let TimerState::Active { timer } = this.head_timer {
1203            if timer.as_mut().poll(cx).is_ready() {
1204                // Return 408 when a request head takes too long.
1205
1206                trace!("timed out on slow request; replying with 408 and closing connection");
1207
1208                let _ = self.as_mut().send_error_response(
1209                    Response::with_body(StatusCode::REQUEST_TIMEOUT, ()),
1210                    BoxBody::new(()),
1211                );
1212
1213                self.project().flags.insert(Flags::SHUTDOWN);
1214            }
1215        };
1216
1217        Ok(())
1218    }
1219
1220    fn poll_ka_timer(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Result<(), DispatchError> {
1221        let this = self.as_mut().project();
1222        if let TimerState::Active { timer } = this.ka_timer {
1223            debug_assert!(
1224                this.flags.contains(Flags::KEEP_ALIVE),
1225                "keep-alive flag should be set when timer is active",
1226            );
1227            debug_assert!(
1228                this.state.is_none(),
1229                "dispatcher should not be in keep-alive phase if state is not none: {:?}",
1230                this.state,
1231            );
1232
1233            // Assert removed by @robjtede on account of issue #2655. There are cases where an I/O
1234            // flush can be pending after entering the keep-alive state causing the subsequent flush
1235            // wake up to panic here. This appears to be a Linux-only problem. Leaving original code
1236            // below for posterity because a simple and reliable test could not be found to trigger
1237            // the behavior.
1238            // debug_assert!(
1239            //     this.write_buf.is_empty(),
1240            //     "dispatcher should not be in keep-alive phase if write_buf is not empty",
1241            // );
1242
1243            // keep-alive timer has timed out
1244            if timer.as_mut().poll(cx).is_ready() {
1245                // no tasks at hand
1246                trace!("timer timed out; closing connection");
1247                this.flags.insert(Flags::SHUTDOWN);
1248
1249                if let Some(deadline) = this.config.client_disconnect_deadline() {
1250                    // start shutdown timeout if enabled
1251                    this.shutdown_timer
1252                        .set_and_init(cx, sleep_until(deadline.into()), line!());
1253                } else {
1254                    // no shutdown timeout, drop socket
1255                    this.flags.insert(Flags::WRITE_DISCONNECT);
1256                }
1257            }
1258        }
1259
1260        Ok(())
1261    }
1262
1263    fn poll_shutdown_timer(
1264        mut self: Pin<&mut Self>,
1265        cx: &mut Context<'_>,
1266    ) -> Result<(), DispatchError> {
1267        let this = self.as_mut().project();
1268        if let TimerState::Active { timer } = this.shutdown_timer {
1269            debug_assert!(
1270                this.flags.intersects(Flags::LINGER | Flags::SHUTDOWN),
1271                "shutdown or linger flag should be set when timer is active",
1272            );
1273
1274            if timer.as_mut().poll(cx).is_ready() {
1275                if this.flags.contains(Flags::LINGER) {
1276                    trace!("timed-out during linger; shutting down connection");
1277                    this.flags.remove(Flags::LINGER);
1278                    this.flags.insert(Flags::SHUTDOWN);
1279                    this.shutdown_timer.clear(line!());
1280                } else {
1281                    trace!("timed-out during shutdown");
1282                    return Err(DispatchError::DisconnectTimeout);
1283                }
1284            }
1285        }
1286
1287        Ok(())
1288    }
1289
1290    fn poll_graceful_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) {
1291        let this = self.as_mut().project();
1292
1293        let notified = this
1294            .graceful_shutdown
1295            .as_mut()
1296            .is_some_and(|signal| signal.as_mut().poll(cx).is_ready());
1297
1298        if notified {
1299            *this.graceful_shutdown = None;
1300            this.flags.remove(Flags::KEEP_ALIVE);
1301            this.flags.insert(Flags::DRAINING);
1302
1303            if this.ka_timer.is_enabled() {
1304                this.ka_timer.clear(line!());
1305            }
1306        }
1307    }
1308
1309    /// Poll head, keep-alive, and disconnect timer.
1310    fn poll_timers(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Result<(), DispatchError> {
1311        self.as_mut().poll_head_timer(cx)?;
1312        self.as_mut().poll_ka_timer(cx)?;
1313        self.as_mut().poll_shutdown_timer(cx)?;
1314
1315        Ok(())
1316    }
1317
1318    /// Returns true when I/O stream can be disconnected after write to it.
1319    ///
1320    /// It covers these conditions:
1321    /// - `std::io::ErrorKind::ConnectionReset` after partial read;
1322    /// - all data read done.
1323    #[inline(always)] // TODO: bench this inline
1324    fn read_available(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Result<bool, DispatchError> {
1325        let this = self.project();
1326
1327        if this.flags.contains(Flags::READ_DISCONNECT) {
1328            return Ok(false);
1329        };
1330
1331        let mut io = Pin::new(this.io.as_mut().unwrap());
1332
1333        let mut read_some = false;
1334
1335        loop {
1336            // Return early when read buf exceed decoder's max buffer size.
1337            if this.read_buf.len() >= MAX_BUFFER_SIZE {
1338                // At this point it's not known IO stream is still scheduled to be waked up so
1339                // force wake up dispatcher just in case.
1340                //
1341                // Reason:
1342                // AsyncRead mostly would only have guarantee wake up when the poll_read
1343                // return Poll::Pending.
1344                //
1345                // Case:
1346                // When read_buf is beyond max buffer size the early return could be successfully
1347                // be parsed as a new Request. This case would not generate ParseError::TooLarge and
1348                // at this point IO stream is not fully read to Pending and would result in
1349                // dispatcher stuck until timeout (keep-alive).
1350                //
1351                // Note:
1352                // This is a perf choice to reduce branch on <Request as MessageType>::decode.
1353                //
1354                // A Request head too large to parse is only checked on `httparse::Status::Partial`.
1355
1356                match this.payload.as_ref().map(|p| p.need_read(cx)) {
1357                    // Payload consumer is alive but applying backpressure. Wait for its waker.
1358                    Some(PayloadStatus::Pause) => {}
1359
1360                    // Consumer dropped means drain/discard mode; keep polling to make progress.
1361                    Some(PayloadStatus::Dropped) | Some(PayloadStatus::Read) | None => {
1362                        cx.waker().wake_by_ref()
1363                    }
1364                }
1365
1366                return Ok(false);
1367            }
1368
1369            // grow buffer if necessary.
1370            let remaining = this.read_buf.capacity() - this.read_buf.len();
1371            if remaining < LW_BUFFER_SIZE {
1372                this.read_buf.reserve(HW_BUFFER_SIZE - remaining);
1373            }
1374
1375            match tokio_util::io::poll_read_buf(io.as_mut(), cx, this.read_buf) {
1376                Poll::Ready(Ok(n)) => {
1377                    // When draining a dropped request payload, keep FINISHED set so the
1378                    // disconnect/keep-alive decision can be made once the payload is fully drained.
1379                    if !this.payload.as_ref().is_some_and(|pl| pl.is_dropped()) {
1380                        this.flags.remove(Flags::FINISHED);
1381                    }
1382
1383                    if n == 0 {
1384                        return Ok(true);
1385                    }
1386
1387                    read_some = true;
1388                }
1389
1390                Poll::Pending => {
1391                    return Ok(false);
1392                }
1393
1394                Poll::Ready(Err(err)) => {
1395                    return match err.kind() {
1396                        // convert WouldBlock error to the same as Pending return
1397                        io::ErrorKind::WouldBlock => Ok(false),
1398
1399                        // connection reset after partial read
1400                        io::ErrorKind::ConnectionReset if read_some => Ok(true),
1401
1402                        _ => Err(DispatchError::Io(err)),
1403                    };
1404                }
1405            }
1406        }
1407    }
1408
1409    /// call upgrade service with request.
1410    fn upgrade(self: Pin<&mut Self>, req: Request) -> U::Future {
1411        let mut this = self.project();
1412        let mut parts = FramedParts::with_read_buf(
1413            this.io.take().unwrap(),
1414            mem::take(this.codec),
1415            mem::take(this.read_buf),
1416        );
1417
1418        this.ensure_body_buffered();
1419
1420        parts.write_buf = mem::take(this.write_buf);
1421        let framed = Framed::from_parts(parts);
1422        this.flow.upgrade.as_ref().unwrap().call((req, framed))
1423    }
1424}
1425
1426impl<T, S, B, X, U> Future for Dispatcher<T, S, B, X, U>
1427where
1428    T: AsyncRead + AsyncWrite + Unpin,
1429
1430    S: Service<Request>,
1431    S::Error: Into<Response<BoxBody>>,
1432    S::Response: Into<Response<B>>,
1433
1434    B: MessageBody,
1435
1436    X: Service<Request, Response = Request>,
1437    X::Error: Into<Response<BoxBody>>,
1438
1439    U: Service<(Request, Framed<T, Codec>), Response = ()>,
1440    U::Error: fmt::Display,
1441{
1442    type Output = Result<(), DispatchError>;
1443
1444    #[inline]
1445    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
1446        let this = self.as_mut().project();
1447
1448        #[cfg(test)]
1449        {
1450            *this.poll_count += 1;
1451        }
1452
1453        match this.inner.project() {
1454            DispatcherStateProj::Upgrade { fut: upgrade } => upgrade.poll(cx).map_err(|err| {
1455                error!("Upgrade handler error: {}", err);
1456                DispatchError::Upgrade
1457            }),
1458
1459            DispatcherStateProj::Normal { mut inner } => {
1460                trace!("start flags: {:?}", &inner.flags);
1461
1462                trace_timer_states(
1463                    "start",
1464                    &inner.head_timer,
1465                    &inner.ka_timer,
1466                    &inner.shutdown_timer,
1467                );
1468
1469                inner.as_mut().poll_graceful_shutdown(cx);
1470                inner.as_mut().poll_timers(cx)?;
1471
1472                let poll = if inner.flags.contains(Flags::LINGER) {
1473                    match inner.as_mut().poll_linger(cx)? {
1474                        Poll::Ready(()) => {
1475                            cx.waker().wake_by_ref();
1476                            Poll::Pending
1477                        }
1478                        Poll::Pending => Poll::Pending,
1479                    }
1480                } else if inner.flags.contains(Flags::SHUTDOWN) {
1481                    if inner.flags.contains(Flags::WRITE_DISCONNECT) {
1482                        Poll::Ready(Ok(()))
1483                    } else {
1484                        // flush buffer and wait on blocked
1485                        ready!(inner.as_mut().poll_flush(cx))?;
1486                        Pin::new(inner.as_mut().project().io.as_mut().unwrap())
1487                            .poll_shutdown(cx)
1488                            .map_err(DispatchError::from)
1489                    }
1490                } else {
1491                    // read from I/O stream and fill read buffer
1492                    let should_disconnect = inner.as_mut().read_available(cx)?;
1493
1494                    // after reading something from stream, clear keep-alive timer
1495                    if !inner.read_buf.is_empty() && inner.flags.contains(Flags::KEEP_ALIVE) {
1496                        let inner = inner.as_mut().project();
1497                        inner.flags.remove(Flags::KEEP_ALIVE);
1498                        inner.ka_timer.clear(line!());
1499                    }
1500
1501                    if !inner.flags.contains(Flags::STARTED) {
1502                        inner.as_mut().project().flags.insert(Flags::STARTED);
1503
1504                        if let Some(deadline) = inner.config.client_request_deadline() {
1505                            inner.as_mut().project().head_timer.set_and_init(
1506                                cx,
1507                                sleep_until(deadline.into()),
1508                                line!(),
1509                            );
1510                        }
1511                    }
1512
1513                    inner.as_mut().poll_request(cx)?;
1514
1515                    if should_disconnect {
1516                        // I/O stream should to be closed
1517                        let inner = inner.as_mut().project();
1518                        inner.flags.insert(Flags::READ_DISCONNECT);
1519                        if let Some(mut payload) = inner.payload.take() {
1520                            payload.set_error(PayloadError::Incomplete(None));
1521                            payload.feed_eof();
1522                        }
1523                    };
1524
1525                    loop {
1526                        // poll response to populate write buffer
1527                        // drain indicates whether write buffer should be emptied before next run
1528                        let drain = match inner.as_mut().poll_response(cx)? {
1529                            PollResponse::DrainWriteBuf => true,
1530
1531                            PollResponse::DoNothing => {
1532                                // KEEP_ALIVE is set in send_response_inner if client allows it
1533                                // FINISHED is set after writing last chunk of response
1534                                // A partial head uses its own deadline instead of the idle timeout.
1535                                if inner.flags.contains(Flags::KEEP_ALIVE | Flags::FINISHED)
1536                                    && !matches!(inner.head_timer, TimerState::Active { .. })
1537                                {
1538                                    if let Some(timer) = inner.config.keep_alive_deadline() {
1539                                        inner.as_mut().project().ka_timer.set_and_init(
1540                                            cx,
1541                                            sleep_until(timer.into()),
1542                                            line!(),
1543                                        );
1544                                    }
1545                                }
1546
1547                                false
1548                            }
1549
1550                            // upgrade request and goes Upgrade variant of DispatcherState.
1551                            PollResponse::Upgrade(req) => {
1552                                let upgrade = inner.upgrade(req);
1553                                self.as_mut()
1554                                    .project()
1555                                    .inner
1556                                    .set(DispatcherState::Upgrade { fut: upgrade });
1557                                return self.poll(cx);
1558                            }
1559                        };
1560
1561                        // we didn't get WouldBlock from write operation, so data get written to
1562                        // kernel completely (macOS) and we have to write again otherwise response
1563                        // can get stuck
1564                        //
1565                        // TODO: want to find a reference for this behavior
1566                        // see introduced commit: 3872d3ba
1567                        let flush_was_ready = inner.as_mut().poll_flush(cx)?.is_ready();
1568
1569                        // this assert seems to always be true but not willing to commit to it until
1570                        // we understand what Nikolay meant when writing the above comment
1571                        // debug_assert!(flush_was_ready);
1572
1573                        if !flush_was_ready || !drain {
1574                            break;
1575                        }
1576                    }
1577
1578                    // client is gone
1579                    if inner.flags.contains(Flags::WRITE_DISCONNECT) {
1580                        trace!("client is gone; disconnecting");
1581                        return Poll::Ready(Ok(()));
1582                    }
1583
1584                    let inner_p = inner.as_mut().project();
1585                    let state_is_none = inner_p.state.is_none();
1586
1587                    // If the read-half is closed, we start the shutdown procedure if either is
1588                    // true:
1589                    //
1590                    // - state is [`State::None`], which means that we're done with request
1591                    //   processing, so if the client closed its writer-side it means that it won't
1592                    //   send more requests.
1593                    // - The user requested to not allow half-closures
1594                    if inner_p.flags.contains(Flags::READ_DISCONNECT)
1595                        && (!inner_p.config.h1_allow_half_closed() || state_is_none)
1596                    {
1597                        trace!("read half closed; start shutdown");
1598                        inner_p.flags.insert(Flags::SHUTDOWN);
1599                    }
1600
1601                    // keep-alive and stream errors
1602                    if state_is_none && inner_p.write_buf.is_empty() && inner_p.write_body.is_none()
1603                    {
1604                        if let Some(err) = inner_p.error.take() {
1605                            error!("stream error: {}", &err);
1606                            return Poll::Ready(Err(err));
1607                        }
1608
1609                        // disconnect if keep-alive is not enabled
1610                        if inner_p.flags.contains(Flags::FINISHED)
1611                            && !inner_p.flags.contains(Flags::KEEP_ALIVE)
1612                            && inner_p.payload.is_none()
1613                        {
1614                            inner_p.flags.remove(Flags::FINISHED);
1615                            inner_p.flags.insert(Flags::SHUTDOWN);
1616                            return self.poll(cx);
1617                        }
1618
1619                        // disconnect if shutdown
1620                        if inner_p.flags.contains(Flags::SHUTDOWN) {
1621                            return self.poll(cx);
1622                        }
1623                    }
1624
1625                    trace_timer_states(
1626                        "end",
1627                        inner_p.head_timer,
1628                        inner_p.ka_timer,
1629                        inner_p.shutdown_timer,
1630                    );
1631
1632                    if inner_p.flags.intersects(Flags::LINGER | Flags::SHUTDOWN) {
1633                        cx.waker().wake_by_ref();
1634                    }
1635                    Poll::Pending
1636                };
1637
1638                trace!("end flags: {:?}", &inner.flags);
1639
1640                poll
1641            }
1642        }
1643    }
1644}
1645
1646#[derive(Debug)]
1647enum ChunkDisposition {
1648    /// The chunk is stored in `write_body` for direct writing.
1649    WriteDirectly,
1650
1651    /// The caller must encode this chunk into `write_buf`.
1652    NeedsBuffering(Bytes),
1653}
1654
1655/// Stores a large chunk of a fixed-length body in `write_body` without copying it into `write_buf`.
1656///
1657/// Returns `WriteDirectly` if the chunk was stored, or `NeedsBuffering(chunk)` for the caller to encode
1658/// into `write_buf`.
1659///
1660/// Chunks smaller than `min_len` are always buffered. Connections without vectored writes also use
1661/// the buffer, so the head and body can be passed to the I/O in one write.
1662///
1663/// Uniquely owned chunks with spare capacity are also buffered. For example, the `String` produced
1664/// by JSON serialization can have more capacity than the body needs. Copying its contents into
1665/// `write_buf` lets the original allocation be freed instead of retaining that spare capacity while
1666/// the body is pending.
1667fn prepare_chunk(
1668    codec: &mut Codec,
1669    write_body: &mut Option<Bytes>,
1670    write_vectored: bool,
1671    min_len: usize,
1672    chunk: Bytes,
1673) -> ChunkDisposition {
1674    if chunk.len() < min_len || !write_vectored {
1675        return ChunkDisposition::NeedsBuffering(chunk);
1676    }
1677
1678    let chunk = match chunk.try_into_mut() {
1679        Ok(buf) if buf.capacity() > buf.len() => {
1680            return ChunkDisposition::NeedsBuffering(buf.freeze());
1681        }
1682        Ok(buf) => buf.freeze(),
1683        Err(chunk) => chunk,
1684    };
1685
1686    match codec.encoder.try_take_chunk(&chunk) {
1687        Some(body) => {
1688            *write_body = Some(body);
1689            ChunkDisposition::WriteDirectly
1690        }
1691        None => ChunkDisposition::NeedsBuffering(chunk),
1692    }
1693}
1694
1695fn should_close_for_unread_payload(
1696    payload: Option<&PayloadSender>,
1697    payload_drainable: bool,
1698) -> bool {
1699    let payload_unfinished = payload.is_some();
1700    let drain_payload = payload.is_some_and(|pl| pl.is_dropped()) && payload_drainable;
1701
1702    payload_unfinished && !drain_payload
1703}
1704
1705#[allow(dead_code)]
1706fn trace_timer_states(
1707    label: &str,
1708    head_timer: &TimerState,
1709    ka_timer: &TimerState,
1710    shutdown_timer: &TimerState,
1711) {
1712    trace!("{} timers:", label);
1713
1714    if head_timer.is_enabled() {
1715        trace!("  head {}", &head_timer);
1716    }
1717
1718    if ka_timer.is_enabled() {
1719        trace!("  keep-alive {}", &ka_timer);
1720    }
1721
1722    if shutdown_timer.is_enabled() {
1723        trace!("  shutdown {}", &shutdown_timer);
1724    }
1725}
1726
1727#[cfg(test)]
1728#[path = "dispatcher_tests.rs"]
1729mod dispatcher_tests;
1730
1731#[cfg(test)]
1732#[path = "dispatcher_buffer_tests.rs"]
1733mod buffer_tests;