1use std::{
2 cmp,
3 collections::VecDeque,
4 fmt,
5 future::Future,
6 io, mem, net,
7 pin::Pin,
8 rc::Rc,
9 task::{Context, Poll},
10};
11
12use actix_codec::{Framed, FramedParts};
13use actix_service::Service;
14use bitflags::bitflags;
15use bytes::{Buf, Bytes, BytesMut};
16use futures_core::ready;
17use pin_project_lite::pin_project;
18use tokio::{
19 io::{AsyncRead, AsyncWrite},
20 time::sleep_until,
21};
22use tokio_util::codec::{Decoder as _, Encoder as _};
23use tracing::{error, trace};
24
25use super::{
26 codec::Codec,
27 decoder::MAX_BUFFER_SIZE,
28 payload::{Payload, PayloadSender, PayloadStatus},
29 timer::TimerState,
30 Message, MessageType,
31};
32use crate::{
33 body::{BodySize, BoxBody, MessageBody},
34 config::ServiceConfig,
35 error::{DispatchError, ParseError, PayloadError},
36 service::HttpFlow,
37 ConnectionType, Error, Extensions, HttpMessage, OnConnectData, Request, Response, StatusCode,
38};
39
40const LW_BUFFER_SIZE: usize = 1024;
41const HW_BUFFER_SIZE: usize = 1024 * 8;
42const MAX_PIPELINED_MESSAGES: usize = 16;
43
44const WRITE_BUFFER_RETENTION_LIMIT: usize = 1024 * 128;
46
47bitflags! {
48 #[derive(Debug, Clone, Copy)]
49 pub struct Flags: u8 {
50 const STARTED = 0b0000_0001;
52
53 const FINISHED = 0b0000_0010;
55
56 const KEEP_ALIVE = 0b0000_0100;
58
59 const SHUTDOWN = 0b0000_1000;
61
62 const READ_DISCONNECT = 0b0001_0000;
64
65 const WRITE_DISCONNECT = 0b0010_0000;
67
68 const LINGER = 0b0100_0000;
70
71 const DRAINING = 0b1000_0000;
78 }
79}
80
81#[cfg(not(test))]
87pin_project! {
88 pub struct Dispatcher<T, S, B, X, U>
90 where
91 S: Service<Request>,
92 S::Error: Into<Response<BoxBody>>,
93
94 B: MessageBody,
95
96 X: Service<Request, Response = Request>,
97 X::Error: Into<Response<BoxBody>>,
98
99 U: Service<(Request, Framed<T, Codec>), Response = ()>,
100 U::Error: fmt::Display,
101 {
102 #[pin]
103 inner: DispatcherState<T, S, B, X, U>,
104 }
105}
106
107#[cfg(test)]
108pin_project! {
109 pub struct Dispatcher<T, S, B, X, U>
111 where
112 S: Service<Request>,
113 S::Error: Into<Response<BoxBody>>,
114
115 B: MessageBody,
116
117 X: Service<Request, Response = Request>,
118 X::Error: Into<Response<BoxBody>>,
119
120 U: Service<(Request, Framed<T, Codec>), Response = ()>,
121 U::Error: fmt::Display,
122 {
123 #[pin]
124 pub(super) inner: DispatcherState<T, S, B, X, U>,
125
126 pub(super) poll_count: u64,
128 }
129}
130
131pin_project! {
132 #[project = DispatcherStateProj]
133 pub(super) enum DispatcherState<T, S, B, X, U>
134 where
135 S: Service<Request>,
136 S::Error: Into<Response<BoxBody>>,
137
138 B: MessageBody,
139
140 X: Service<Request, Response = Request>,
141 X::Error: Into<Response<BoxBody>>,
142
143 U: Service<(Request, Framed<T, Codec>), Response = ()>,
144 U::Error: fmt::Display,
145 {
146 Normal { #[pin] inner: InnerDispatcher<T, S, B, X, U> },
147 Upgrade { #[pin] fut: U::Future },
148 }
149}
150
151pin_project! {
152 #[project = InnerDispatcherProj]
153 pub(super) struct InnerDispatcher<T, S, B, X, U>
154 where
155 S: Service<Request>,
156 S::Error: Into<Response<BoxBody>>,
157
158 B: MessageBody,
159
160 X: Service<Request, Response = Request>,
161 X::Error: Into<Response<BoxBody>>,
162
163 U: Service<(Request, Framed<T, Codec>), Response = ()>,
164 U::Error: fmt::Display,
165 {
166 flow: Rc<HttpFlow<S, X, U>>,
167 pub(super) flags: Flags,
168 peer_addr: Option<net::SocketAddr>,
169 conn_data: Option<Rc<Extensions>>,
170 config: ServiceConfig,
171 error: Option<DispatchError>,
172
173 #[pin]
174 pub(super) state: State<S, B, X>,
175 payload: Option<PayloadSender>,
177 payload_drainable: bool,
179 messages: VecDeque<DispatcherMessage>,
180
181 head_timer: TimerState,
182 ka_timer: TimerState,
183 shutdown_timer: TimerState,
184 graceful_shutdown: Option<crate::config::GracefulShutdownFuture>,
185
186 pub(super) io: Option<T>,
187 read_buf: BytesMut,
188
189 write_buf: BytesMut,
195
196 write_body: Option<Bytes>,
209
210 h1_write_buffer_size: usize,
212
213 write_buf_needs_reset: bool,
215
216 codec: Codec,
217 }
218}
219
220impl<T, S, B, X, U> InnerDispatcherProj<'_, T, S, B, X, U>
221where
222 S: Service<Request>,
223 S::Error: Into<Response<BoxBody>>,
224
225 B: MessageBody,
226
227 X: Service<Request, Response = Request>,
228 X::Error: Into<Response<BoxBody>>,
229
230 U: Service<(Request, Framed<T, Codec>), Response = ()>,
231 U::Error: fmt::Display,
232{
233 fn ensure_body_buffered(&mut self) {
235 if let Some(body) = self.write_body.take() {
236 self.write_buf.extend_from_slice(&body);
237 }
238 }
239}
240
241enum DispatcherMessage {
242 Item(Request),
243 Upgrade(Request),
244 Error(Response<()>),
245}
246
247pin_project! {
248 #[project = StateProj]
249 pub(super) enum State<S, B, X>
250 where
251 S: Service<Request>,
252 X: Service<Request, Response = Request>,
253 B: MessageBody,
254 {
255 None,
256 ExpectCall { #[pin] fut: X::Future },
257 ServiceCall { #[pin] fut: S::Future },
258 SendPayload { #[pin] body: B },
259 SendErrorPayload { #[pin] body: BoxBody },
260 }
261}
262
263impl<S, B, X> State<S, B, X>
264where
265 S: Service<Request>,
266 X: Service<Request, Response = Request>,
267 B: MessageBody,
268{
269 pub(super) fn is_none(&self) -> bool {
270 matches!(self, State::None)
271 }
272}
273
274impl<S, B, X> fmt::Debug for State<S, B, X>
275where
276 S: Service<Request>,
277 X: Service<Request, Response = Request>,
278 B: MessageBody,
279{
280 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
281 match self {
282 Self::None => write!(f, "State::None"),
283 Self::ExpectCall { .. } => f.debug_struct("State::ExpectCall").finish_non_exhaustive(),
284 Self::ServiceCall { .. } => {
285 f.debug_struct("State::ServiceCall").finish_non_exhaustive()
286 }
287 Self::SendPayload { .. } => {
288 f.debug_struct("State::SendPayload").finish_non_exhaustive()
289 }
290 Self::SendErrorPayload { .. } => f
291 .debug_struct("State::SendErrorPayload")
292 .finish_non_exhaustive(),
293 }
294 }
295}
296
297#[derive(Debug)]
298enum PollResponse {
299 Upgrade(Request),
300 DoNothing,
301 DrainWriteBuf,
302}
303
304impl<T, S, B, X, U> Dispatcher<T, S, B, X, U>
305where
306 T: AsyncRead + AsyncWrite + Unpin,
307
308 S: Service<Request>,
309 S::Error: Into<Response<BoxBody>>,
310 S::Response: Into<Response<B>>,
311
312 B: MessageBody,
313
314 X: Service<Request, Response = Request>,
315 X::Error: Into<Response<BoxBody>>,
316
317 U: Service<(Request, Framed<T, Codec>), Response = ()>,
318 U::Error: fmt::Display,
319{
320 pub(crate) fn new(
322 io: T,
323 flow: Rc<HttpFlow<S, X, U>>,
324 config: ServiceConfig,
325 peer_addr: Option<net::SocketAddr>,
326 conn_data: OnConnectData,
327 ) -> Self {
328 Dispatcher {
329 inner: DispatcherState::Normal {
330 inner: InnerDispatcher {
331 flow,
332 flags: Flags::empty(),
333 peer_addr,
334 conn_data: conn_data.0.map(Rc::new),
335 config: config.clone(),
336 error: None,
337
338 state: State::None,
339 payload: None,
340 payload_drainable: false,
341 messages: VecDeque::new(),
342
343 head_timer: TimerState::new(config.client_request_deadline().is_some()),
344 ka_timer: TimerState::new(config.keep_alive().enabled()),
345 shutdown_timer: TimerState::new(config.client_disconnect_deadline().is_some()),
346 graceful_shutdown: config.graceful_shutdown(),
347
348 io: Some(io),
349 read_buf: BytesMut::with_capacity(HW_BUFFER_SIZE),
350 write_buf: BytesMut::with_capacity(HW_BUFFER_SIZE),
351 write_body: None,
352 h1_write_buffer_size: config.h1_write_buffer_size(),
353 write_buf_needs_reset: false,
354 codec: Codec::new(config),
355 },
356 },
357
358 #[cfg(test)]
359 poll_count: 0,
360 }
361 }
362}
363
364impl<T, S, B, X, U> InnerDispatcher<T, S, B, X, U>
365where
366 T: AsyncRead + AsyncWrite + Unpin,
367
368 S: Service<Request>,
369 S::Error: Into<Response<BoxBody>>,
370 S::Response: Into<Response<B>>,
371
372 B: MessageBody,
373
374 X: Service<Request, Response = Request>,
375 X::Error: Into<Response<BoxBody>>,
376
377 U: Service<(Request, Framed<T, Codec>), Response = ()>,
378 U::Error: fmt::Display,
379{
380 fn can_read(&self, cx: &mut Context<'_>) -> bool {
381 if self.flags.contains(Flags::READ_DISCONNECT) {
382 false
383 } else if let Some(ref info) = self.payload {
384 matches!(
385 info.need_read(cx),
386 PayloadStatus::Read | PayloadStatus::Dropped
387 )
388 } else {
389 true
390 }
391 }
392
393 fn client_disconnected(self: Pin<&mut Self>) {
394 let this = self.project();
395
396 this.flags
397 .insert(Flags::READ_DISCONNECT | Flags::WRITE_DISCONNECT);
398
399 if let Some(mut payload) = this.payload.take() {
400 payload.set_error(PayloadError::Incomplete(None));
401 }
402 }
403
404 fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
405 let InnerDispatcherProj {
406 io,
407 write_buf,
408 write_body,
409 h1_write_buffer_size,
410 write_buf_needs_reset,
411 ..
412 } = self.project();
413
414 let mut io = Pin::new(io.as_mut().unwrap());
415
416 let retention_limit = (*h1_write_buffer_size).max(WRITE_BUFFER_RETENTION_LIMIT);
419
420 if write_buf.capacity() > retention_limit {
421 *write_buf_needs_reset = true;
424 }
425
426 let write_buf_len = write_buf.len();
427 let mut write_buf_written = 0;
428
429 loop {
430 let head = &write_buf[write_buf_written..];
431 let head_len = head.len();
432
433 let polled = match (head, write_body.as_deref()) {
434 ([], None | Some([])) => break,
436
437 ([], Some(body)) => io.as_mut().poll_write(cx, body)?,
439
440 (head, Some(body @ [_, ..])) if io.is_write_vectored() => io
442 .as_mut()
443 .poll_write_vectored(cx, &[io::IoSlice::new(head), io::IoSlice::new(body)])?,
444
445 (head, _) => io.as_mut().poll_write(cx, head)?,
447 };
448
449 match polled {
450 Poll::Ready(0) => {
451 error!("write zero; closing");
452 return Poll::Ready(Err(io::Error::new(io::ErrorKind::WriteZero, "")));
453 }
454
455 Poll::Ready(n) => {
456 let from_head = cmp::min(n, head_len);
460 write_buf_written += from_head;
461
462 if let Some(body) = write_body.as_mut() {
463 let from_body = n - from_head;
464 body.advance(from_body);
465 }
466 }
467
468 Poll::Pending => {
469 write_buf.advance(write_buf_written);
472
473 return Poll::Pending;
474 }
475 }
476 }
477
478 debug_assert_eq!(
479 write_buf_written, write_buf_len,
480 "All buffered bytes should be written before leaving the write loop"
481 );
482
483 if *write_buf_needs_reset {
485 *write_buf = BytesMut::with_capacity(HW_BUFFER_SIZE);
486 *write_buf_needs_reset = false;
487 } else {
488 write_buf.clear();
489 }
490
491 *write_body = None;
493
494 io.poll_flush(cx)
496 }
497
498 fn enter_linger(flags: &mut Flags) {
499 flags.remove(Flags::KEEP_ALIVE);
500 flags.insert(Flags::LINGER | Flags::FINISHED);
501 }
502
503 fn ensure_linger_timer(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> bool {
504 let this = self.as_mut().project();
505
506 if matches!(this.shutdown_timer, TimerState::Active { .. }) {
507 return true;
508 }
509
510 if let Some(deadline) = this.config.client_disconnect_deadline() {
511 this.shutdown_timer
512 .set_and_init(cx, sleep_until(deadline.into()), line!());
513 true
514 } else {
515 false
516 }
517 }
518
519 fn poll_linger(
520 mut self: Pin<&mut Self>,
521 cx: &mut Context<'_>,
522 ) -> Result<Poll<()>, DispatchError> {
523 if self.as_mut().poll_flush(cx)?.is_pending() {
524 return Ok(Poll::Pending);
525 }
526
527 if !self.as_mut().ensure_linger_timer(cx) {
528 let this = self.as_mut().project();
529 this.flags.remove(Flags::LINGER);
530 this.flags.insert(Flags::SHUTDOWN);
531 return Ok(Poll::Ready(()));
532 }
533
534 loop {
535 let should_disconnect = self.as_mut().read_available(cx)?;
536 let this = self.as_mut().project();
537 let mut progressed = false;
538
539 if !this.read_buf.is_empty() {
540 this.read_buf.clear();
541 progressed = true;
542 }
543
544 if should_disconnect {
545 this.flags.remove(Flags::LINGER);
546 this.flags.insert(Flags::READ_DISCONNECT | Flags::SHUTDOWN);
547 return Ok(Poll::Ready(()));
548 }
549
550 if !progressed {
551 return Ok(Poll::Pending);
552 }
553 }
554 }
555
556 fn send_response_inner(
557 self: Pin<&mut Self>,
558 res: Response<()>,
559 body: &impl MessageBody,
560 ) -> Result<BodySize, DispatchError> {
561 let this = self.project();
562
563 let size = body.size();
564
565 this.codec
566 .encode(Message::Item((res, size)), this.write_buf)
567 .map_err(|err| {
568 if let Some(mut payload) = this.payload.take() {
569 payload.set_error(PayloadError::Incomplete(None));
570 }
571
572 DispatchError::Io(err)
573 })?;
574
575 Ok(size)
576 }
577
578 fn send_response(
579 mut self: Pin<&mut Self>,
580 mut res: Response<()>,
581 body: B,
582 ) -> Result<(), DispatchError> {
583 let is_upgrade = res.upgrade();
584 let (draining, close_for_unread_payload) = {
585 let this = self.as_mut().project();
586 (
587 this.flags.contains(Flags::DRAINING),
588 !is_upgrade
589 && should_close_for_unread_payload(
590 this.payload.as_ref(),
591 *this.payload_drainable,
592 ),
593 )
594 };
595 let close_after_response = (!is_upgrade && draining) || close_for_unread_payload;
596
597 if close_after_response {
598 res.head_mut().set_connection_type(ConnectionType::Close);
599 }
600
601 let size = self.as_mut().send_response_inner(res, &body)?;
602 match size {
603 BodySize::None | BodySize::Sized(0) => {
604 let mut this = self.as_mut().project();
605
606 if close_for_unread_payload {
607 if this.config.client_disconnect_deadline().is_some() {
608 Self::enter_linger(this.flags);
609 } else {
610 this.flags.insert(Flags::SHUTDOWN | Flags::FINISHED);
611 }
612 } else {
613 this.flags.insert(Flags::FINISHED);
614 }
615
616 this.state.set(State::None);
617 }
618 _ => self
619 .as_mut()
620 .project()
621 .state
622 .set(State::SendPayload { body }),
623 }
624
625 Ok(())
626 }
627
628 fn send_error_response(
629 mut self: Pin<&mut Self>,
630 mut res: Response<()>,
631 body: BoxBody,
632 ) -> Result<(), DispatchError> {
633 let is_upgrade = res.upgrade();
634 let (draining, close_for_unread_payload) = {
635 let this = self.as_mut().project();
636 (
637 this.flags.contains(Flags::DRAINING),
638 !is_upgrade
639 && should_close_for_unread_payload(
640 this.payload.as_ref(),
641 *this.payload_drainable,
642 ),
643 )
644 };
645 let close_after_response = (!is_upgrade && draining) || close_for_unread_payload;
646
647 if close_after_response {
648 res.head_mut().set_connection_type(ConnectionType::Close);
649 }
650
651 let size = self.as_mut().send_response_inner(res, &body)?;
652 match size {
653 BodySize::None | BodySize::Sized(0) => {
654 let mut this = self.as_mut().project();
655
656 if close_for_unread_payload {
657 if this.config.client_disconnect_deadline().is_some() {
658 Self::enter_linger(this.flags);
659 } else {
660 this.flags.insert(Flags::SHUTDOWN | Flags::FINISHED);
661 }
662 } else {
663 this.flags.insert(Flags::FINISHED);
664 }
665
666 this.state.set(State::None);
667 }
668 _ => self
669 .as_mut()
670 .project()
671 .state
672 .set(State::SendErrorPayload { body }),
673 }
674
675 Ok(())
676 }
677
678 fn send_continue(self: Pin<&mut Self>) {
679 let mut this = self.project();
680
681 this.ensure_body_buffered();
682
683 this.write_buf
684 .extend_from_slice(b"HTTP/1.1 100 Continue\r\n\r\n");
685 }
686
687 fn poll_response(
688 mut self: Pin<&mut Self>,
689 cx: &mut Context<'_>,
690 ) -> Result<PollResponse, DispatchError> {
691 'res: loop {
692 let mut this = self.as_mut().project();
693 match this.state.as_mut().project() {
694 StateProj::None if this.flags.contains(Flags::DRAINING) => {
695 this.messages.clear();
696 this.flags.remove(Flags::KEEP_ALIVE);
697
698 if !this.flags.contains(Flags::LINGER) {
699 this.flags.insert(Flags::SHUTDOWN);
700 }
701
702 return Ok(PollResponse::DoNothing);
703 }
704
705 StateProj::None => match this.messages.pop_front() {
707 Some(DispatcherMessage::Item(req)) => {
709 if req.head().expect() {
711 let fut = this.flow.expect.call(req);
713 this.state.set(State::ExpectCall { fut });
714 } else {
715 let fut = this.flow.service.call(req);
717 this.state.set(State::ServiceCall { fut });
718 };
719 }
720
721 Some(DispatcherMessage::Error(res)) => {
723 self.as_mut().send_error_response(res, BoxBody::new(()))?;
727 }
728
729 Some(DispatcherMessage::Upgrade(req)) => return Ok(PollResponse::Upgrade(req)),
731
732 None => {
734 this.flags.set(
736 Flags::KEEP_ALIVE,
737 this.payload.is_none() && this.codec.keep_alive(),
738 );
739
740 return Ok(PollResponse::DoNothing);
741 }
742 },
743
744 StateProj::ServiceCall { fut } => {
745 match fut.poll(cx) {
746 Poll::Ready(Ok(res)) => {
748 let (res, body) = res.into().replace_body(());
749 self.as_mut().send_response(res, body)?;
750 }
751
752 Poll::Ready(Err(err)) => {
754 let res: Response<BoxBody> = err.into();
755 let (res, body) = res.replace_body(());
756 self.as_mut().send_error_response(res, body)?;
757 }
758
759 Poll::Pending => {
762 if !self.as_mut().poll_request(cx)? {
765 return Ok(PollResponse::DoNothing);
766 }
767 }
769 }
770 }
771
772 StateProj::SendPayload { mut body } => {
773 while this.write_buf.len() < *this.h1_write_buffer_size
776 && this.write_body.is_none()
777 {
778 match body.as_mut().poll_next(cx) {
779 Poll::Ready(Some(Ok(chunk))) => {
780 let write_vectored =
781 this.io.as_ref().is_some_and(|io| io.is_write_vectored());
782
783 if let ChunkDisposition::NeedsBuffering(item) = prepare_chunk(
784 this.codec,
785 this.write_body,
786 write_vectored,
787 *this.h1_write_buffer_size,
788 chunk,
789 ) {
790 this.codec
791 .encode(Message::Chunk(Some(item)), this.write_buf)?;
792 }
793 }
794
795 Poll::Ready(None) => {
796 this.codec.encode(Message::Chunk(None), this.write_buf)?;
797
798 let close_for_unread_payload = should_close_for_unread_payload(
803 this.payload.as_ref(),
804 *this.payload_drainable,
805 );
806 let not_pipelined = this.messages.is_empty();
807
808 this.state.set(State::None);
811
812 if not_pipelined && close_for_unread_payload {
813 if this.config.client_disconnect_deadline().is_some() {
814 Self::enter_linger(this.flags);
815 } else {
816 this.flags.insert(Flags::SHUTDOWN | Flags::FINISHED);
817 }
818 } else {
819 this.flags.insert(Flags::FINISHED);
820 }
821
822 continue 'res;
823 }
824
825 Poll::Ready(Some(Err(err))) => {
826 let err = err.into();
827 tracing::error!("Response payload stream error: {err:?}");
828 this.flags.insert(Flags::FINISHED);
829 return Err(DispatchError::Body(err));
830 }
831
832 Poll::Pending => return Ok(PollResponse::DoNothing),
833 }
834 }
835
836 return Ok(PollResponse::DrainWriteBuf);
839 }
840
841 StateProj::SendErrorPayload { mut body } => {
842 while this.write_buf.len() < *this.h1_write_buffer_size
847 && this.write_body.is_none()
848 {
849 match body.as_mut().poll_next(cx) {
850 Poll::Ready(Some(Ok(chunk))) => {
851 let write_vectored =
852 this.io.as_ref().is_some_and(|io| io.is_write_vectored());
853
854 if let ChunkDisposition::NeedsBuffering(item) = prepare_chunk(
855 this.codec,
856 this.write_body,
857 write_vectored,
858 *this.h1_write_buffer_size,
859 chunk,
860 ) {
861 this.codec
862 .encode(Message::Chunk(Some(item)), this.write_buf)?;
863 }
864 }
865
866 Poll::Ready(None) => {
867 this.codec.encode(Message::Chunk(None), this.write_buf)?;
868
869 let close_for_unread_payload = should_close_for_unread_payload(
874 this.payload.as_ref(),
875 *this.payload_drainable,
876 );
877 let not_pipelined = this.messages.is_empty();
878
879 this.state.set(State::None);
882
883 if not_pipelined && close_for_unread_payload {
884 if this.config.client_disconnect_deadline().is_some() {
885 Self::enter_linger(this.flags);
886 } else {
887 this.flags.insert(Flags::SHUTDOWN | Flags::FINISHED);
888 }
889 } else {
890 this.flags.insert(Flags::FINISHED);
891 }
892
893 continue 'res;
894 }
895
896 Poll::Ready(Some(Err(err))) => {
897 tracing::error!("Response payload stream error: {err:?}");
898 this.flags.insert(Flags::FINISHED);
899 return Err(DispatchError::Body(
900 Error::new_body().with_cause(err).into(),
901 ));
902 }
903
904 Poll::Pending => return Ok(PollResponse::DoNothing),
905 }
906 }
907
908 return Ok(PollResponse::DrainWriteBuf);
911 }
912
913 StateProj::ExpectCall { fut } => {
914 trace!(" calling expect service");
915
916 match fut.poll(cx) {
917 Poll::Ready(Ok(req)) => {
920 this.ensure_body_buffered();
921
922 this.write_buf
923 .extend_from_slice(b"HTTP/1.1 100 Continue\r\n\r\n");
924
925 let fut = this.flow.service.call(req);
926 this.state.set(State::ServiceCall { fut });
927 }
928
929 Poll::Ready(Err(err)) => {
931 let res: Response<BoxBody> = err.into();
932 let (res, body) = res.replace_body(());
933 self.as_mut().send_error_response(res, body)?;
934 }
935
936 Poll::Pending => return Ok(PollResponse::DoNothing),
938 }
939 }
940 }
941 }
942 }
943
944 fn handle_request(
945 mut self: Pin<&mut Self>,
946 req: Request,
947 cx: &mut Context<'_>,
948 ) -> Result<(), DispatchError> {
949 {
951 let mut this = self.as_mut().project();
952
953 if req.head().expect() {
955 let fut = this.flow.expect.call(req);
957 this.state.set(State::ExpectCall { fut });
958 } else {
959 let fut = this.flow.service.call(req);
961 this.state.set(State::ServiceCall { fut });
962 };
963 };
964
965 loop {
967 match self.as_mut().project().state.project() {
968 StateProj::ExpectCall { fut } => {
969 match fut.poll(cx) {
970 Poll::Ready(Ok(req)) => {
972 self.as_mut().send_continue();
973
974 let mut this = self.as_mut().project();
975 let fut = this.flow.service.call(req);
976 this.state.set(State::ServiceCall { fut });
977
978 continue;
979 }
980
981 Poll::Ready(Err(err)) => {
985 let res: Response<BoxBody> = err.into();
986 let (res, body) = res.replace_body(());
987 return self.send_error_response(res, body);
988 }
989
990 Poll::Pending => return Ok(()),
993 }
994 }
995
996 StateProj::ServiceCall { fut } => {
997 return match fut.poll(cx) {
999 Poll::Ready(Ok(res)) => {
1003 let (res, body) = res.into().replace_body(());
1004 self.as_mut().send_response(res, body)
1005 }
1006
1007 Poll::Pending => Ok(()),
1009
1010 Poll::Ready(Err(err)) => {
1012 let res: Response<BoxBody> = err.into();
1013 let (res, body) = res.replace_body(());
1014 self.as_mut().send_error_response(res, body)
1015 }
1016 };
1017 }
1018
1019 _ => {
1020 unreachable!("State must be set to ServiceCall or ExceptCall in handle_request")
1021 }
1022 }
1023 }
1024 }
1025
1026 fn poll_request(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Result<bool, DispatchError> {
1030 if self.flags.contains(Flags::DRAINING) && self.state.is_none() {
1031 return Ok(false);
1032 }
1033
1034 let pipeline_queue_full = self.messages.len() >= MAX_PIPELINED_MESSAGES;
1035 let can_not_read = !self.can_read(cx);
1036
1037 if pipeline_queue_full || can_not_read {
1039 return Ok(false);
1040 }
1041
1042 let mut this = self.as_mut().project();
1043
1044 let mut updated = false;
1045
1046 loop {
1048 match this.codec.decode(this.read_buf) {
1049 Ok(Some(msg)) => {
1050 updated = true;
1051
1052 match msg {
1053 Message::Item(mut req) => {
1054 this.head_timer.clear(line!());
1056
1057 req.head_mut().peer_addr = *this.peer_addr;
1058
1059 req.conn_data.clone_from(this.conn_data);
1060
1061 match this.codec.message_type() {
1062 MessageType::None => *this.payload_drainable = false,
1064
1065 MessageType::Stream if this.flow.upgrade.is_some() => {
1069 *this.payload_drainable = false;
1070 this.messages.push_back(DispatcherMessage::Upgrade(req));
1071 break;
1072 }
1073
1074 MessageType::Payload | MessageType::Stream => {
1076 let (sender, payload) = Payload::create(false);
1082 *req.payload() = crate::Payload::H1 { payload };
1083 *this.payload = Some(sender);
1084 *this.payload_drainable = req.chunked().unwrap_or(false);
1085 }
1086 }
1087
1088 if this.state.is_none() {
1090 self.as_mut().handle_request(req, cx)?;
1091 this = self.as_mut().project();
1092 } else {
1093 this.messages.push_back(DispatcherMessage::Item(req));
1094 }
1095 }
1096
1097 Message::Chunk(Some(chunk)) => {
1098 if let Some(ref mut payload) = this.payload {
1099 payload.feed_data(chunk);
1100 } else {
1101 error!("Internal server error: unexpected payload chunk");
1102 this.flags.insert(Flags::READ_DISCONNECT);
1103 this.messages.push_back(DispatcherMessage::Error(
1104 Response::internal_server_error().drop_body(),
1105 ));
1106 *this.error = Some(DispatchError::InternalError);
1107 break;
1108 }
1109 }
1110
1111 Message::Chunk(None) => {
1112 if let Some(mut payload) = this.payload.take() {
1113 payload.feed_eof();
1114 *this.payload_drainable = false;
1115 } else {
1116 error!("Internal server error: unexpected eof");
1117 this.flags.insert(Flags::READ_DISCONNECT);
1118 this.messages.push_back(DispatcherMessage::Error(
1119 Response::internal_server_error().drop_body(),
1120 ));
1121 *this.error = Some(DispatchError::InternalError);
1122 break;
1123 }
1124 }
1125 }
1126 }
1127
1128 Ok(None) => {
1130 if this.payload.is_none()
1131 && this.codec.keep_alive()
1132 && !this.read_buf.is_empty()
1133 && !matches!(this.head_timer, TimerState::Active { .. })
1134 {
1135 if let Some(deadline) = this.config.client_request_deadline() {
1137 this.head_timer
1138 .set_and_init(cx, sleep_until(deadline.into()), line!());
1139 }
1140 }
1141
1142 break;
1144 }
1145
1146 Err(ParseError::Io(err)) => {
1147 trace!("I/O error: {}", &err);
1148 self.as_mut().client_disconnected();
1149 this = self.as_mut().project();
1150 *this.error = Some(DispatchError::Io(err));
1151 break;
1152 }
1153
1154 Err(ParseError::TooLarge) => {
1155 trace!("request head was too big; returning 431 response");
1156
1157 if let Some(mut payload) = this.payload.take() {
1158 payload.set_error(PayloadError::Overflow);
1159 }
1160
1161 this.messages
1163 .push_back(DispatcherMessage::Error(Response::with_body(
1164 StatusCode::REQUEST_HEADER_FIELDS_TOO_LARGE,
1165 (),
1166 )));
1167
1168 this.flags.insert(Flags::READ_DISCONNECT);
1169 *this.error = Some(ParseError::TooLarge.into());
1170
1171 break;
1172 }
1173
1174 Err(err) => {
1175 trace!("parse error {}", &err);
1176
1177 if let Some(mut payload) = this.payload.take() {
1178 payload.set_error(PayloadError::EncodingCorrupted);
1179 }
1180
1181 this.messages.push_back(DispatcherMessage::Error(
1183 Response::bad_request().drop_body(),
1184 ));
1185
1186 this.flags.insert(Flags::READ_DISCONNECT);
1187 *this.error = Some(err.into());
1188 break;
1189 }
1190 }
1191 }
1192
1193 Ok(updated)
1194 }
1195
1196 fn poll_head_timer(
1197 mut self: Pin<&mut Self>,
1198 cx: &mut Context<'_>,
1199 ) -> Result<(), DispatchError> {
1200 let this = self.as_mut().project();
1201
1202 if let TimerState::Active { timer } = this.head_timer {
1203 if timer.as_mut().poll(cx).is_ready() {
1204 trace!("timed out on slow request; replying with 408 and closing connection");
1207
1208 let _ = self.as_mut().send_error_response(
1209 Response::with_body(StatusCode::REQUEST_TIMEOUT, ()),
1210 BoxBody::new(()),
1211 );
1212
1213 self.project().flags.insert(Flags::SHUTDOWN);
1214 }
1215 };
1216
1217 Ok(())
1218 }
1219
1220 fn poll_ka_timer(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Result<(), DispatchError> {
1221 let this = self.as_mut().project();
1222 if let TimerState::Active { timer } = this.ka_timer {
1223 debug_assert!(
1224 this.flags.contains(Flags::KEEP_ALIVE),
1225 "keep-alive flag should be set when timer is active",
1226 );
1227 debug_assert!(
1228 this.state.is_none(),
1229 "dispatcher should not be in keep-alive phase if state is not none: {:?}",
1230 this.state,
1231 );
1232
1233 if timer.as_mut().poll(cx).is_ready() {
1245 trace!("timer timed out; closing connection");
1247 this.flags.insert(Flags::SHUTDOWN);
1248
1249 if let Some(deadline) = this.config.client_disconnect_deadline() {
1250 this.shutdown_timer
1252 .set_and_init(cx, sleep_until(deadline.into()), line!());
1253 } else {
1254 this.flags.insert(Flags::WRITE_DISCONNECT);
1256 }
1257 }
1258 }
1259
1260 Ok(())
1261 }
1262
1263 fn poll_shutdown_timer(
1264 mut self: Pin<&mut Self>,
1265 cx: &mut Context<'_>,
1266 ) -> Result<(), DispatchError> {
1267 let this = self.as_mut().project();
1268 if let TimerState::Active { timer } = this.shutdown_timer {
1269 debug_assert!(
1270 this.flags.intersects(Flags::LINGER | Flags::SHUTDOWN),
1271 "shutdown or linger flag should be set when timer is active",
1272 );
1273
1274 if timer.as_mut().poll(cx).is_ready() {
1275 if this.flags.contains(Flags::LINGER) {
1276 trace!("timed-out during linger; shutting down connection");
1277 this.flags.remove(Flags::LINGER);
1278 this.flags.insert(Flags::SHUTDOWN);
1279 this.shutdown_timer.clear(line!());
1280 } else {
1281 trace!("timed-out during shutdown");
1282 return Err(DispatchError::DisconnectTimeout);
1283 }
1284 }
1285 }
1286
1287 Ok(())
1288 }
1289
1290 fn poll_graceful_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) {
1291 let this = self.as_mut().project();
1292
1293 let notified = this
1294 .graceful_shutdown
1295 .as_mut()
1296 .is_some_and(|signal| signal.as_mut().poll(cx).is_ready());
1297
1298 if notified {
1299 *this.graceful_shutdown = None;
1300 this.flags.remove(Flags::KEEP_ALIVE);
1301 this.flags.insert(Flags::DRAINING);
1302
1303 if this.ka_timer.is_enabled() {
1304 this.ka_timer.clear(line!());
1305 }
1306 }
1307 }
1308
1309 fn poll_timers(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Result<(), DispatchError> {
1311 self.as_mut().poll_head_timer(cx)?;
1312 self.as_mut().poll_ka_timer(cx)?;
1313 self.as_mut().poll_shutdown_timer(cx)?;
1314
1315 Ok(())
1316 }
1317
1318 #[inline(always)] fn read_available(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Result<bool, DispatchError> {
1325 let this = self.project();
1326
1327 if this.flags.contains(Flags::READ_DISCONNECT) {
1328 return Ok(false);
1329 };
1330
1331 let mut io = Pin::new(this.io.as_mut().unwrap());
1332
1333 let mut read_some = false;
1334
1335 loop {
1336 if this.read_buf.len() >= MAX_BUFFER_SIZE {
1338 match this.payload.as_ref().map(|p| p.need_read(cx)) {
1357 Some(PayloadStatus::Pause) => {}
1359
1360 Some(PayloadStatus::Dropped) | Some(PayloadStatus::Read) | None => {
1362 cx.waker().wake_by_ref()
1363 }
1364 }
1365
1366 return Ok(false);
1367 }
1368
1369 let remaining = this.read_buf.capacity() - this.read_buf.len();
1371 if remaining < LW_BUFFER_SIZE {
1372 this.read_buf.reserve(HW_BUFFER_SIZE - remaining);
1373 }
1374
1375 match tokio_util::io::poll_read_buf(io.as_mut(), cx, this.read_buf) {
1376 Poll::Ready(Ok(n)) => {
1377 if !this.payload.as_ref().is_some_and(|pl| pl.is_dropped()) {
1380 this.flags.remove(Flags::FINISHED);
1381 }
1382
1383 if n == 0 {
1384 return Ok(true);
1385 }
1386
1387 read_some = true;
1388 }
1389
1390 Poll::Pending => {
1391 return Ok(false);
1392 }
1393
1394 Poll::Ready(Err(err)) => {
1395 return match err.kind() {
1396 io::ErrorKind::WouldBlock => Ok(false),
1398
1399 io::ErrorKind::ConnectionReset if read_some => Ok(true),
1401
1402 _ => Err(DispatchError::Io(err)),
1403 };
1404 }
1405 }
1406 }
1407 }
1408
1409 fn upgrade(self: Pin<&mut Self>, req: Request) -> U::Future {
1411 let mut this = self.project();
1412 let mut parts = FramedParts::with_read_buf(
1413 this.io.take().unwrap(),
1414 mem::take(this.codec),
1415 mem::take(this.read_buf),
1416 );
1417
1418 this.ensure_body_buffered();
1419
1420 parts.write_buf = mem::take(this.write_buf);
1421 let framed = Framed::from_parts(parts);
1422 this.flow.upgrade.as_ref().unwrap().call((req, framed))
1423 }
1424}
1425
1426impl<T, S, B, X, U> Future for Dispatcher<T, S, B, X, U>
1427where
1428 T: AsyncRead + AsyncWrite + Unpin,
1429
1430 S: Service<Request>,
1431 S::Error: Into<Response<BoxBody>>,
1432 S::Response: Into<Response<B>>,
1433
1434 B: MessageBody,
1435
1436 X: Service<Request, Response = Request>,
1437 X::Error: Into<Response<BoxBody>>,
1438
1439 U: Service<(Request, Framed<T, Codec>), Response = ()>,
1440 U::Error: fmt::Display,
1441{
1442 type Output = Result<(), DispatchError>;
1443
1444 #[inline]
1445 fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
1446 let this = self.as_mut().project();
1447
1448 #[cfg(test)]
1449 {
1450 *this.poll_count += 1;
1451 }
1452
1453 match this.inner.project() {
1454 DispatcherStateProj::Upgrade { fut: upgrade } => upgrade.poll(cx).map_err(|err| {
1455 error!("Upgrade handler error: {}", err);
1456 DispatchError::Upgrade
1457 }),
1458
1459 DispatcherStateProj::Normal { mut inner } => {
1460 trace!("start flags: {:?}", &inner.flags);
1461
1462 trace_timer_states(
1463 "start",
1464 &inner.head_timer,
1465 &inner.ka_timer,
1466 &inner.shutdown_timer,
1467 );
1468
1469 inner.as_mut().poll_graceful_shutdown(cx);
1470 inner.as_mut().poll_timers(cx)?;
1471
1472 let poll = if inner.flags.contains(Flags::LINGER) {
1473 match inner.as_mut().poll_linger(cx)? {
1474 Poll::Ready(()) => {
1475 cx.waker().wake_by_ref();
1476 Poll::Pending
1477 }
1478 Poll::Pending => Poll::Pending,
1479 }
1480 } else if inner.flags.contains(Flags::SHUTDOWN) {
1481 if inner.flags.contains(Flags::WRITE_DISCONNECT) {
1482 Poll::Ready(Ok(()))
1483 } else {
1484 ready!(inner.as_mut().poll_flush(cx))?;
1486 Pin::new(inner.as_mut().project().io.as_mut().unwrap())
1487 .poll_shutdown(cx)
1488 .map_err(DispatchError::from)
1489 }
1490 } else {
1491 let should_disconnect = inner.as_mut().read_available(cx)?;
1493
1494 if !inner.read_buf.is_empty() && inner.flags.contains(Flags::KEEP_ALIVE) {
1496 let inner = inner.as_mut().project();
1497 inner.flags.remove(Flags::KEEP_ALIVE);
1498 inner.ka_timer.clear(line!());
1499 }
1500
1501 if !inner.flags.contains(Flags::STARTED) {
1502 inner.as_mut().project().flags.insert(Flags::STARTED);
1503
1504 if let Some(deadline) = inner.config.client_request_deadline() {
1505 inner.as_mut().project().head_timer.set_and_init(
1506 cx,
1507 sleep_until(deadline.into()),
1508 line!(),
1509 );
1510 }
1511 }
1512
1513 inner.as_mut().poll_request(cx)?;
1514
1515 if should_disconnect {
1516 let inner = inner.as_mut().project();
1518 inner.flags.insert(Flags::READ_DISCONNECT);
1519 if let Some(mut payload) = inner.payload.take() {
1520 payload.set_error(PayloadError::Incomplete(None));
1521 payload.feed_eof();
1522 }
1523 };
1524
1525 loop {
1526 let drain = match inner.as_mut().poll_response(cx)? {
1529 PollResponse::DrainWriteBuf => true,
1530
1531 PollResponse::DoNothing => {
1532 if inner.flags.contains(Flags::KEEP_ALIVE | Flags::FINISHED)
1536 && !matches!(inner.head_timer, TimerState::Active { .. })
1537 {
1538 if let Some(timer) = inner.config.keep_alive_deadline() {
1539 inner.as_mut().project().ka_timer.set_and_init(
1540 cx,
1541 sleep_until(timer.into()),
1542 line!(),
1543 );
1544 }
1545 }
1546
1547 false
1548 }
1549
1550 PollResponse::Upgrade(req) => {
1552 let upgrade = inner.upgrade(req);
1553 self.as_mut()
1554 .project()
1555 .inner
1556 .set(DispatcherState::Upgrade { fut: upgrade });
1557 return self.poll(cx);
1558 }
1559 };
1560
1561 let flush_was_ready = inner.as_mut().poll_flush(cx)?.is_ready();
1568
1569 if !flush_was_ready || !drain {
1574 break;
1575 }
1576 }
1577
1578 if inner.flags.contains(Flags::WRITE_DISCONNECT) {
1580 trace!("client is gone; disconnecting");
1581 return Poll::Ready(Ok(()));
1582 }
1583
1584 let inner_p = inner.as_mut().project();
1585 let state_is_none = inner_p.state.is_none();
1586
1587 if inner_p.flags.contains(Flags::READ_DISCONNECT)
1595 && (!inner_p.config.h1_allow_half_closed() || state_is_none)
1596 {
1597 trace!("read half closed; start shutdown");
1598 inner_p.flags.insert(Flags::SHUTDOWN);
1599 }
1600
1601 if state_is_none && inner_p.write_buf.is_empty() && inner_p.write_body.is_none()
1603 {
1604 if let Some(err) = inner_p.error.take() {
1605 error!("stream error: {}", &err);
1606 return Poll::Ready(Err(err));
1607 }
1608
1609 if inner_p.flags.contains(Flags::FINISHED)
1611 && !inner_p.flags.contains(Flags::KEEP_ALIVE)
1612 && inner_p.payload.is_none()
1613 {
1614 inner_p.flags.remove(Flags::FINISHED);
1615 inner_p.flags.insert(Flags::SHUTDOWN);
1616 return self.poll(cx);
1617 }
1618
1619 if inner_p.flags.contains(Flags::SHUTDOWN) {
1621 return self.poll(cx);
1622 }
1623 }
1624
1625 trace_timer_states(
1626 "end",
1627 inner_p.head_timer,
1628 inner_p.ka_timer,
1629 inner_p.shutdown_timer,
1630 );
1631
1632 if inner_p.flags.intersects(Flags::LINGER | Flags::SHUTDOWN) {
1633 cx.waker().wake_by_ref();
1634 }
1635 Poll::Pending
1636 };
1637
1638 trace!("end flags: {:?}", &inner.flags);
1639
1640 poll
1641 }
1642 }
1643 }
1644}
1645
1646#[derive(Debug)]
1647enum ChunkDisposition {
1648 WriteDirectly,
1650
1651 NeedsBuffering(Bytes),
1653}
1654
1655fn prepare_chunk(
1668 codec: &mut Codec,
1669 write_body: &mut Option<Bytes>,
1670 write_vectored: bool,
1671 min_len: usize,
1672 chunk: Bytes,
1673) -> ChunkDisposition {
1674 if chunk.len() < min_len || !write_vectored {
1675 return ChunkDisposition::NeedsBuffering(chunk);
1676 }
1677
1678 let chunk = match chunk.try_into_mut() {
1679 Ok(buf) if buf.capacity() > buf.len() => {
1680 return ChunkDisposition::NeedsBuffering(buf.freeze());
1681 }
1682 Ok(buf) => buf.freeze(),
1683 Err(chunk) => chunk,
1684 };
1685
1686 match codec.encoder.try_take_chunk(&chunk) {
1687 Some(body) => {
1688 *write_body = Some(body);
1689 ChunkDisposition::WriteDirectly
1690 }
1691 None => ChunkDisposition::NeedsBuffering(chunk),
1692 }
1693}
1694
1695fn should_close_for_unread_payload(
1696 payload: Option<&PayloadSender>,
1697 payload_drainable: bool,
1698) -> bool {
1699 let payload_unfinished = payload.is_some();
1700 let drain_payload = payload.is_some_and(|pl| pl.is_dropped()) && payload_drainable;
1701
1702 payload_unfinished && !drain_payload
1703}
1704
1705#[allow(dead_code)]
1706fn trace_timer_states(
1707 label: &str,
1708 head_timer: &TimerState,
1709 ka_timer: &TimerState,
1710 shutdown_timer: &TimerState,
1711) {
1712 trace!("{} timers:", label);
1713
1714 if head_timer.is_enabled() {
1715 trace!(" head {}", &head_timer);
1716 }
1717
1718 if ka_timer.is_enabled() {
1719 trace!(" keep-alive {}", &ka_timer);
1720 }
1721
1722 if shutdown_timer.is_enabled() {
1723 trace!(" shutdown {}", &shutdown_timer);
1724 }
1725}
1726
1727#[cfg(test)]
1728#[path = "dispatcher_tests.rs"]
1729mod dispatcher_tests;
1730
1731#[cfg(test)]
1732#[path = "dispatcher_buffer_tests.rs"]
1733mod buffer_tests;