use std::time::{Duration, Instant};
use bytes::Bytes;
use hyper::header::HeaderValue;
use hyper::{HeaderMap, Request};
use plecto_control::{HashInput, UpstreamGroup};
use crate::headers::{copy_headers, copy_headers_direct};
use crate::metrics::ServerMetrics;
use crate::retry::{self, AttemptOutcome};
use crate::upstream_client::{UpstreamClient, UpstreamSendError};
use crate::{ReqBody, ResponseBody};
pub(crate) enum ForwardOutcome {
Response(hyper::Response<ResponseBody>, plecto_control::Pick),
OverallTimeout,
PerTryTimeout,
SendFailed(UpstreamSendError),
BuildFailed(hyper::http::Error),
}
pub(crate) enum ForwardBody {
Bodyless,
OneShot(ReqBody),
Replayable(Bytes),
}
impl ForwardBody {
fn replayable(&self) -> bool {
match self {
ForwardBody::Bodyless | ForwardBody::Replayable(_) => true,
ForwardBody::OneShot(_) => false,
}
}
pub(crate) fn take_oneshot(&mut self) -> Option<ReqBody> {
match self {
ForwardBody::OneShot(_) => match std::mem::replace(self, ForwardBody::Bodyless) {
ForwardBody::OneShot(body) => Some(body),
ForwardBody::Bodyless | ForwardBody::Replayable(_) => None,
},
ForwardBody::Bodyless | ForwardBody::Replayable(_) => None,
}
}
fn attempt_body(&mut self) -> ReqBody {
match self {
ForwardBody::Bodyless => crate::body::empty_req(),
ForwardBody::OneShot(body) => std::mem::replace(body, crate::body::empty_req()),
ForwardBody::Replayable(bytes) => crate::body::req_full(bytes.clone()),
}
}
}
fn set_framing(dst: Option<&mut HeaderMap>, body: &ForwardBody, original: &HeaderMap) {
let Some(dst) = dst else { return };
dst.remove(hyper::header::CONTENT_LENGTH);
match body {
ForwardBody::Bodyless => {}
ForwardBody::OneShot(_) => {
if let Some(len) = original.get(hyper::header::CONTENT_LENGTH) {
dst.insert(hyper::header::CONTENT_LENGTH, len.clone());
}
}
ForwardBody::Replayable(bytes) => {
dst.insert(
hyper::header::CONTENT_LENGTH,
HeaderValue::from(bytes.len()),
);
}
}
}
pub(crate) enum AttemptHeaders<'a> {
Chain(&'a [plecto_control::Header]),
Direct(&'a HeaderMap),
}
pub(crate) struct ForwardRequest<'a> {
pub(crate) method: &'a str,
pub(crate) headers: AttemptHeaders<'a>,
pub(crate) original_headers: &'a HeaderMap,
pub(crate) authority: &'a str,
pub(crate) upstream_path: &'a str,
pub(crate) traceparent: &'a str,
pub(crate) upgrade_token: Option<&'a str>,
}
#[allow(clippy::too_many_arguments)]
pub(crate) async fn forward_with_retry<C: UpstreamClient>(
client: &C,
metrics: &ServerMetrics,
group: &UpstreamGroup,
mut pick: plecto_control::Pick,
hash_key: Option<HashInput<'_>>,
forward: ForwardRequest<'_>,
mut body: ForwardBody,
per_try_bound: Duration,
overall_deadline: Option<Instant>,
max_retries: u64,
) -> ForwardOutcome {
let replayable = body.replayable();
let mut tries_left = max_retries;
let mut attempt: u32 = 0;
loop {
let per_try = match retry::per_try_timeout(overall_deadline, per_try_bound, Instant::now())
{
retry::PerTryTimeout::OverallDeadlineElapsed => return ForwardOutcome::OverallTimeout,
retry::PerTryTimeout::Bounded(d) => d,
};
let attempt_body = body.attempt_body();
let uri = format!(
"{}://{}{}",
group.scheme(),
pick.address(),
forward.upstream_path
);
let mut builder = Request::builder().method(forward.method).uri(uri);
match forward.headers {
AttemptHeaders::Chain(headers) => copy_headers(builder.headers_mut(), headers),
AttemptHeaders::Direct(map) => copy_headers_direct(builder.headers_mut(), map),
}
if let Some(h) = builder.headers_mut()
&& !h.contains_key(hyper::header::HOST)
&& let Ok(v) = HeaderValue::from_str(forward.authority)
{
h.insert(hyper::header::HOST, v);
}
set_framing(builder.headers_mut(), &body, forward.original_headers);
if let Some(h) = builder.headers_mut()
&& let Ok(v) = HeaderValue::from_str(forward.traceparent)
{
h.insert("traceparent", v);
}
if group.scheme() == "https"
&& crate::headers::te_requests_trailers(forward.original_headers)
&& let Some(h) = builder.headers_mut()
{
h.insert(hyper::header::TE, HeaderValue::from_static("trailers"));
}
if let Some(token) = forward.upgrade_token
&& let Some(h) = builder.headers_mut()
&& let Ok(v) = HeaderValue::from_str(token)
{
h.insert(hyper::header::UPGRADE, v);
h.insert(
hyper::header::CONNECTION,
HeaderValue::from_static("upgrade"),
);
}
let upstream_req = match builder.body(attempt_body) {
Ok(req) => req,
Err(e) => return ForwardOutcome::BuildFailed(e),
};
let send = client.request(upstream_req);
let outcome = if per_try.is_zero() {
Some(send.await)
} else {
tokio::time::timeout(per_try, send).await.ok()
};
let overall_elapsed = overall_deadline.is_some_and(|d| Instant::now() >= d);
let attempt_outcome = match &outcome {
Some(Ok(resp)) => {
let retriable_5xx = retry::is_retriable_5xx(resp.status());
if group.record_outcome(pick.instance(), retriable_5xx) {
metrics.inc_outlier_ejection();
}
AttemptOutcome::Response { retriable_5xx }
}
None => {
if overall_elapsed {
return ForwardOutcome::OverallTimeout;
}
AttemptOutcome::TimedOut
}
Some(Err(e)) => {
let connect = e.is_connect();
if connect {
pick.record_passive_failure();
}
AttemptOutcome::Failed { connect }
}
};
let should_retry = retry::should_attempt_retry(
attempt_outcome,
forward.method,
replayable,
tries_left,
overall_elapsed,
);
if should_retry && let Some(next_pick) = group.pick_excluding(pick.instance(), hash_key) {
metrics.inc_retries();
retry::backoff(attempt).await;
attempt += 1;
tries_left -= 1;
pick = next_pick;
continue;
}
return match outcome {
Some(Ok(resp)) => ForwardOutcome::Response(resp, pick),
None => ForwardOutcome::PerTryTimeout,
Some(Err(e)) => ForwardOutcome::SendFailed(e),
};
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::metrics::ServerMetrics;
use crate::upstream_client::SendErrorKind;
use crate::upstream_client::fake::{FakeUpstreamClient, Scripted};
fn test_group(
addrs: &[&str],
max_retries: u64,
) -> std::sync::Arc<plecto_control::UpstreamGroup> {
let addr_list = addrs
.iter()
.map(|a| format!("\"{a}\""))
.collect::<Vec<_>>()
.join(", ");
let toml = format!(
r#"
[[upstream]]
name = "test"
addresses = [{addr_list}]
max_retries = {max_retries}
[upstream.health]
path = "/healthz"
healthy_threshold = 1
"#
);
let manifest = plecto_control::Manifest::from_toml(&toml).unwrap();
let registry = plecto_control::UpstreamRegistry::new();
registry
.reconcile(&manifest.upstreams, std::path::Path::new("."))
.unwrap();
let group = registry.group("test").unwrap();
for inst in &group.endpoints().instances {
inst.record_probe_success();
}
group
}
fn req<'a>(method: &'a str, headers: &'a HeaderMap) -> ForwardRequest<'a> {
ForwardRequest {
method,
headers: AttemptHeaders::Chain(&[]),
original_headers: headers,
authority: "test-authority",
upstream_path: "/",
traceparent: "test-trace",
upgrade_token: None,
}
}
#[tokio::test]
async fn retries_onto_another_attempt_after_a_retriable_5xx_then_succeeds() {
let group = test_group(&["127.0.0.1:1", "127.0.0.1:2"], 2);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![
Scripted::Status(503),
Scripted::Status(503),
Scripted::Status(200),
]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("GET", &headers),
ForwardBody::Bodyless,
Duration::from_secs(5),
None,
2,
)
.await;
match outcome {
ForwardOutcome::Response(resp, _) => assert_eq!(resp.status(), 200),
_ => panic!("expected the retry sequence to end in a 200 response"),
}
assert_eq!(
client.calls(),
3,
"two retriable 5xxs then a success = 3 attempts"
);
}
#[tokio::test]
async fn non_idempotent_post_is_not_retried_after_a_per_try_timeout() {
let group = test_group(&["127.0.0.1:1", "127.0.0.1:2"], 2);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![Scripted::Hang(Duration::from_millis(200))]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("POST", &headers),
ForwardBody::Bodyless,
Duration::from_millis(20),
None,
2,
)
.await;
assert!(
matches!(outcome, ForwardOutcome::PerTryTimeout),
"expected a per-try timeout outcome"
);
assert_eq!(
client.calls(),
1,
"a non-idempotent method must not be retried after a timeout"
);
}
#[tokio::test]
async fn overall_deadline_already_elapsed_fails_closed_without_a_send() {
let group = test_group(&["127.0.0.1:1"], 2);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![Scripted::Status(200)]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let elapsed_deadline = Some(Instant::now() - Duration::from_millis(1));
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("GET", &headers),
ForwardBody::Bodyless,
Duration::from_secs(5),
elapsed_deadline,
2,
)
.await;
assert!(matches!(outcome, ForwardOutcome::OverallTimeout));
assert_eq!(
client.calls(),
0,
"an already-elapsed overall deadline must not attempt any send"
);
}
#[tokio::test]
async fn connect_failure_is_retried_even_for_a_non_idempotent_method() {
let group = test_group(&["127.0.0.1:1", "127.0.0.1:2"], 1);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![
Scripted::SendError(SendErrorKind::Connect),
Scripted::Status(200),
]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("POST", &headers),
ForwardBody::Bodyless,
Duration::from_secs(5),
None,
1,
)
.await;
match outcome {
ForwardOutcome::Response(resp, _) => assert_eq!(resp.status(), 200),
_ => panic!("expected success after retrying the connect failure"),
}
assert_eq!(
client.calls(),
2,
"a connect failure (upstream never received the request) is safe to retry for any method"
);
}
#[tokio::test]
async fn replayable_body_is_resent_intact_on_a_connect_failure() {
let group = test_group(&["127.0.0.1:1", "127.0.0.1:2"], 1);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![
Scripted::SendError(SendErrorKind::Connect),
Scripted::Status(200),
]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("POST", &headers),
ForwardBody::Replayable(Bytes::from_static(b"buffered payload")),
Duration::from_secs(5),
None,
1,
)
.await;
match outcome {
ForwardOutcome::Response(resp, _) => assert_eq!(resp.status(), 200),
_ => panic!("expected the connect failure to be retried onto the other instance"),
}
assert_eq!(client.calls(), 2);
assert_eq!(
client.bodies(),
vec![Bytes::from_static(b"buffered payload"); 2],
"every attempt must re-send the buffered body intact"
);
}
#[tokio::test]
async fn replayable_idempotent_request_is_retried_on_a_retriable_5xx() {
let group = test_group(&["127.0.0.1:1", "127.0.0.1:2"], 1);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![Scripted::Status(503), Scripted::Status(200)]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("PUT", &headers),
ForwardBody::Replayable(Bytes::from_static(b"idempotent payload")),
Duration::from_secs(5),
None,
1,
)
.await;
match outcome {
ForwardOutcome::Response(resp, _) => assert_eq!(resp.status(), 200),
_ => panic!("expected the 503 to be retried onto the healthy instance"),
}
assert_eq!(client.calls(), 2);
}
#[tokio::test]
async fn replayable_non_idempotent_request_is_not_retried_on_a_timeout() {
let group = test_group(&["127.0.0.1:1", "127.0.0.1:2"], 2);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![Scripted::Hang(Duration::from_millis(200))]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("POST", &headers),
ForwardBody::Replayable(Bytes::from_static(b"post payload")),
Duration::from_millis(20),
None,
2,
)
.await;
assert!(matches!(outcome, ForwardOutcome::PerTryTimeout));
assert_eq!(
client.calls(),
1,
"a non-idempotent method must not be retried after a timeout, replayable or not"
);
}
#[tokio::test]
async fn replayable_retry_shares_the_bounded_budget() {
let group = test_group(&["127.0.0.1:1", "127.0.0.1:2"], 2);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![Scripted::Status(503)]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("PUT", &headers),
ForwardBody::Replayable(Bytes::from_static(b"exhausted payload")),
Duration::from_secs(5),
None,
2,
)
.await;
match outcome {
ForwardOutcome::Response(resp, _) => assert_eq!(
resp.status(),
503,
"once the budget is spent the last 503 surfaces"
),
_ => panic!("expected the final 503 response"),
}
assert_eq!(
client.calls(),
3,
"max_retries = 2 bounds a replayable request to 3 attempts total"
);
}
#[tokio::test]
async fn oneshot_streamed_body_is_never_retried() {
let group = test_group(&["127.0.0.1:1", "127.0.0.1:2"], 2);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![
Scripted::SendError(SendErrorKind::Connect),
Scripted::Status(200),
]);
let metrics = ServerMetrics::new();
let headers = HeaderMap::new();
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
req("POST", &headers),
ForwardBody::OneShot(crate::body::req_full(Bytes::from_static(b"streamed"))),
Duration::from_secs(5),
None,
2,
)
.await;
assert!(
matches!(outcome, ForwardOutcome::SendFailed(_)),
"a streamed body must surface the connect failure unretried"
);
assert_eq!(client.calls(), 1);
}
#[tokio::test]
async fn host_is_filled_from_authority_when_the_attempt_headers_carry_none() {
let group = test_group(&["127.0.0.1:1"], 0);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![Scripted::Status(200)]);
let metrics = ServerMetrics::new();
let original = HeaderMap::new();
let forward = ForwardRequest {
method: "GET",
headers: AttemptHeaders::Chain(&[]),
original_headers: &original,
authority: "h2-client.example",
upstream_path: "/",
traceparent: "test-trace",
upgrade_token: None,
};
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
forward,
ForwardBody::Bodyless,
Duration::from_secs(5),
None,
0,
)
.await;
assert!(matches!(outcome, ForwardOutcome::Response(..)));
let sent = client.headers();
assert_eq!(
sent[0].get(hyper::header::HOST).unwrap(),
"h2-client.example"
);
}
#[tokio::test]
async fn existing_host_header_is_not_overridden_by_authority() {
let group = test_group(&["127.0.0.1:1"], 0);
let pick = group.pick(None).unwrap();
let client = FakeUpstreamClient::new(vec![Scripted::Status(200)]);
let metrics = ServerMetrics::new();
let original = HeaderMap::new();
let chain_headers = vec![plecto_control::Header {
name: "host".to_string(),
value: b"h1-client.example".to_vec(),
}];
let forward = ForwardRequest {
method: "GET",
headers: AttemptHeaders::Chain(&chain_headers),
original_headers: &original,
authority: "should-not-be-used.example",
upstream_path: "/",
traceparent: "test-trace",
upgrade_token: None,
};
let outcome = forward_with_retry(
&client,
&metrics,
&group,
pick,
None,
forward,
ForwardBody::Bodyless,
Duration::from_secs(5),
None,
0,
)
.await;
assert!(matches!(outcome, ForwardOutcome::Response(..)));
let sent = client.headers();
assert_eq!(
sent[0].get(hyper::header::HOST).unwrap(),
"h1-client.example"
);
}
}