use std::io;
use std::sync::Arc;
use std::time::Duration;
use bytes::Bytes;
use foundation_core::url::Uri;
use foundation_core::valtron::{Pipe, PipeReceiver, PipeSender, Stream, TryRecvError, TrySendError};
use foundation_netio::http::ClientRequestBuilder;
use foundation_netio::http2::frame::ErrorCode;
use foundation_netio::http2::types::{H2Frame, H2IncomingFrame, SimpleIncomingRequestHeader};
use foundation_netio::shared::client::body_reader::{
SendSafeBodyBytesItem, SendSafeBodyBytesIterator,
};
use foundation_netio::shared::client::SystemDnsResolver;
use foundation_netio::shared::http::{
pushable_request_body, Proto, RequestDescriptor, SendSafeBody, SimpleHeader, SimpleHeaders,
SimpleIncomingRequest, SimpleMethod, SimpleUrl,
};
use foundation_http::native::serve::{BoxFuture, H2Serve};
use foundation_http::shared::context::ContextBag;
use foundation_connectrpc::shared::protocol::connect::{
streaming_content_type as connect_streaming_content_type,
};
use foundation_connectrpc::shared::protocol::grpc::content_type as grpc_content_type;
use foundation_connectrpc::shared::transport::{BodyStream, HeadStream, SendBody, DEFAULT_PIPE_DEPTH};
use foundation_connectrpc::{
parse_connect_content_type, CancelSignal, ConnectHandler, H2Transport, Transport,
};
use futures_util::StreamExt;
use crate::config::BackendProtocol;
use crate::forward::{forward_request_headers, strip_hop_by_hop, upstream_url, SharedHttpClient};
use crate::rpc_relay::{
build_streaming_relay_to_grpc, connect_unary_roundtrip, translate_connect_deadline_to_grpc,
};
use crate::runtime::BackendLease;
use crate::state::ProxyState;
const POLL_BACKOFF: Duration = Duration::from_millis(1);
pub struct H2ProxyHandler {
state: Arc<ProxyState>,
}
impl H2ProxyHandler {
#[must_use]
pub fn new(state: Arc<ProxyState>) -> Self {
Self { state }
}
}
impl H2Serve for H2ProxyHandler {
fn serve_h2(
&self,
_bag: Arc<ContextBag>,
header: SimpleIncomingRequestHeader,
body: PipeReceiver<H2IncomingFrame>,
tx: PipeSender<H2Frame>,
) -> BoxFuture<'static, io::Result<()>> {
let host = header.authority.clone();
let path = header.url.url.clone();
let method = header.method.clone();
let orig_headers = header.headers.clone();
let client_ip = header
.connection
.peer_addr
.as_ref()
.map(|a| a.ip().to_string())
.unwrap_or_else(|| "unknown".to_string());
let scheme = self.state.scheme().to_string();
let client = self.state.client().clone();
let state = self.state.clone();
Box::pin(async move {
let Some(service) = state.router().route(&host, &path) else {
let _ = tx.send(h2_status(404, true)).await;
return Ok(());
};
let Some((lease, _idx)) = service.pick_sticky(None) else {
let _ = tx.send(h2_status(503, true)).await;
return Ok(());
};
if is_grpc_request(&orig_headers) {
forward_upstream_h2(
lease, method, path, orig_headers, client_ip, scheme, body, &tx,
)
.await;
} else if lease.backend().protocol() == BackendProtocol::Grpc
&& is_connect_streaming_request(&orig_headers)
{
relay_connect_streaming_h2(
lease, header, orig_headers, client_ip, scheme, body, &tx,
)
.await;
} else if lease.backend().protocol() == BackendProtocol::Grpc
&& is_connect_unary_request(&orig_headers)
{
relay_connect_unary_h2(
lease, header, orig_headers, client_ip, scheme, body, &tx,
)
.await;
} else {
std::thread::spawn(move || {
forward_upstream(
lease, method, host, path, orig_headers, client_ip, scheme, client, body,
&tx,
);
});
}
Ok(())
})
}
}
#[allow(clippy::too_many_arguments)]
fn forward_upstream(
lease: BackendLease,
method: SimpleMethod,
host: String,
path: String,
orig_headers: foundation_netio::shared::http::SimpleHeaders,
client_ip: String,
scheme: String,
client: SharedHttpClient,
body: PipeReceiver<H2IncomingFrame>,
tx: &PipeSender<H2Frame>,
) {
let url = upstream_url(lease.backend().url(), &path);
let mut headers = forward_request_headers(&orig_headers, &client_ip, &scheme);
headers.insert(SimpleHeader::HOST, vec![host]);
let (pushable, body_stream) = pushable_request_body();
let feeder = std::thread::spawn(move || {
loop {
match body.try_recv() {
Ok(H2IncomingFrame::Data(bytes)) => {
let mut chunk = bytes;
loop {
match pushable.try_push(chunk) {
Ok(()) => break,
Err(TrySendError::Full(returned)) => {
chunk = returned;
std::thread::sleep(POLL_BACKOFF);
}
Err(TrySendError::Closed(_)) => return,
}
}
}
Ok(H2IncomingFrame::Reset(_)) => break,
Err(TryRecvError::Empty) => std::thread::sleep(POLL_BACKOFF),
Err(TryRecvError::Closed) => break,
}
}
pushable.close();
});
let builder = match ClientRequestBuilder::<SystemDnsResolver>::new(method, &url) {
Ok(b) => b.headers(headers).body(body_stream),
Err(e) => {
tracing::warn!(%url, "H2 build upstream request failed: {e}");
let _ = blocking_send(tx, h2_status(502, true));
let _ = feeder.join();
return;
}
};
let response = match client.request(builder).and_then(|req| req.send()) {
Ok(resp) => resp,
Err(e) => {
tracing::warn!(%url, "H2 upstream request failed: {e}");
let _ = blocking_send(tx, h2_status(502, true));
let _ = feeder.join();
return;
}
};
let (status, resp_headers, resp_body, pool, conn) = response.into_parts();
let out_headers = strip_hop_by_hop(&resp_headers);
let h2_hdrs = simple_headers_to_h2(&out_headers);
let code = u16::try_from(Into::<usize>::into(status)).unwrap_or(502);
if blocking_send(
tx,
H2Frame::Headers {
status: code,
headers: h2_hdrs,
end_stream: false,
},
)
.is_err()
{
let _ = feeder.join();
return;
}
let mut chunks = SendSafeBodyBytesIterator::new(resp_body);
loop {
match chunks.next() {
Some(Stream::Next(SendSafeBodyBytesItem::Chunk(bytes))) => {
if bytes.is_empty() {
continue;
}
if blocking_send(
tx,
H2Frame::Data {
payload: bytes,
end_stream: false,
},
)
.is_err()
{
let _ = feeder.join();
return;
}
}
Some(Stream::Next(SendSafeBodyBytesItem::StreamError(e))) => {
tracing::warn!(%url, "H2 upstream body stream error: {e}");
break;
}
Some(Stream::Delayed(d)) => std::thread::sleep(d),
Some(_) => std::thread::sleep(POLL_BACKOFF),
None => break,
}
}
let _ = blocking_send(
tx,
H2Frame::Data {
payload: Bytes::new(),
end_stream: true,
},
);
if let (Some(pool), Some(mut stream)) = (pool, conn) {
stream.drain_stream();
pool.return_to_pool(stream);
}
let _ = feeder.join();
drop(lease);
}
fn is_grpc_request(headers: &SimpleHeaders) -> bool {
headers
.get(&SimpleHeader::CONTENT_TYPE)
.and_then(|values| values.first())
.is_some_and(|ct| ct == "application/grpc" || ct.starts_with("application/grpc+"))
}
fn is_connect_streaming_request(headers: &SimpleHeaders) -> bool {
headers
.get(&SimpleHeader::CONTENT_TYPE)
.and_then(|values| values.first())
.and_then(|ct| parse_connect_content_type(ct))
.is_some_and(|(_, is_streaming)| is_streaming)
}
fn is_connect_unary_request(headers: &SimpleHeaders) -> bool {
headers
.get(&SimpleHeader::CONTENT_TYPE)
.and_then(|values| values.first())
.and_then(|ct| parse_connect_content_type(ct))
.is_some_and(|(_, is_streaming)| !is_streaming)
}
#[allow(clippy::too_many_arguments)]
async fn forward_upstream_h2(
lease: BackendLease,
method: SimpleMethod,
path: String,
orig_headers: SimpleHeaders,
client_ip: String,
scheme: String,
body: PipeReceiver<H2IncomingFrame>,
tx: &PipeSender<H2Frame>,
) {
let url = upstream_url(lease.backend().url(), &path);
let headers = forward_request_headers(&orig_headers, &client_ip, &scheme);
let uri = match Uri::parse(&url) {
Ok(u) => u,
Err(e) => {
tracing::warn!(%url, "H2 upstream URI parse failed: {e}");
let _ = tx.send(h2_status(502, true)).await;
return;
}
};
let request = RequestDescriptor {
proto: Proto::HTTP20,
request_url: SimpleUrl::url_only(&url),
request_uri: uri,
headers,
method,
};
let stream = match H2Transport::new().open(request) {
Ok(s) => s,
Err(e) => {
tracing::warn!(%url, "H2 upstream open failed: {e}");
let _ = tx.send(h2_status(502, true)).await;
return;
}
};
let send_body = stream.send_body;
let head = stream.head;
let recv_body = stream.recv_body;
let trailers = stream.trailers;
futures_util::future::join(
request_relay(body, send_body),
response_relay(tx, head, recv_body, trailers, &url),
)
.await;
drop(lease);
}
async fn request_relay(body: PipeReceiver<H2IncomingFrame>, send_body: Arc<dyn SendBody>) {
while let Some(frame) = body.receive().await {
match frame {
H2IncomingFrame::Data(bytes) => {
if bytes.is_empty() {
continue;
}
if send_body.send_async(bytes).await.is_err() {
return;
}
}
H2IncomingFrame::Reset(_) => break,
}
}
send_body.close();
}
async fn response_relay(
tx: &PipeSender<H2Frame>,
mut head: HeadStream,
mut recv_body: BodyStream,
trailers: PipeReceiver<SimpleHeaders>,
url: &str,
) {
let (status, resp_headers) = loop {
match head.next().await {
Some(Ok((status, headers))) => {
if u16::try_from(Into::<usize>::into(status.clone())).unwrap_or(0) < 200 {
continue;
}
break (status, headers);
}
Some(Err(e)) => {
tracing::warn!(%url, "H2 upstream head error: {e}");
let _ = tx.send(h2_status(502, true)).await;
return;
}
None => {
let _ = tx.send(h2_status(502, true)).await;
return;
}
}
};
let code = u16::try_from(Into::<usize>::into(status)).unwrap_or(502);
if tx
.send(H2Frame::Headers {
status: code,
headers: simple_headers_to_h2(&strip_hop_by_hop(&resp_headers)),
end_stream: false,
})
.await
.is_err()
{
return;
}
while let Some(chunk) = recv_body.next().await {
match chunk {
Ok(bytes) => {
if bytes.is_empty() {
continue;
}
if tx
.send(H2Frame::Data {
payload: bytes,
end_stream: false,
})
.await
.is_err()
{
return;
}
}
Err(e) => {
tracing::warn!(%url, "H2 upstream body error: {e}");
let _ = tx
.send(H2Frame::Reset {
error_code: ErrorCode::InternalError,
})
.await;
return;
}
}
}
match trailers.receive().await {
Some(t) if !t.is_empty() => {
let _ = tx
.send(H2Frame::Trailers {
headers: simple_headers_to_h2(&t),
})
.await;
}
_ => {
let _ = tx
.send(H2Frame::Data {
payload: Bytes::new(),
end_stream: true,
})
.await;
}
}
}
#[allow(clippy::too_many_arguments)]
async fn relay_connect_streaming_h2(
lease: BackendLease,
header: SimpleIncomingRequestHeader,
orig_headers: SimpleHeaders,
client_ip: String,
scheme: String,
body: PipeReceiver<H2IncomingFrame>,
tx: &PipeSender<H2Frame>,
) {
let url = upstream_url(lease.backend().url(), &header.url.url);
let inbound_ct = orig_headers
.get(&SimpleHeader::CONTENT_TYPE)
.and_then(|v| v.first())
.cloned()
.unwrap_or_else(|| "application/connect+json".to_string());
let Some((codec, is_streaming)) = parse_connect_content_type(&inbound_ct) else {
tracing::warn!(%url, content_type = %inbound_ct, "unrecognized connect content-type");
let _ = tx.send(h2_status(415, true)).await;
return;
};
if !is_streaming {
tracing::warn!(%url, content_type = %inbound_ct, "connect unary misrouted to streaming relay");
let _ = tx.send(h2_status(415, true)).await;
return;
}
let mut out_headers = forward_request_headers(&orig_headers, &client_ip, &scheme);
out_headers.insert(SimpleHeader::CONTENT_TYPE, vec![grpc_content_type(&codec)]);
out_headers.insert(
SimpleHeader::from("te".to_string()),
vec!["trailers".to_string()],
);
translate_connect_deadline_to_grpc(&mut out_headers);
let request = match SimpleIncomingRequest::builder()
.with_proto(Proto::HTTP20)
.with_uri(header.uri)
.with_url(header.url)
.with_headers(orig_headers)
.with_method(header.method)
.with_connection(header.connection)
.build()
{
Ok(r) => r,
Err(e) => {
tracing::warn!(%url, "build connect request failed: {e}");
let _ = tx.send(h2_status(400, true)).await;
return;
}
};
let (body_tx, body_rx) = Pipe::<Bytes>::with_depth(DEFAULT_PIPE_DEPTH);
let (responder_tx, responder_rx) = Pipe::<Bytes>::with_depth(DEFAULT_PIPE_DEPTH);
let (driver, cancel) = match build_streaming_relay_to_grpc(
ConnectHandler,
request,
body_rx,
responder_tx,
url.clone(),
out_headers,
) {
Ok(relay) => relay,
Err(e) => {
tracing::warn!(%url, "connect→grpc h2 relay setup failed: {e}");
let _ = tx.send(h2_status(502, true)).await;
return;
}
};
let mut resp_headers = SimpleHeaders::new();
resp_headers.insert(
SimpleHeader::CONTENT_TYPE,
vec![connect_streaming_content_type(&codec)],
);
if tx
.send(H2Frame::Headers {
status: 200,
headers: simple_headers_to_h2(&resp_headers),
end_stream: false,
})
.await
.is_err()
{
return;
}
let relay = futures_util::future::join3(
feed_request_body(body, body_tx, cancel.clone()),
driver,
drain_response_body(responder_rx, tx),
);
let client_gone = {
let cancel = cancel.clone();
async move {
tx.closed().await;
cancel.cancel();
}
};
let relay = Box::pin(relay);
let client_gone = Box::pin(client_gone);
match futures_util::future::select(relay, client_gone).await {
futures_util::future::Either::Left(_) => {}
futures_util::future::Either::Right(_) => {}
}
drop(lease);
}
#[allow(clippy::too_many_arguments)]
async fn relay_connect_unary_h2(
lease: BackendLease,
header: SimpleIncomingRequestHeader,
orig_headers: SimpleHeaders,
client_ip: String,
scheme: String,
body: PipeReceiver<H2IncomingFrame>,
tx: &PipeSender<H2Frame>,
) {
let url = upstream_url(lease.backend().url(), &header.url.url);
let inbound_ct = orig_headers
.get(&SimpleHeader::CONTENT_TYPE)
.and_then(|v| v.first())
.cloned()
.unwrap_or_else(|| "application/json".to_string());
let Some((codec, _)) = parse_connect_content_type(&inbound_ct) else {
tracing::warn!(%url, content_type = %inbound_ct, "unrecognized connect content-type");
let _ = tx.send(h2_status(415, true)).await;
return;
};
let mut req_body = Vec::new();
while let Some(frame) = body.receive().await {
match frame {
H2IncomingFrame::Data(bytes) => req_body.extend_from_slice(&bytes),
H2IncomingFrame::Reset(_) => break,
}
}
let req_body = Bytes::from(req_body);
let mut out_headers = forward_request_headers(&orig_headers, &client_ip, &scheme);
out_headers.insert(SimpleHeader::CONTENT_TYPE, vec![grpc_content_type(&codec)]);
out_headers.insert(
SimpleHeader::from("te".to_string()),
vec!["trailers".to_string()],
);
translate_connect_deadline_to_grpc(&mut out_headers);
let request = match SimpleIncomingRequest::builder()
.with_proto(Proto::HTTP20)
.with_uri(header.uri)
.with_url(header.url)
.with_headers(orig_headers)
.with_method(header.method)
.with_connection(header.connection)
.build()
{
Ok(r) => r,
Err(e) => {
tracing::warn!(%url, "build connect request failed: {e}");
let _ = tx.send(h2_status(400, true)).await;
return;
}
};
let response =
match connect_unary_roundtrip(request, req_body, codec, url.clone(), out_headers).await {
Ok(r) => r,
Err(e) => {
tracing::warn!(%url, "connect unary→grpc h2 relay failed: {e}");
let _ = tx.send(h2_status(502, true)).await;
return;
}
};
let code = u16::try_from(Into::<usize>::into(response.status)).unwrap_or(500);
if tx
.send(H2Frame::Headers {
status: code,
headers: simple_headers_to_h2(&response.headers),
end_stream: false,
})
.await
.is_err()
{
return;
}
let body_bytes = match response.body {
Some(SendSafeBody::Bytes(b)) => Bytes::from(b),
_ => Bytes::new(),
};
let _ = tx
.send(H2Frame::Data {
payload: body_bytes,
end_stream: true,
})
.await;
drop(lease);
}
async fn feed_request_body(
body: PipeReceiver<H2IncomingFrame>,
body_tx: PipeSender<Bytes>,
cancel: CancelSignal,
) {
while let Some(frame) = body.receive().await {
match frame {
H2IncomingFrame::Data(bytes) => {
if bytes.is_empty() {
continue;
}
if body_tx.send(bytes).await.is_err() {
break;
}
}
H2IncomingFrame::Reset(_) => {
cancel.cancel();
break;
}
}
}
body_tx.close();
}
async fn drain_response_body(responder_rx: PipeReceiver<Bytes>, tx: &PipeSender<H2Frame>) {
while let Some(bytes) = responder_rx.receive().await {
if bytes.is_empty() {
continue;
}
if tx
.send(H2Frame::Data {
payload: bytes,
end_stream: false,
})
.await
.is_err()
{
return;
}
}
let _ = tx
.send(H2Frame::Data {
payload: Bytes::new(),
end_stream: true,
})
.await;
}
fn simple_headers_to_h2(headers: &SimpleHeaders) -> Vec<(Bytes, Bytes)> {
let mut out: Vec<(Bytes, Bytes)> = Vec::new();
for (name, values) in headers.iter() {
let nb = Bytes::from(name.to_string().into_bytes());
for value in values {
out.push((nb.clone(), Bytes::from(value.clone().into_bytes())));
}
}
out
}
fn blocking_send(tx: &PipeSender<H2Frame>, frame: H2Frame) -> Result<(), ()> {
let mut item = frame;
loop {
match tx.try_send(item) {
Ok(()) => return Ok(()),
Err(TrySendError::Full(returned)) => {
item = returned;
std::thread::sleep(POLL_BACKOFF);
}
Err(TrySendError::Closed(_)) => return Err(()),
}
}
}
pub fn h2_status(code: u16, end_stream: bool) -> H2Frame {
H2Frame::Headers {
status: code,
headers: vec![],
end_stream,
}
}