use super::*;
#[inline(always)]
pub(crate) fn take_h3_buffer() -> Vec<u8> {
H3_BUFFER_POOL.with(|pool: &RefCell<Vec<Vec<u8>>>| {
let mut buf: Vec<u8> = pool
.borrow_mut()
.pop()
.unwrap_or_else(|| Vec::with_capacity(H3_READ_CHUNK_SIZE));
buf.clear();
buf
})
}
#[inline(always)]
pub(crate) fn return_h3_buffer(mut buffer: Vec<u8>) {
if buffer.capacity() == 0 || buffer.capacity() > MAX_POOLED_BUFFER_CAPACITY {
return;
}
buffer.clear();
H3_BUFFER_POOL.with(|pool: &RefCell<Vec<Vec<u8>>>| {
let mut pool: RefMut<'_, Vec<Vec<u8>>> = pool.borrow_mut();
if pool.len() < MAX_POOLED_H3_BUFFERS {
pool.push(buffer);
}
});
}
#[inline(always)]
fn is_grease_id(id: u64) -> bool {
id >= 0x21 && (id - 0x21).is_multiple_of(0x1f)
}
#[inline(always)]
fn is_reserved_h2_frame(id: u64) -> bool {
let low_reserved: bool = id == 0x02 || id == 0x06;
let high_reserved: bool = id == 0x08 || id == 0x09;
low_reserved || high_reserved
}
pub(crate) async fn serve_h3_connection(
env: ConnectionEnv,
peers: StreamPeers,
connection: QuicConnection,
) {
let mut control_stream: Option<QuicSendStream> = match connection.open_uni().await {
Ok(mut send) => {
if send.write_all(H3_SERVER_CONTROL_SETTINGS).await.is_err() {
return;
}
Some(send)
}
Err(_) => return,
};
let control_state: H3ControlState = H3ControlState::default();
let mut first_req: Pin<&mut Option<_>> = pin!(None);
let mut extra_reqs: Option<FuturesUnordered<_>> = None;
let mut first_uni: Pin<&mut Option<_>> = pin!(None);
let mut extra_unis: Option<FuturesUnordered<_>> = None;
loop {
let has_extra_reqs: bool = extra_reqs
.as_ref()
.is_some_and(|queue: &FuturesUnordered<_>| !queue.is_empty());
let has_extra_unis: bool = extra_unis
.as_ref()
.is_some_and(|queue: &FuturesUnordered<_>| !queue.is_empty());
tokio::select! {
biased;
_ = async {
if let Some(fut) = first_req.as_mut().as_pin_mut() {
let _: Result<(), Box<dyn Any + Send>> = fut.await;
}
}, if first_req.is_some() => {
first_req.set(None);
}
_ = async {
if let Some(queue) = extra_reqs.as_mut() {
let _: Option<Result<(), Box<dyn Any + Send>>> = queue.next().await;
}
}, if has_extra_reqs => {}
_ = async {
if let Some(fut) = first_uni.as_mut().as_pin_mut() {
fut.await;
}
}, if first_uni.is_some() => {
first_uni.set(None);
}
_ = async {
if let Some(queue) = extra_unis.as_mut() {
let _: Option<()> = queue.next().await;
}
}, if has_extra_unis => {}
accepted = connection.accept_bi() => {
match accepted {
Ok((send, recv)) => {
let req_env: H3RequestEnv = H3RequestEnv {
shared: Arc::clone(&env.shared),
peers,
};
if first_req.is_none() {
first_req.set(Some(
AssertUnwindSafe(h3_request_task(req_env, &connection, send, recv))
.catch_unwind(),
));
} else {
extra_reqs
.get_or_insert_with(FuturesUnordered::new)
.push(
AssertUnwindSafe(h3_request_task(req_env, &connection, send, recv))
.catch_unwind(),
);
}
}
Err(_) => break,
}
}
uni = connection.accept_uni() => {
match uni {
Ok(recv) => {
if first_uni.is_none() {
first_uni.set(Some(handle_h3_uni_stream(
&connection,
recv,
&control_state,
)));
} else {
extra_unis
.get_or_insert_with(FuturesUnordered::new)
.push(handle_h3_uni_stream(&connection, recv, &control_state));
}
}
Err(_) => break,
}
}
}
}
let _: Option<QuicSendStream> = control_stream.take();
}
fn validate_h3_settings_payload(payload: &[u8]) -> Result<(), u32> {
let mut pos: usize = 0;
let mut seen: [u64; 16] = [0; 16];
let mut seen_len: usize = 0;
let mut overflow: Option<HashSet<u64>> = None;
while pos < payload.len() {
let Some((id, id_len)) = decode_varint(&payload[pos..]) else {
return Err(H3_FRAME_ERROR);
};
pos += id_len;
let Some((_, val_len)) = decode_varint(&payload[pos..]) else {
return Err(H3_FRAME_ERROR);
};
pos += val_len;
if (0x02..=0x05).contains(&id) {
return Err(H3_SETTINGS_ERROR);
}
if seen[..seen_len].contains(&id) {
return Err(H3_SETTINGS_ERROR);
}
if seen_len < seen.len() {
seen[seen_len] = id;
seen_len += 1;
} else {
let set: &mut HashSet<u64> = overflow.get_or_insert_with(HashSet::new);
if !set.insert(id) {
return Err(H3_SETTINGS_ERROR);
}
}
}
Ok(())
}
async fn drive_h3_control_stream(connection: &QuicConnection, recv: &mut QuicRecvStream) {
let mut buf: Vec<u8> = take_h3_buffer();
let mut pos: usize = 0;
let mut settings_seen: bool = false;
loop {
if pos == buf.len() {
buf.clear();
pos = 0;
}
let header: Option<(u64, usize)> =
match read_h3_frame_header(recv, &mut buf, &mut pos).await {
Ok(Some(pair)) => Some(pair),
Ok(None) => None,
Err(_) => {
return_h3_buffer(buf);
connection.close(
VarInt::from_u32(H3_FRAME_ERROR),
H3_REASON_MALFORMED_CONTROL,
);
return;
}
};
let Some((frame_type, payload_len)) = header else {
return_h3_buffer(buf);
connection.close(
VarInt::from_u32(H3_CLOSED_CRITICAL_STREAM),
H3_REASON_CONTROL_CLOSED,
);
return;
};
if !settings_seen {
if frame_type != H3_FRAME_SETTINGS {
return_h3_buffer(buf);
connection.close(
VarInt::from_u32(H3_MISSING_SETTINGS),
H3_REASON_MISSING_SETTINGS,
);
return;
}
if payload_len > H3_MAX_SETTINGS_FRAME_LEN {
return_h3_buffer(buf);
connection.close(
VarInt::from_u32(H3_FRAME_ERROR),
H3_REASON_SETTINGS_TOO_LARGE,
);
return;
}
match ensure_h3_bytes(recv, &mut buf, pos, payload_len).await {
Ok(true) => {}
Ok(false) => {
return_h3_buffer(buf);
connection.close(
VarInt::from_u32(H3_CLOSED_CRITICAL_STREAM),
H3_REASON_SETTINGS_TRUNCATED,
);
return;
}
Err(_) => {
return_h3_buffer(buf);
connection.close(VarInt::from_u32(H3_FRAME_ERROR), H3_REASON_CONTROL_READ);
return;
}
}
let end: usize = pos + payload_len;
if let Err(code) = validate_h3_settings_payload(&buf[pos..end]) {
return_h3_buffer(buf);
connection.close(VarInt::from_u32(code), H3_REASON_INVALID_SETTINGS);
return;
}
pos = end;
settings_seen = true;
continue;
}
match frame_type {
H3_FRAME_SETTINGS | H3_FRAME_DATA | H3_FRAME_HEADERS | H3_FRAME_PUSH_PROMISE => {
return_h3_buffer(buf);
connection.close(
VarInt::from_u32(H3_FRAME_UNEXPECTED),
H3_REASON_UNEXPECTED_CONTROL,
);
return;
}
other if is_reserved_h2_frame(other) => {
return_h3_buffer(buf);
connection.close(
VarInt::from_u32(H3_FRAME_UNEXPECTED),
H3_REASON_RESERVED_FRAME,
);
return;
}
_ => {
if skip_h3_payload(recv, &mut buf, &mut pos, payload_len)
.await
.is_err()
{
return_h3_buffer(buf);
connection.close(
VarInt::from_u32(H3_CLOSED_CRITICAL_STREAM),
H3_REASON_CONTROL_TRUNCATED,
);
return;
}
}
}
}
}
async fn handle_h3_uni_stream(
connection: &QuicConnection,
mut recv: QuicRecvStream,
state: &H3ControlState,
) {
let mut header_buf: [u8; 8] = [0; 8];
let mut filled: usize = 0;
let stream_type: u64 = loop {
if filled > 0
&& let Some((val, _)) = decode_varint(&header_buf[..filled])
{
break val;
}
if filled == header_buf.len() {
return;
}
match recv.read(&mut header_buf[filled..filled + 1]).await {
Ok(Some(n)) if n > 0 => filled += n,
_ => return,
}
};
match stream_type {
H3_STREAM_CONTROL => {
if state.control_seen.swap(true, AtomicOrdering::AcqRel) {
connection.close(
VarInt::from_u32(H3_STREAM_CREATION_ERROR),
H3_REASON_DUPLICATE_CONTROL,
);
return;
}
drive_h3_control_stream(connection, &mut recv).await;
}
H3_STREAM_QPACK_ENCODER => {
if state.qpack_enc_seen.swap(true, AtomicOrdering::AcqRel) {
connection.close(
VarInt::from_u32(H3_STREAM_CREATION_ERROR),
H3_REASON_DUPLICATE_ENCODER,
);
return;
}
let mut buf: [u8; 64] = [0; 64];
loop {
match recv.read(&mut buf).await {
Ok(Some(0)) | Ok(None) => {
connection.close(
VarInt::from_u32(H3_CLOSED_CRITICAL_STREAM),
H3_REASON_ENCODER_CLOSED,
);
return;
}
Ok(Some(n)) => {
if buf[..n].iter().any(|b: &u8| *b != QPACK_SET_CAPACITY_ZERO) {
connection.close(
VarInt::from_u32(H3_QPACK_ENCODER_STREAM_ERROR),
H3_REASON_QPACK_DYNAMIC_UNSUPPORTED,
);
return;
}
}
Err(_) => return,
}
}
}
H3_STREAM_QPACK_DECODER => {
if state.qpack_dec_seen.swap(true, AtomicOrdering::AcqRel) {
connection.close(
VarInt::from_u32(H3_STREAM_CREATION_ERROR),
H3_REASON_DUPLICATE_DECODER,
);
return;
}
let mut buf: [u8; 64] = [0; 64];
loop {
match recv.read(&mut buf).await {
Ok(Some(0)) | Ok(None) => {
connection.close(
VarInt::from_u32(H3_CLOSED_CRITICAL_STREAM),
H3_REASON_DECODER_CLOSED,
);
return;
}
Ok(Some(n)) => {
if buf[..n].iter().any(|b: &u8| {
let is_inc: bool = (*b & 0xc0) == 0x00;
let is_ack: bool = (*b & 0x80) == 0x80;
is_inc || is_ack
}) {
connection.close(
VarInt::from_u32(H3_QPACK_DECODER_STREAM_ERROR),
H3_REASON_QPACK_DECODER_INVALID,
);
return;
}
}
Err(_) => return,
}
}
}
H3_STREAM_PUSH => {
connection.close(
VarInt::from_u32(H3_STREAM_CREATION_ERROR),
H3_REASON_CLIENT_PUSH,
);
}
_ => {
let _: Result<(), _> = recv.stop(VarInt::from_u32(H3_NO_ERROR));
}
}
}
#[inline(always)]
async fn ensure_h3_bytes(
recv: &mut QuicRecvStream,
buf: &mut Vec<u8>,
pos: usize,
needed: usize,
) -> Result<bool, RequestError> {
while buf.len() - pos < needed {
let old_len: usize = buf.len();
let grow: usize = H3_READ_CHUNK_SIZE.max(needed - (old_len - pos));
buf.resize(old_len + grow, 0);
match recv.read(&mut buf[old_len..]).await {
Ok(Some(n)) if n > 0 => {
buf.truncate(old_len + n);
}
Ok(Some(_)) | Ok(None) => {
buf.truncate(old_len);
return Ok(false);
}
Err(error) => {
buf.truncate(old_len);
return Err(RequestError::Request(error.to_string()));
}
}
}
Ok(true)
}
async fn read_h3_frame_header(
recv: &mut QuicRecvStream,
buf: &mut Vec<u8>,
pos: &mut usize,
) -> Result<Option<(u64, usize)>, RequestError> {
let frame_type: u64 = loop {
if *pos < buf.len() {
let need: usize = 1usize << (buf[*pos] >> 6);
if buf.len() - *pos >= need
&& let Some((val, consumed)) = decode_varint(&buf[*pos..])
{
*pos += consumed;
break val;
}
}
let need_more: usize = if *pos < buf.len() {
1usize << (buf[*pos] >> 6)
} else {
1
};
if !ensure_h3_bytes(recv, buf, *pos, need_more).await? {
if *pos == buf.len() {
return Ok(None);
}
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
};
let payload_len_u64: u64 = loop {
if *pos < buf.len() {
let need: usize = 1usize << (buf[*pos] >> 6);
if buf.len() - *pos >= need
&& let Some((val, consumed)) = decode_varint(&buf[*pos..])
{
*pos += consumed;
break val;
}
}
let need_more: usize = if *pos < buf.len() {
1usize << (buf[*pos] >> 6)
} else {
1
};
if !ensure_h3_bytes(recv, buf, *pos, need_more).await? {
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
};
let payload_len: usize = usize::try_from(payload_len_u64)
.map_err(|_| RequestError::HttpRead(HttpStatus::BadRequest))?;
Ok(Some((frame_type, payload_len)))
}
async fn skip_h3_payload(
recv: &mut QuicRecvStream,
buf: &mut Vec<u8>,
pos: &mut usize,
mut remaining: usize,
) -> Result<(), RequestError> {
while remaining > 0 {
let avail: usize = buf.len() - *pos;
if avail > 0 {
let take: usize = avail.min(remaining);
*pos += take;
remaining -= take;
continue;
}
buf.clear();
*pos = 0;
if !ensure_h3_bytes(recv, buf, 0, 1).await? {
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
}
Ok(())
}
async fn read_h3_request_headers(
connection: &QuicConnection,
recv: &mut QuicRecvStream,
buf: &mut Vec<u8>,
pos: &mut usize,
config: &RequestConfig,
request: &mut Request,
target: &mut String,
) -> Result<Option<H3DecodedRequestMeta>, RequestError> {
loop {
let Some((frame_type, payload_len)) = read_h3_frame_header(recv, buf, pos).await? else {
return Ok(None);
};
if frame_type != H3_FRAME_HEADERS {
let forbidden: bool =
frame_type == H3_FRAME_DATA || H3_FORBIDDEN_REQUEST_FRAMES.contains(&frame_type);
if forbidden || is_reserved_h2_frame(frame_type) {
let reason: &[u8] = if is_reserved_h2_frame(frame_type) {
H3_REASON_RESERVED_FRAME
} else {
H3_REASON_UNEXPECTED_REQUEST_FRAME
};
connection.close(VarInt::from_u32(H3_FRAME_UNEXPECTED), reason);
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
skip_h3_payload(recv, buf, pos, payload_len).await?;
continue;
}
if payload_len > H3_MAX_HEADERS_FRAME_LEN {
return Err(RequestError::HeaderValueTooLong(
HttpStatus::RequestHeaderFieldsTooLarge,
));
}
if !ensure_h3_bytes(recv, buf, *pos, payload_len).await? {
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
let payload_end: usize = *pos + payload_len;
let meta: H3DecodedRequestMeta =
decode_qpack_request(&buf[*pos..payload_end], config, request, target)?;
*pos = payload_end;
return Ok(Some(meta));
}
}
async fn collect_h3_data_frames(
connection: &QuicConnection,
recv: &mut QuicRecvStream,
buf: &mut Vec<u8>,
pos: &mut usize,
max_body_size: usize,
body: &mut Vec<u8>,
) -> Result<(), RequestError> {
let mut trailers_seen: bool = false;
loop {
if *pos == buf.len() {
buf.clear();
*pos = 0;
}
let Some((frame_type, payload_len)) = read_h3_frame_header(recv, buf, pos).await? else {
return Ok(());
};
match frame_type {
H3_FRAME_DATA => {
if trailers_seen {
connection.close(
VarInt::from_u32(H3_FRAME_UNEXPECTED),
H3_REASON_FRAME_AFTER_TRAILERS,
);
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
if max_body_size != DEFAULT_LOW_SECURITY_MAX_BODY_SIZE
&& body.len().saturating_add(payload_len) > max_body_size
{
let _: Result<(), _> = recv.stop(VarInt::from_u32(H3_NO_ERROR));
return Err(RequestError::ContentLengthTooLarge(
HttpStatus::PayloadTooLarge,
));
}
let mut remaining: usize = payload_len;
while remaining > 0 {
let avail: usize = buf.len() - *pos;
if avail > 0 {
let take: usize = avail.min(remaining);
body.extend_from_slice(&buf[*pos..*pos + take]);
*pos += take;
remaining -= take;
continue;
}
let old_len: usize = body.len();
let chunk_len: usize = remaining.min(MAX_POOLED_BUFFER_CAPACITY);
body.resize(old_len + chunk_len, 0);
match recv.read(&mut body[old_len..]).await {
Ok(Some(n)) if n > 0 => {
body.truncate(old_len + n);
remaining -= n;
}
Ok(Some(_)) | Ok(None) => {
body.truncate(old_len);
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
Err(error) => {
body.truncate(old_len);
return Err(RequestError::Request(error.to_string()));
}
}
}
}
H3_FRAME_HEADERS => {
if trailers_seen {
connection.close(
VarInt::from_u32(H3_FRAME_UNEXPECTED),
H3_REASON_FRAME_AFTER_TRAILERS,
);
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
trailers_seen = true;
skip_h3_payload(recv, buf, pos, payload_len).await?;
}
other if is_grease_id(other) => {
skip_h3_payload(recv, buf, pos, payload_len).await?;
}
H3_FRAME_SETTINGS
| H3_FRAME_GOAWAY
| H3_FRAME_CANCEL_PUSH
| H3_FRAME_PUSH_PROMISE
| H3_FRAME_MAX_PUSH_ID => {
connection.close(
VarInt::from_u32(H3_FRAME_UNEXPECTED),
H3_REASON_UNEXPECTED_REQUEST_FRAME,
);
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
other if is_reserved_h2_frame(other) => {
connection.close(
VarInt::from_u32(H3_FRAME_UNEXPECTED),
H3_REASON_RESERVED_FRAME,
);
return Err(RequestError::HttpRead(HttpStatus::BadRequest));
}
_ => {
skip_h3_payload(recv, buf, pos, payload_len).await?;
}
}
}
}
pub(crate) async fn h3_request_task(
env: H3RequestEnv,
connection: &QuicConnection,
mut send_stream: QuicSendStream,
mut recv_stream: QuicRecvStream,
) {
let H3RequestEnv { shared, peers } = env;
let server: &Server = &shared.server;
let request_config: RequestConfig = shared.request_config;
let mut read_buf: Vec<u8> = take_h3_buffer();
let mut read_pos: usize = 0;
let mut ctx: Context = take_pooled_context();
let mut target: String = take_target_buffer();
let read_timeout_ms: u64 = request_config.get_read_timeout_ms();
let head_result: Result<Option<H3DecodedRequestMeta>, RequestError> =
if read_timeout_ms == DEFAULT_LOW_SECURITY_READ_TIMEOUT_MS {
read_h3_request_headers(
connection,
&mut recv_stream,
&mut read_buf,
&mut read_pos,
&request_config,
ctx.get_mut_request(),
&mut target,
)
.await
} else {
let mut reading: Pin<&mut _> = pin!(read_h3_request_headers(
connection,
&mut recv_stream,
&mut read_buf,
&mut read_pos,
&request_config,
ctx.get_mut_request(),
&mut target,
));
if let Some(done) = poll_once_protocol(reading.as_mut()).await {
done
} else {
match timeout(Duration::from_millis(read_timeout_ms), reading).await {
Ok(done) => done,
Err(_) => Err(RequestError::ReadTimeout(HttpStatus::RequestTimeout)),
}
}
};
let meta: H3DecodedRequestMeta = match head_result {
Ok(Some(meta)) => meta,
Ok(None) => {
return_h3_buffer(read_buf);
return_target_buffer(target);
return_pooled_context(ctx);
return;
}
Err(error) => {
let _: Result<(), _> = recv_stream.stop(VarInt::from_u32(H3_NO_ERROR));
return_h3_buffer(read_buf);
return_target_buffer(target);
h3_fail_request(
server,
request_config,
peers,
send_stream,
&mut ctx,
HttpVersion::Http3,
error,
)
.await;
return_pooled_context(ctx);
return;
}
};
let max_body_size: usize = request_config.get_max_body_size();
let body: &mut Vec<u8> = ctx.get_mut_request().get_mut_body();
body.clear();
if let Some(expected) = meta.declared_content_length
&& expected > 0
{
body.reserve(expected.min(MAX_POOLED_BUFFER_CAPACITY));
}
let body_result: Result<(), RequestError> = {
let mut reading: Pin<&mut _> = pin!(async {
if meta.expect_continue {
send_stream
.write_all(H3_CONTINUE_FRAME)
.await
.map_err(|error: QuicWriteError| RequestError::Request(error.to_string()))?;
}
collect_h3_data_frames(
connection,
&mut recv_stream,
&mut read_buf,
&mut read_pos,
max_body_size,
body,
)
.await
});
if read_timeout_ms == DEFAULT_LOW_SECURITY_READ_TIMEOUT_MS {
reading.await
} else if let Some(done) = poll_once_protocol(reading.as_mut()).await {
done
} else {
match timeout(Duration::from_millis(read_timeout_ms), reading).await {
Ok(done) => done,
Err(_) => Err(RequestError::ReadTimeout(HttpStatus::RequestTimeout)),
}
}
};
return_h3_buffer(read_buf);
let body_check: Result<(), RequestError> =
body_result.and_then(|()| match meta.declared_content_length {
None => Ok(()),
Some(declared) => match ctx.get_request().get_body().len().cmp(&declared) {
Ordering::Equal => Ok(()),
_ => Err(RequestError::Request(CONTENT_LENGTH_MISMATCH.to_owned())),
},
});
if let Err(error) = body_check {
let _: Result<(), _> = recv_stream.stop(VarInt::from_u32(H3_NO_ERROR));
return_target_buffer(target);
h3_fail_request(
server,
request_config,
peers,
send_stream,
&mut ctx,
HttpVersion::Http3,
error,
)
.await;
return_pooled_context(ctx);
return;
}
let mut stream: Stream = Stream::from_transport(
H3DirectTransport::new(send_stream, peers, meta.is_head),
request_config,
false,
);
server
.run_protocol_lifecycle_in_place(&shared.pipeline, &mut stream, &mut ctx, target)
.await;
let _: Result<(), io::Error> = stream.get_mut_stream().shutdown().await;
return_pooled_context(ctx);
}
#[cold]
pub(crate) async fn h3_fail_request(
server: &Server,
request_config: RequestConfig,
peers: StreamPeers,
send_stream: QuicSendStream,
ctx: &mut Context,
version: HttpVersion,
error: RequestError,
) {
ctx.get_mut_request().reset().set_version(version.clone());
ctx.get_mut_response().reset().set_version(version.clone());
ctx.clear_route_params();
ctx.reset_request_attributes();
let mut stream: Stream = Stream::from_transport(
H3DirectTransport::new(send_stream, peers, false),
request_config,
false,
);
reply_to_request_error(server, &mut stream, ctx, &error, Some(version)).await;
let _: Result<(), io::Error> = stream.get_mut_stream().shutdown().await;
}