use std::collections::HashMap;
use memchr::{memchr, memmem};
use tokio::io::AsyncWrite;
use crate::util::write_all;
const CHUNK_SIZE_LINE_MAX_LEN: usize = 10 + 64;
const TRAILER_HEADER_LEN: usize = 4096;
pub struct ChunkTransfer {
state: ChunkTransferState,
read_size_buf: Vec<u8>,
trailer_header_buf: [u8; TRAILER_HEADER_LEN],
trailer_headers: HashMap<String, String>,
}
#[derive(Debug, PartialEq, Eq)]
enum ChunkTransferState {
ReadSize {
cached_len: usize,
},
ReadData {
chunk_len: usize,
remaining_len: usize,
},
ReadTrailer {
cached_len: usize,
},
Done,
}
impl ChunkTransfer {
pub fn new() -> Self {
Self {
state: ChunkTransferState::ReadSize { cached_len: 0 },
read_size_buf: Vec::with_capacity(CHUNK_SIZE_LINE_MAX_LEN),
trailer_header_buf: [0u8; TRAILER_HEADER_LEN],
trailer_headers: HashMap::new(),
}
}
pub async fn run<T>(
&mut self,
data: &[u8],
maybe_forward_stream: &mut Option<&mut T>,
) -> std::io::Result<()>
where
T: AsyncWrite + Unpin + ?Sized,
{
use ChunkTransferState::*;
let mut start_offset = 0usize;
while start_offset < data.len() {
let unused = &data[start_offset..data.len()];
match self.state {
ReadSize { cached_len } => {
let space_available = CHUNK_SIZE_LINE_MAX_LEN.saturating_sub(cached_len);
let copy_len = std::cmp::min(space_available, unused.len());
self.read_size_buf.extend_from_slice(&unused[0..copy_len]);
let new_cached_len = self.read_size_buf.len();
match memmem::find(&self.read_size_buf, b"\r\n") {
Some(crlf_index) => {
let line_slice = &self.read_size_buf[0..crlf_index];
if line_slice.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData, "empty chunk size line",
));
}
let hex_end_index =
memchr(b';', line_slice).unwrap_or(line_slice.len());
let hex_part_slice = &line_slice[0..hex_end_index];
if hex_part_slice.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"chunk size hex part is empty",
));
}
let hex_str = match std::str::from_utf8(hex_part_slice) {
Ok(s) => s,
Err(e) => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("invalid utf8 in chunk size hex part: {}", e),
));
}
};
let trimmed_hex_str = hex_str.trim();
if trimmed_hex_str.is_empty() {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"chunk size hex part is empty after trim",
));
}
let chunk_len = match usize::from_str_radix(trimmed_hex_str, 16) {
Ok(len) => len,
Err(e) => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("invalid hex size ('{}'): {}", trimmed_hex_str, e),
));
}
};
let size_header_len = crlf_index + 2;
if let Some(ref mut forward_stream) = maybe_forward_stream {
write_all(forward_stream, &self.read_size_buf[0..size_header_len])
.await?;
}
let used_from_unused = size_header_len - cached_len;
start_offset += used_from_unused;
self.read_size_buf.clear();
if chunk_len > 0 {
self.state = ChunkTransferState::ReadData {
chunk_len,
remaining_len: chunk_len + 2, };
} else {
self.state = ChunkTransferState::ReadTrailer { cached_len: 0 };
}
}
None => {
if self.read_size_buf.len() >= CHUNK_SIZE_LINE_MAX_LEN {
return Err(std::io::Error::other(format!(
"chunk size line exceeded max length ({}) without CRLF",
CHUNK_SIZE_LINE_MAX_LEN
)));
}
self.state = ChunkTransferState::ReadSize {
cached_len: new_cached_len,
};
start_offset += copy_len;
assert!(start_offset == data.len());
}
}
}
ReadData {
chunk_len,
remaining_len,
} => {
let forward_len = std::cmp::min(unused.len(), remaining_len);
if let Some(ref mut forward_stream) = maybe_forward_stream {
write_all(forward_stream, &unused[0..forward_len]).await?;
}
let new_remaining_len = remaining_len - forward_len;
start_offset += forward_len;
if new_remaining_len == 0 {
self.state = ChunkTransferState::ReadSize { cached_len: 0 };
} else {
self.state = ChunkTransferState::ReadData {
chunk_len,
remaining_len: new_remaining_len,
}
}
}
ReadTrailer { cached_len } => {
let copy_len = std::cmp::min(TRAILER_HEADER_LEN - cached_len, unused.len());
let new_cached_len = cached_len + copy_len;
self.trailer_header_buf[cached_len..new_cached_len]
.copy_from_slice(&unused[0..copy_len]);
match memmem::find(&self.trailer_header_buf[0..new_cached_len], b"\r\n") {
Some(i) => {
let trailer_line_len = i + 2;
if let Some(ref mut forward_stream) = maybe_forward_stream {
write_all(
forward_stream,
&self.trailer_header_buf[0..trailer_line_len],
)
.await?;
}
let used_len = trailer_line_len - cached_len;
start_offset += used_len;
if i > 0 {
let trailer_header_str =
match std::str::from_utf8(&self.trailer_header_buf[0..i]) {
Ok(s) => s,
Err(e) => {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData, format!(
"failed to parse trailer header utf8: {}",
e
),
));
}
};
let tokens: Vec<&str> = trailer_header_str.splitn(2, ':').collect();
if tokens.len() != 2 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData, format!(
"invalid trailer header format: {}",
trailer_header_str
),
));
}
let header_key = tokens[0].trim().to_lowercase();
let header_value = tokens[1].trim().to_string();
self.trailer_headers.insert(header_key, header_value);
self.state = ChunkTransferState::ReadTrailer { cached_len: 0 };
} else {
self.state = ChunkTransferState::Done;
}
}
None => {
if new_cached_len == self.trailer_header_buf.len() {
return Err(std::io::Error::other(
"could not read trailer header, line exceeded max length",
));
}
self.state = ChunkTransferState::ReadTrailer {
cached_len: new_cached_len,
};
start_offset += copy_len;
assert!(start_offset == data.len());
}
}
}
Done => {
return Err(std::io::Error::other(
"extra data received after chunked transfer completion",
));
}
}
}
Ok(())
}
pub fn is_done(&self) -> bool {
self.state == ChunkTransferState::Done
}
pub fn trailer_headers(&mut self) -> &mut HashMap<String, String> {
&mut self.trailer_headers
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Duration;
use tokio::io::{AsyncWriteExt, BufWriter};
use tokio::time::timeout;
struct MockWriter {
buf: BufWriter<Vec<u8>>,
}
impl MockWriter {
fn new() -> Self {
Self {
buf: BufWriter::new(Vec::new()),
}
}
async fn get_written_data(mut self) -> Vec<u8> {
self.buf.flush().await.expect("Failed to flush mock writer");
self.buf.into_inner()
}
}
impl AsyncWrite for MockWriter {
fn poll_write(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
std::pin::Pin::new(&mut self.buf).poll_write(cx, buf)
}
fn poll_flush(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::pin::Pin::new(&mut self.buf).poll_flush(cx)
}
fn poll_shutdown(
mut self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::pin::Pin::new(&mut self.buf).poll_shutdown(cx)
}
}
async fn run_transfer_fragmented(
input_fragments: &[&[u8]],
forward: bool,
) -> (std::io::Result<()>, ChunkTransfer, Option<Vec<u8>>) {
let mut transfer = ChunkTransfer::new();
let mut writer = if forward {
Some(MockWriter::new())
} else {
None
};
let mut final_result = Ok(());
for (i, fragment) in input_fragments.iter().enumerate() {
if transfer.is_done() && !fragment.is_empty() {
final_result = Err(std::io::Error::new(
std::io::ErrorKind::Other,
format!("Extra data fragment {} received after completion", i),
));
break;
}
if fragment.is_empty() {
continue;
}
let mut opt_writer_ref = writer.as_mut();
match transfer.run(fragment, &mut opt_writer_ref).await {
Ok(_) => {}
Err(e) => {
final_result = Err(e);
break; }
}
}
if final_result.is_ok() && !transfer.is_done() {
final_result = Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof, "Transfer did not reach Done state after all fragments",
));
}
let written_data = if let Some(w) = writer {
Some(w.get_written_data().await)
} else {
None
};
(final_result, transfer, written_data)
}
#[tokio::test]
async fn test_single_chunk_no_fragmentation() {
let input = b"A\r\n0123456789\r\n0\r\n\r\n";
let expected_forward = input; let (result, mut transfer, written_data) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
assert!(transfer.trailer_headers().is_empty());
}
#[tokio::test]
async fn test_multiple_chunks_no_fragmentation() {
let input = b"3\r\nABC\r\n5\r\n12345\r\n0\r\n\r\n";
let expected_forward = input;
let (result, mut transfer, written_data) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
assert!(transfer.trailer_headers().is_empty());
}
#[tokio::test]
async fn test_zero_length_chunk_termination() {
let input = b"0\r\n\r\n";
let expected_forward = input;
let (result, mut transfer, written_data) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
assert!(transfer.trailer_headers().is_empty());
}
#[tokio::test]
async fn test_fragmented_size_line() {
let fragments = [b"A\r".as_slice(), b"\n0123456789\r\n0\r\n\r\n".as_slice()];
let expected_forward = b"A\r\n0123456789\r\n0\r\n\r\n";
let (result, transfer, written_data) = run_transfer_fragmented(&fragments, true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
}
#[tokio::test]
async fn test_fragmented_size_line_mid_hex() {
let expected_len = 0xAB; let chunk_data = vec![b'X'; expected_len];
let mut full_input = Vec::new();
full_input.extend_from_slice(b"AB\r\n");
full_input.extend_from_slice(&chunk_data);
full_input.extend_from_slice(b"\r\n0\r\n\r\n");
let fragments_dynamic = [
b"A".as_slice(),
b"B\r\n".as_slice(),
&chunk_data[0..10],
&chunk_data[10..],
b"\r\n0\r\n\r\n".as_slice(),
];
let (result, transfer, written_data) =
run_transfer_fragmented(&fragments_dynamic, true).await;
assert!(result.is_ok(), "Result was {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), full_input);
}
#[tokio::test]
async fn test_fragmented_chunk_data() {
let fragments = [
b"A\r\n0123".as_slice(),
b"456789\r\n".as_slice(),
b"0\r\n\r\n".as_slice(),
];
let expected_forward = b"A\r\n0123456789\r\n0\r\n\r\n";
let (result, transfer, written_data) = run_transfer_fragmented(&fragments, true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
}
#[tokio::test]
async fn test_fragmented_chunk_crlf() {
let fragments = [b"A\r\n0123456789\r".as_slice(), b"\n0\r\n\r\n".as_slice()];
let expected_forward = b"A\r\n0123456789\r\n0\r\n\r\n";
let (result, transfer, written_data) = run_transfer_fragmented(&fragments, true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
}
#[tokio::test]
async fn test_fragmented_zero_chunk_line() {
let fragments = [b"A\r\n0123456789\r\n0\r".as_slice(), b"\n\r\n".as_slice()];
let expected_forward = b"A\r\n0123456789\r\n0\r\n\r\n";
let (result, transfer, written_data) = run_transfer_fragmented(&fragments, true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
}
#[tokio::test]
async fn test_fragmented_final_crlf() {
let fragments = [b"0\r\n\r".as_slice(), b"\n".as_slice()];
let expected_forward = b"0\r\n\r\n";
let (result, transfer, written_data) = run_transfer_fragmented(&fragments, true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
}
#[tokio::test]
async fn test_with_trailers_no_fragmentation() {
let input = b"3\r\nABC\r\n0\r\nTrailer-Key: Trailer Value\r\nAnother: Yes\r\n\r\n";
let (result, mut transfer, written_data) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), input);
assert_eq!(transfer.trailer_headers().len(), 2);
assert_eq!(
transfer.trailer_headers().get("trailer-key"),
Some(&"Trailer Value".to_string())
);
assert_eq!(
transfer.trailer_headers().get("another"),
Some(&"Yes".to_string())
);
}
#[tokio::test]
async fn test_with_trailers_fragmented() {
let fragments = [
b"3\r\nABC\r\n0\r\nTrai".as_slice(),
b"ler-Key: Trailer Value\r\n".as_slice(),
b"Another: Yes\r\n\r".as_slice(),
b"\n".as_slice(),
];
let expected_forward =
b"3\r\nABC\r\n0\r\nTrailer-Key: Trailer Value\r\nAnother: Yes\r\n\r\n";
let (result, mut transfer, written_data) = run_transfer_fragmented(&fragments, true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
assert_eq!(transfer.trailer_headers().len(), 2);
assert_eq!(
transfer.trailer_headers().get("trailer-key"),
Some(&"Trailer Value".to_string())
);
assert_eq!(
transfer.trailer_headers().get("another"),
Some(&"Yes".to_string())
);
}
#[tokio::test]
async fn test_no_forwarding() {
let input = b"A\r\n0123456789\r\n0\r\n\r\n";
let (result, mut transfer, written_data) = run_transfer_fragmented(&[input], false).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert!(written_data.is_none()); assert!(transfer.trailer_headers().is_empty());
}
#[tokio::test]
async fn test_error_invalid_hex_size() {
let input = b"G\r\nData\r\n";
let (result, transfer, _) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("invalid hex size ('G')"));
assert!(!transfer.is_done()); }
#[tokio::test]
async fn test_error_empty_size_line_content() {
let input = b"\r\nData\r\n";
let (result, transfer, _) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("empty chunk size line"));
assert!(!transfer.is_done());
}
#[tokio::test]
async fn test_error_size_line_only_extension() {
let input = b";extension\r\nData\r\n";
let (result, transfer, _) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("chunk size hex part is empty"));
assert!(!transfer.is_done());
}
#[tokio::test]
async fn test_error_size_line_too_long() {
let long_line = vec![b'1'; CHUNK_SIZE_LINE_MAX_LEN];
let mut input = long_line;
input.extend_from_slice(b"A\r\nData\r\n");
let (result, transfer, _) = run_transfer_fragmented(&[&input], true).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::Other); assert!(err.to_string().contains("exceeded max length"));
assert!(!transfer.is_done());
}
#[tokio::test]
async fn test_error_trailer_line_too_long() {
let mut input = b"0\r\n".to_vec();
let long_trailer = vec![b'a'; TRAILER_HEADER_LEN + 1];
input.extend_from_slice(&long_trailer);
input.extend_from_slice(b"\r\n\r\n");
let (result, transfer, _) = run_transfer_fragmented(&[&input], true).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::Other); assert!(err
.to_string()
.contains("could not read trailer header, line exceeded max length"));
assert!(!transfer.is_done());
}
#[tokio::test]
async fn test_error_invalid_trailer_format() {
let input = b"0\r\nInvalid Header Line\r\n\r\n";
let (result, transfer, _) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
assert!(err.to_string().contains("invalid trailer header format"));
assert!(!transfer.is_done());
}
#[tokio::test]
async fn test_error_data_after_final_trailer_crlf_same_run() {
let input = b"0\r\nTrailer: Val\r\n\r\nExtraData";
let (result, transfer, _) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::Other);
assert!(err
.to_string()
.contains("extra data received after chunked transfer completion"));
assert_eq!(transfer.state, ChunkTransferState::Done);
}
#[tokio::test]
async fn test_error_data_after_final_trailer_crlf_separate_run() {
let fragments = [b"0\r\n\r\n".as_slice(), b"ExtraData".as_slice()];
let (result, transfer, _) = run_transfer_fragmented(&fragments, true).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), std::io::ErrorKind::Other);
assert!(err
.to_string()
.contains("Extra data fragment 1 received after completion"));
assert!(transfer.is_done()); }
#[tokio::test]
async fn test_state_transitions_detailed() {
let mut transfer = ChunkTransfer::new();
let mut writer = MockWriter::new();
let mut opt_writer_ref = Some(&mut writer as &mut (dyn AsyncWrite + Unpin));
assert_eq!(
transfer.state,
ChunkTransferState::ReadSize { cached_len: 0 }
);
assert!(transfer.read_size_buf.is_empty());
transfer.run(b"A", &mut opt_writer_ref).await.unwrap();
assert_eq!(
transfer.state,
ChunkTransferState::ReadSize { cached_len: 1 } );
assert_eq!(&transfer.read_size_buf, b"A");
transfer
.run(b"\r\n0123", &mut opt_writer_ref)
.await
.unwrap();
match transfer.state {
ChunkTransferState::ReadData {
chunk_len,
remaining_len,
} => {
assert_eq!(chunk_len, 10);
assert_eq!(remaining_len, 10 + 2 - 4); }
_ => panic!("Incorrect state: {:?}", transfer.state),
}
assert!(transfer.read_size_buf.is_empty());
transfer
.run(b"456789\r\n5\r", &mut opt_writer_ref)
.await
.unwrap();
assert_eq!(
transfer.state,
ChunkTransferState::ReadSize { cached_len: 2 } );
assert_eq!(&transfer.read_size_buf, b"5\r");
transfer
.run(b"\nABCDE\r\n0\r\nTrailer:", &mut opt_writer_ref)
.await
.unwrap();
match transfer.state {
ChunkTransferState::ReadTrailer { cached_len } => {
assert_eq!(cached_len, 8); assert_eq!(&transfer.trailer_header_buf[0..cached_len], b"Trailer:");
}
_ => panic!("Incorrect state: {:?}", transfer.state),
}
assert!(transfer.read_size_buf.is_empty());
transfer
.run(b" Value\r\n\r\n", &mut opt_writer_ref)
.await
.unwrap();
assert!(transfer.is_done());
assert_eq!(
transfer.trailer_headers().get("trailer"), Some(&"Value".to_string()) );
let expected_forward = b"A\r\n0123456789\r\n5\r\nABCDE\r\n0\r\nTrailer: Value\r\n\r\n";
let written_data = writer.get_written_data().await;
assert_eq!(written_data, expected_forward);
}
#[tokio::test]
async fn test_chunk_extension_ignored() {
let input = b"A;extension=value\r\n0123456789\r\n0\r\n\r\n";
let expected_forward = input; let (result, mut transfer, written_data) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
assert!(transfer.trailer_headers().is_empty());
}
#[tokio::test]
async fn test_chunk_extension_fragmented() {
let fragments = [
b"A;ext".as_slice(),
b"ension=value\r\n".as_slice(),
b"0123456789\r\n".as_slice(),
b"0;".as_slice(), b"last=chunk\r\n".as_slice(),
b"\r\n".as_slice(), ];
let expected_forward = b"A;extension=value\r\n0123456789\r\n0;last=chunk\r\n\r\n";
let (result, mut transfer, written_data) = run_transfer_fragmented(&fragments, true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
assert!(transfer.trailer_headers().is_empty());
}
#[tokio::test]
async fn test_chunk_extension_with_space_before_crlf() {
let input = b"A ; extension=value \r\n0123456789\r\n0\r\n\r\n";
let expected_forward = input;
let (result, mut transfer, written_data) = run_transfer_fragmented(&[input], true).await;
assert!(result.is_ok(), "Result was: {:?}", result);
assert!(transfer.is_done());
assert_eq!(written_data.unwrap(), expected_forward);
assert!(transfer.trailer_headers().is_empty());
}
#[tokio::test]
async fn test_hang_on_final_chunk_in_single_buffer() {
let mut transfer = ChunkTransfer::new();
let mut writer = MockWriter::new();
let mut opt_writer_ref = Some(&mut writer as &mut (dyn AsyncWrite + Unpin));
let final_chunk_and_termination = b"0\r\n\r\n";
let processing_result = timeout(
Duration::from_millis(100), transfer.run(final_chunk_and_termination, &mut opt_writer_ref),
)
.await;
assert!(
processing_result.is_ok(),
"The 'run' method timed out, indicating a hang."
);
let inner_result =
processing_result.expect("Test timed out, which should not happen with fixed code");
assert!(
inner_result.is_ok(),
"The 'run' method returned an I/O error: {:?}",
inner_result
);
assert!(
transfer.is_done(),
"The transfer did not reach the 'Done' state."
);
let written_data = writer.get_written_data().await;
assert_eq!(written_data, final_chunk_and_termination);
}
}