use crate::error::{Error, Result};
use bytes::{Bytes, BytesMut};
use futures::stream::{BoxStream, Stream, StreamExt};
use std::fmt;
use tokio::io::AsyncReadExt;
pub const FILE_CHUNK_SIZE: usize = 64 * 1024;
pub struct BodyStream {
inner: BoxStream<'static, Result<Bytes>>,
size: Option<u64>,
}
impl BodyStream {
pub fn new<S>(stream: S) -> Self
where
S: Stream<Item = Result<Bytes>> + Send + 'static,
{
Self {
inner: stream.boxed(),
size: None,
}
}
pub fn with_size(mut self, bytes: u64) -> Self {
self.size = Some(bytes);
self
}
pub fn size(&self) -> Option<u64> {
self.size
}
pub fn into_stream(self) -> BoxStream<'static, Result<Bytes>> {
self.inner
}
pub fn into_enforced_stream(self) -> BoxStream<'static, Result<Bytes>> {
match self.size {
Some(size) => enforce_size(self.inner, size).boxed(),
None => self.inner,
}
}
}
fn enforce_size(
stream: BoxStream<'static, Result<Bytes>>,
declared: u64,
) -> impl Stream<Item = Result<Bytes>> + Send + 'static {
if declared == 0 {
return futures::stream::empty().boxed();
}
struct State {
inner: BoxStream<'static, Result<Bytes>>,
remaining: u64,
declared: u64,
finished: bool,
}
futures::stream::unfold(
State {
inner: stream,
remaining: declared,
declared,
finished: false,
},
|mut state| async move {
if state.finished {
return None;
}
match state.inner.next().await {
Some(Ok(mut chunk)) => {
let len = chunk.len() as u64;
if len > state.remaining {
log::warn!(
"Response body stream produced more than its declared {} bytes; \
truncating to the declared length",
state.declared
);
chunk.truncate(state.remaining as usize);
}
state.remaining -= chunk.len() as u64;
if state.remaining == 0 {
state.finished = true;
}
Some((Ok(chunk), state))
}
Some(Err(error)) => {
state.finished = true;
Some((Err(error), state))
}
None => {
state.finished = true;
if state.remaining > 0 {
let sent = state.declared - state.remaining;
Some((
Err(Error::internal(format!(
"response body stream ended after {} of {} declared bytes",
sent, state.declared
))),
state,
))
} else {
None
}
}
}
},
)
.boxed()
}
impl fmt::Debug for BodyStream {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("BodyStream")
.field("size", &self.size)
.finish_non_exhaustive()
}
}
#[derive(Debug)]
pub enum Body {
Full(Vec<u8>),
Stream(BodyStream),
}
impl Body {
pub fn empty() -> Self {
Body::Full(Vec::new())
}
pub fn from_stream<S>(stream: S) -> Self
where
S: Stream<Item = Result<Bytes>> + Send + 'static,
{
Body::Stream(BodyStream::new(stream))
}
pub fn from_sized_stream<S>(stream: S, size: u64) -> Self
where
S: Stream<Item = Result<Bytes>> + Send + 'static,
{
Body::Stream(BodyStream::new(stream).with_size(size))
}
pub fn from_file(file: tokio::fs::File, size: u64) -> Self {
Self::from_sized_stream(file_chunks(file, FILE_CHUNK_SIZE), size)
}
pub fn is_stream(&self) -> bool {
matches!(self, Body::Stream(_))
}
pub fn len_hint(&self) -> Option<usize> {
match self {
Body::Full(bytes) => Some(bytes.len()),
Body::Stream(stream) => stream.size().and_then(|n| usize::try_from(n).ok()),
}
}
pub fn is_empty(&self) -> bool {
match self {
Body::Full(bytes) => bytes.is_empty(),
Body::Stream(_) => false,
}
}
pub fn as_slice(&self) -> Option<&[u8]> {
match self {
Body::Full(bytes) => Some(bytes),
Body::Stream(_) => None,
}
}
pub fn as_mut_vec(&mut self) -> Option<&mut Vec<u8>> {
match self {
Body::Full(bytes) => Some(bytes),
Body::Stream(_) => None,
}
}
pub fn into_vec(self) -> Option<Vec<u8>> {
match self {
Body::Full(bytes) => Some(bytes),
Body::Stream(_) => None,
}
}
}
fn file_chunks(
file: tokio::fs::File,
chunk_size: usize,
) -> impl Stream<Item = Result<Bytes>> + Send + 'static {
futures::stream::try_unfold(file, move |mut file| async move {
let mut buf = BytesMut::zeroed(chunk_size);
let read = file.read(&mut buf).await.map_err(Error::Io)?;
if read == 0 {
return Ok(None);
}
buf.truncate(read);
Ok(Some((buf.freeze(), file)))
})
}
impl Default for Body {
fn default() -> Self {
Body::empty()
}
}
impl From<Vec<u8>> for Body {
fn from(bytes: Vec<u8>) -> Self {
Body::Full(bytes)
}
}
impl From<&[u8]> for Body {
fn from(bytes: &[u8]) -> Self {
Body::Full(bytes.to_vec())
}
}
impl From<Bytes> for Body {
fn from(bytes: Bytes) -> Self {
Body::Full(bytes.to_vec())
}
}
impl From<String> for Body {
fn from(text: String) -> Self {
Body::Full(text.into_bytes())
}
}
impl From<&str> for Body {
fn from(text: &str) -> Self {
Body::Full(text.as_bytes().to_vec())
}
}
impl From<BodyStream> for Body {
fn from(stream: BodyStream) -> Self {
Body::Stream(stream)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error::Error;
#[test]
fn full_body_reports_length_and_emptiness() {
let body = Body::from("hello");
assert!(!body.is_stream());
assert_eq!(body.len_hint(), Some(5));
assert!(!body.is_empty());
assert_eq!(body.as_slice(), Some(&b"hello"[..]));
assert!(Body::empty().is_empty());
assert_eq!(Body::empty().len_hint(), Some(0));
}
#[test]
fn unsized_stream_hides_its_length() {
let body = Body::from_stream(futures::stream::iter(vec![
Ok(Bytes::from_static(b"a")),
Ok(Bytes::from_static(b"b")),
]));
assert!(body.is_stream());
assert_eq!(body.len_hint(), None);
assert!(!body.is_empty());
assert_eq!(body.as_slice(), None);
assert_eq!(body.into_vec(), None);
}
#[test]
fn sized_stream_exposes_declared_length() {
let body = Body::from_sized_stream(
futures::stream::iter(vec![Ok(Bytes::from_static(b"12345"))]),
5,
);
assert!(body.is_stream());
assert_eq!(body.len_hint(), Some(5));
}
#[test]
fn as_mut_vec_edits_buffered_and_refuses_streamed() {
let mut buffered = Body::from("hi");
buffered
.as_mut_vec()
.expect("buffered body is editable")
.extend_from_slice(b"!");
assert_eq!(buffered.as_slice(), Some(&b"hi!"[..]));
let mut streamed = Body::from_stream(futures::stream::empty());
assert!(streamed.as_mut_vec().is_none());
}
#[tokio::test]
async fn stream_yields_its_chunks_in_order() {
let body = Body::from_stream(futures::stream::iter(vec![
Ok(Bytes::from_static(b"one")),
Ok(Bytes::from_static(b"two")),
]));
let Body::Stream(stream) = body else {
panic!("expected a streaming body");
};
let chunks: Vec<Bytes> = stream
.into_stream()
.map(|chunk| chunk.expect("chunk should be Ok"))
.collect()
.await;
assert_eq!(
chunks,
vec![Bytes::from_static(b"one"), Bytes::from_static(b"two")]
);
}
#[tokio::test]
async fn enforced_stream_passes_an_exact_size_through_unchanged() {
let body = Body::from_sized_stream(
futures::stream::iter(vec![
Ok(Bytes::from_static(b"abc")),
Ok(Bytes::from_static(b"de")),
]),
5,
);
let Body::Stream(stream) = body else {
panic!("expected a streaming body");
};
let chunks: Vec<Result<Bytes>> = stream.into_enforced_stream().collect().await;
let bytes: Vec<u8> = chunks
.into_iter()
.flat_map(|c| c.expect("no error for an exact size"))
.collect();
assert_eq!(bytes, b"abcde");
}
#[tokio::test]
async fn enforced_stream_truncates_an_over_producing_source() {
let body = Body::from_sized_stream(
futures::stream::iter(vec![
Ok(Bytes::from_static(b"abc")),
Ok(Bytes::from_static(b"defg")),
Ok(Bytes::from_static(b"never polled")),
]),
4,
);
let Body::Stream(stream) = body else {
panic!("expected a streaming body");
};
let chunks: Vec<Result<Bytes>> = stream.into_enforced_stream().collect().await;
assert_eq!(chunks.len(), 2);
let bytes: Vec<u8> = chunks
.into_iter()
.flat_map(|c| c.expect("truncation is not an error"))
.collect();
assert_eq!(bytes, b"abcd");
}
#[tokio::test]
async fn enforced_stream_with_zero_declared_size_never_polls_the_source() {
let polled = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let flag = polled.clone();
let source = futures::stream::poll_fn(move |_| {
flag.store(true, std::sync::atomic::Ordering::SeqCst);
std::task::Poll::Ready(Some(Ok(Bytes::from_static(b"unexpected"))))
});
let body = Body::from_sized_stream(source, 0);
let Body::Stream(stream) = body else {
panic!("expected a streaming body");
};
let chunks: Vec<Result<Bytes>> = stream.into_enforced_stream().collect().await;
assert!(chunks.is_empty());
assert!(!polled.load(std::sync::atomic::Ordering::SeqCst));
}
#[tokio::test]
async fn enforced_stream_fails_when_the_source_ends_short() {
let body = Body::from_sized_stream(
futures::stream::iter(vec![Ok(Bytes::from_static(b"abc"))]),
10,
);
let Body::Stream(stream) = body else {
panic!("expected a streaming body");
};
let chunks: Vec<Result<Bytes>> = stream.into_enforced_stream().collect().await;
assert_eq!(chunks.len(), 2);
assert_eq!(chunks[0].as_ref().unwrap(), &Bytes::from_static(b"abc"));
let message = chunks[1].as_ref().unwrap_err().to_string();
assert!(
message.contains("3 of 10"),
"error should say how short it fell: {message}"
);
}
#[tokio::test]
async fn stream_errors_surface_to_the_consumer() {
let body = Body::from_stream(futures::stream::iter(vec![
Ok(Bytes::from_static(b"partial")),
Err(Error::internal("disk read failed")),
]));
let Body::Stream(stream) = body else {
panic!("expected a streaming body");
};
let results: Vec<Result<Bytes>> = stream.into_stream().collect().await;
assert_eq!(results.len(), 2);
assert!(results[0].is_ok());
assert!(results[1].is_err());
}
}