use std::{error::Error as StdError, io};
use bytes::Bytes;
use futures_util::{Stream, StreamExt as _};
use serde::Serialize;
use crate::{Body, Error, Result};
pub(crate) const CONTENT_TYPE: &str = "application/x-ndjson";
pub(crate) const DOCUMENT_CONTENT_TYPE: &str = "application/json";
pub(crate) fn encode<S, T, E>(stream: S, item_limit: usize) -> Body
where
S: Stream<Item = std::result::Result<T, E>> + Send + 'static,
T: Serialize,
E: StdError + Send + Sync + 'static,
{
Body::stream(stream.map(move |item| {
item.map_err(|_| Error::body()).and_then(|value| {
let value_limit = item_limit.checked_sub(1).ok_or_else(Error::body)?;
let mut encoded = encode_value(&value, value_limit)?;
encoded.push(b'\n');
Ok(Bytes::from(encoded))
})
}))
}
pub(crate) fn encode_array<S, T, E>(stream: S, item_limit: usize) -> Body
where
S: Stream<Item = std::result::Result<T, E>> + Send + 'static,
T: Serialize,
E: StdError + Send + Sync + 'static,
{
let opening = futures_util::stream::once(async { Ok::<_, Error>(Bytes::from_static(b"[")) });
let values = stream.enumerate().map(move |(index, item)| {
item.map_err(|_| Error::body()).and_then(|value| {
let value = encode_value(&value, item_limit)?;
if index == 0 {
return Ok(Bytes::from(value));
}
let mut encoded = Vec::with_capacity(value.len() + 1);
encoded.push(b',');
encoded.extend_from_slice(&value);
Ok(Bytes::from(encoded))
})
});
let closing = futures_util::stream::once(async { Ok::<_, Error>(Bytes::from_static(b"]")) });
Body::stream(opening.chain(values).chain(closing))
}
fn encode_value<T>(value: &T, limit: usize) -> Result<Vec<u8>>
where
T: Serialize,
{
let mut encoded = LimitedVec::new(limit);
serde_json::to_writer(&mut encoded, value).map_err(|_| Error::body())?;
Ok(encoded.into_inner())
}
struct LimitedVec {
bytes: Vec<u8>,
limit: usize,
}
impl LimitedVec {
fn new(limit: usize) -> Self {
Self {
bytes: Vec::with_capacity(limit.min(1024)),
limit,
}
}
fn into_inner(self) -> Vec<u8> {
self.bytes
}
}
impl io::Write for LimitedVec {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
let length = self
.bytes
.len()
.checked_add(bytes.len())
.filter(|length| *length <= self.limit)
.ok_or_else(|| io::Error::other("JSON value exceeds the configured stream limit"))?;
self.bytes.reserve(length.saturating_sub(self.bytes.len()));
self.bytes.extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
#[derive(Debug)]
pub(super) struct Document {
data: Vec<u8>,
limit: usize,
}
impl Document {
pub(super) fn new(limit: usize) -> Self {
Self {
data: Vec::new(),
limit,
}
}
pub(super) fn push_one(&mut self, bytes: &[u8], offset: &mut usize) -> Result<Option<Bytes>> {
let data = &bytes[*offset..];
let new_len = self
.data
.len()
.checked_add(data.len())
.filter(|length| *length <= self.limit)
.ok_or_else(|| Error::decode("JSON document exceeds the configured stream limit"))?;
self.data.reserve(new_len.saturating_sub(self.data.len()));
self.data.extend_from_slice(data);
*offset = bytes.len();
Ok(None)
}
pub(super) fn finish(&mut self) -> Option<Bytes> {
Some(Bytes::from(std::mem::take(&mut self.data)))
}
}
#[derive(Debug)]
pub(super) struct Lines {
record: Vec<u8>,
limit: usize,
}
impl Lines {
pub(super) fn new(limit: usize) -> Self {
Self {
record: Vec::new(),
limit,
}
}
pub(super) fn push_one(&mut self, bytes: &[u8], offset: &mut usize) -> Result<Option<Bytes>> {
let remaining = &bytes[*offset..];
if let Some(newline) = remaining.iter().position(|byte| *byte == b'\n') {
self.record.extend_from_slice(&remaining[..newline]);
*offset += newline + 1;
self.ensure_record_fits()?;
return self.finish_record().map(Some);
}
self.record.extend_from_slice(remaining);
*offset = bytes.len();
self.ensure_record_fits()?;
Ok(None)
}
pub(super) fn finish(&mut self) -> Result<Option<Bytes>> {
if self.record.is_empty() {
return Ok(None);
}
self.finish_record().map(Some)
}
fn finish_record(&mut self) -> Result<Bytes> {
if self.record.last() == Some(&b'\r') {
self.record.truncate(self.record.len() - 1);
}
if self.record.is_empty() {
return Err(Error::decode("NDJSON record is empty"));
}
if self.record.len() > self.limit {
return Err(Error::decode(
"NDJSON record exceeds the configured stream limit",
));
}
Ok(Bytes::from(std::mem::take(&mut self.record)))
}
fn ensure_record_fits(&self) -> Result<()> {
let may_end_with_cr =
self.record.len() == self.limit + 1 && self.record.last() == Some(&b'\r');
if self.record.len() > self.limit && !may_end_with_cr {
return Err(Error::decode(
"NDJSON record exceeds the configured stream limit",
));
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use http_body_util::BodyExt as _;
use std::convert::Infallible;
fn decode_document(decoder: &mut Document, chunks: &[&[u8]]) -> Result<Bytes> {
for chunk in chunks {
let mut offset = 0;
assert!(decoder.push_one(chunk, &mut offset)?.is_none());
assert_eq!(offset, chunk.len());
}
decoder
.finish()
.ok_or_else(|| Error::decode("JSON decoder did not emit its document"))
}
fn decode_lines(decoder: &mut Lines, chunks: &[&[u8]]) -> Result<Vec<Bytes>> {
let mut records = Vec::new();
for chunk in chunks {
let mut offset = 0;
while offset < chunk.len() {
if let Some(record) = decoder.push_one(chunk, &mut offset)? {
records.push(record);
}
}
}
if let Some(record) = decoder.finish()? {
records.push(record);
}
Ok(records)
}
#[test]
fn document_decoder_preserves_json_split_across_body_frames() {
let document =
decode_document(&mut Document::new(1024), &[br#"{"text":"fir"#, br#"st"}"#]).unwrap();
assert_eq!(document, r#"{"text":"first"}"#);
}
#[test]
fn line_decoder_preserves_records_split_across_body_frames() {
let records = decode_lines(
&mut Lines::new(1024),
&[
br#"{"text":"fir"#,
b"st\"}\n{\"text\":\"second\"}\r",
b"\n{\"text\":\"third\"}",
],
)
.unwrap();
assert_eq!(
records,
[
Bytes::from_static(br#"{"text":"first"}"#),
Bytes::from_static(br#"{"text":"second"}"#),
Bytes::from_static(br#"{"text":"third"}"#),
]
);
}
#[test]
fn decoders_enforce_the_item_limit() {
assert_eq!(
decode_document(&mut Document::new(3), &[b"123"]).unwrap(),
"123"
);
assert!(
decode_document(&mut Document::new(2), &[b"123"])
.unwrap_err()
.is_decode()
);
assert_eq!(
decode_lines(&mut Lines::new(3), &[b"123\r\n"]).unwrap()[0],
"123"
);
assert!(
decode_lines(&mut Lines::new(2), &[b"123\n"])
.unwrap_err()
.is_decode()
);
}
#[tokio::test]
async fn encoder_serializes_one_value_per_body_frame() {
let stream = futures_util::stream::iter([
Ok::<_, Infallible>(serde_json::json!({ "id": 1 })),
Ok(serde_json::json!({ "id": 2 })),
]);
let frames = encode(stream, 1024).collect().await.unwrap().to_bytes();
assert_eq!(frames, "{\"id\":1}\n{\"id\":2}\n");
}
#[tokio::test]
async fn array_encoder_frames_a_valid_document_without_collecting_the_sequence() {
let stream = futures_util::stream::iter([
Ok::<_, Infallible>(serde_json::json!({ "id": 1 })),
Ok(serde_json::json!({ "id": 2 })),
]);
let frames = encode_array(stream, 1024)
.collect()
.await
.unwrap()
.to_bytes();
assert_eq!(frames, "[{\"id\":1},{\"id\":2}]");
}
#[tokio::test]
async fn encoders_reject_one_value_beyond_the_item_limit() {
let record = futures_util::stream::iter([Ok::<_, Infallible>("1234")]);
assert!(
encode(record, 3)
.frame()
.await
.unwrap()
.unwrap_err()
.is_body()
);
let value = futures_util::stream::iter([Ok::<_, Infallible>("1234")]);
let mut body = encode_array(value, 3);
assert!(body.frame().await.unwrap().is_ok());
assert!(body.frame().await.unwrap().unwrap_err().is_body());
let record = futures_util::stream::iter([Ok::<_, Infallible>(1)]);
assert!(
encode(record, 1)
.frame()
.await
.unwrap()
.unwrap_err()
.is_body()
);
}
}