use std::fmt;
use bytes::{Bytes, BytesMut};
use http_body::Body as _;
use serde::de::DeserializeOwned;
use crate::{Body, Error, Result};
const RECORD_LIMIT: usize = 32 * 1024 * 1024;
pub struct Ndjson {
body: Body,
pending: BytesMut,
searched: usize,
finished: bool,
}
impl Ndjson {
pub(crate) fn new(body: Body) -> Self {
Self {
body,
pending: BytesMut::new(),
searched: 0,
finished: false,
}
}
pub async fn json<T>(&mut self) -> Result<Option<T>>
where
T: DeserializeOwned,
{
let record = match self.next_record().await {
Ok(record) => record,
Err(error) => {
self.finish();
return Err(error);
}
};
let Some(record) = record else {
return Ok(None);
};
match serde_json::from_slice(&record) {
Ok(value) => Ok(Some(value)),
Err(_) => {
self.finish();
Err(Error::decode("NDJSON record could not be decoded"))
}
}
}
async fn next_record(&mut self) -> Result<Option<Bytes>> {
loop {
if let Some(newline) = self.pending[self.searched..]
.iter()
.position(|byte| *byte == b'\n')
.map(|offset| self.searched + offset)
{
let mut record = self.pending.split_to(newline + 1);
self.searched = 0;
record.truncate(newline);
if record.last() == Some(&b'\r') {
record.truncate(record.len() - 1);
}
return validate_record(record.freeze()).map(Some);
}
self.searched = self.pending.len();
if self.finished {
if self.pending.is_empty() {
return Ok(None);
}
let record = self.pending.split().freeze();
self.searched = 0;
return validate_record(record).map(Some);
}
ensure_pending_record_fits(&self.pending)?;
match std::future::poll_fn(|context| {
std::pin::Pin::new(&mut self.body).poll_frame(context)
})
.await
{
Some(Ok(frame)) => {
if let Ok(data) = frame.into_data() {
self.pending.extend_from_slice(&data);
}
}
Some(Err(error)) => return Err(error),
None => self.finished = true,
}
}
}
fn finish(&mut self) {
self.finished = true;
self.pending.clear();
self.searched = 0;
}
}
impl fmt::Debug for Ndjson {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.debug_struct("Ndjson").finish_non_exhaustive()
}
}
fn validate_record(record: Bytes) -> Result<Bytes> {
if record.is_empty() {
return Err(Error::decode("NDJSON record is empty"));
}
if record.len() > RECORD_LIMIT {
return Err(Error::decode("NDJSON record exceeds the 32 MiB limit"));
}
Ok(record)
}
fn ensure_pending_record_fits(pending: &[u8]) -> Result<()> {
let may_be_crlf = pending.len() == RECORD_LIMIT + 1 && pending.last() == Some(&b'\r');
if pending.len() > RECORD_LIMIT && !may_be_crlf {
return Err(Error::decode("NDJSON record exceeds the 32 MiB limit"));
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::convert::Infallible;
use futures_util::stream;
use http_body::Frame;
use http_body_util::StreamBody;
use serde::Deserialize;
use super::*;
#[derive(Debug, Deserialize, Eq, PartialEq)]
struct Message {
text: String,
}
fn reader(chunks: &[&'static str]) -> Ndjson {
let frames = chunks
.iter()
.map(|chunk| Ok::<_, Infallible>(Frame::data(Bytes::copy_from_slice(chunk.as_bytes()))))
.collect::<Vec<_>>();
Ndjson::new(Body::wrap(StreamBody::new(stream::iter(frames))))
}
#[tokio::test]
async fn json_preserves_records_split_across_body_frames() {
let mut ndjson = reader(&[
"{\"text\":\"fir",
"st\"}\n{\"text\":\"second\"}\r",
"\n{\"text\":\"third\"}",
]);
assert_eq!(
ndjson.json::<Message>().await.unwrap(),
Some(Message {
text: "first".to_owned()
})
);
assert_eq!(
ndjson.json::<Message>().await.unwrap(),
Some(Message {
text: "second".to_owned()
})
);
assert_eq!(
ndjson.json::<Message>().await.unwrap(),
Some(Message {
text: "third".to_owned()
})
);
assert_eq!(ndjson.json::<Message>().await.unwrap(), None);
}
#[tokio::test]
async fn json_exhausts_the_reader_after_a_decode_error() {
let mut malformed = reader(&["not-json\n{\"text\":\"ignored\"}\n"]);
assert!(malformed.json::<Message>().await.unwrap_err().is_decode());
assert_eq!(malformed.json::<Message>().await.unwrap(), None);
let mut empty = reader(&["\n"]);
assert!(empty.json::<Message>().await.unwrap_err().is_decode());
assert_eq!(empty.json::<Message>().await.unwrap(), None);
}
}