tez 0.0.1

A lightweight, ergonomic, and fast high-level HTTP client for Rust
Documentation
use std::fmt;

use bytes::{Bytes, BytesMut};
use http_body::Body as _;
use serde::de::DeserializeOwned;

use crate::{Body, Error, Result};

const RECORD_LIMIT: usize = 32 * 1024 * 1024;

/// An experimental, one-shot reader for newline-delimited JSON responses.
///
/// Call [`Ndjson::json`] repeatedly to deserialize records as they arrive. The
/// reader accepts both LF and CRLF separators and also accepts a final record
/// without a trailing newline.
pub struct Ndjson {
    body: Body,
    pending: BytesMut,
    searched: usize,
    finished: bool,
}

impl Ndjson {
    pub(crate) fn new(body: Body) -> Self {
        Self {
            body,
            pending: BytesMut::new(),
            searched: 0,
            finished: false,
        }
    }

    /// Reads and deserializes the next JSON record.
    ///
    /// Returns `None` after the response body has ended. Empty records,
    /// malformed JSON, invalid UTF-8, and records larger than 32 MiB are decode
    /// errors. After any error, the reader is exhausted.
    pub async fn json<T>(&mut self) -> Result<Option<T>>
    where
        T: DeserializeOwned,
    {
        let record = match self.next_record().await {
            Ok(record) => record,
            Err(error) => {
                self.finish();
                return Err(error);
            }
        };
        let Some(record) = record else {
            return Ok(None);
        };

        match serde_json::from_slice(&record) {
            Ok(value) => Ok(Some(value)),
            Err(_) => {
                self.finish();
                Err(Error::decode("NDJSON record could not be decoded"))
            }
        }
    }

    async fn next_record(&mut self) -> Result<Option<Bytes>> {
        loop {
            if let Some(newline) = self.pending[self.searched..]
                .iter()
                .position(|byte| *byte == b'\n')
                .map(|offset| self.searched + offset)
            {
                let mut record = self.pending.split_to(newline + 1);
                self.searched = 0;
                record.truncate(newline);
                if record.last() == Some(&b'\r') {
                    record.truncate(record.len() - 1);
                }
                return validate_record(record.freeze()).map(Some);
            }
            self.searched = self.pending.len();

            if self.finished {
                if self.pending.is_empty() {
                    return Ok(None);
                }
                let record = self.pending.split().freeze();
                self.searched = 0;
                return validate_record(record).map(Some);
            }

            ensure_pending_record_fits(&self.pending)?;
            match std::future::poll_fn(|context| {
                std::pin::Pin::new(&mut self.body).poll_frame(context)
            })
            .await
            {
                Some(Ok(frame)) => {
                    if let Ok(data) = frame.into_data() {
                        self.pending.extend_from_slice(&data);
                    }
                }
                Some(Err(error)) => return Err(error),
                None => self.finished = true,
            }
        }
    }

    fn finish(&mut self) {
        self.finished = true;
        self.pending.clear();
        self.searched = 0;
    }
}

impl fmt::Debug for Ndjson {
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
        formatter.debug_struct("Ndjson").finish_non_exhaustive()
    }
}

fn validate_record(record: Bytes) -> Result<Bytes> {
    if record.is_empty() {
        return Err(Error::decode("NDJSON record is empty"));
    }
    if record.len() > RECORD_LIMIT {
        return Err(Error::decode("NDJSON record exceeds the 32 MiB limit"));
    }
    Ok(record)
}

fn ensure_pending_record_fits(pending: &[u8]) -> Result<()> {
    let may_be_crlf = pending.len() == RECORD_LIMIT + 1 && pending.last() == Some(&b'\r');
    if pending.len() > RECORD_LIMIT && !may_be_crlf {
        return Err(Error::decode("NDJSON record exceeds the 32 MiB limit"));
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use std::convert::Infallible;

    use futures_util::stream;
    use http_body::Frame;
    use http_body_util::StreamBody;
    use serde::Deserialize;

    use super::*;

    #[derive(Debug, Deserialize, Eq, PartialEq)]
    struct Message {
        text: String,
    }

    fn reader(chunks: &[&'static str]) -> Ndjson {
        let frames = chunks
            .iter()
            .map(|chunk| Ok::<_, Infallible>(Frame::data(Bytes::copy_from_slice(chunk.as_bytes()))))
            .collect::<Vec<_>>();
        Ndjson::new(Body::wrap(StreamBody::new(stream::iter(frames))))
    }

    #[tokio::test]
    async fn json_preserves_records_split_across_body_frames() {
        let mut ndjson = reader(&[
            "{\"text\":\"fir",
            "st\"}\n{\"text\":\"second\"}\r",
            "\n{\"text\":\"third\"}",
        ]);

        assert_eq!(
            ndjson.json::<Message>().await.unwrap(),
            Some(Message {
                text: "first".to_owned()
            })
        );
        assert_eq!(
            ndjson.json::<Message>().await.unwrap(),
            Some(Message {
                text: "second".to_owned()
            })
        );
        assert_eq!(
            ndjson.json::<Message>().await.unwrap(),
            Some(Message {
                text: "third".to_owned()
            })
        );
        assert_eq!(ndjson.json::<Message>().await.unwrap(), None);
    }

    #[tokio::test]
    async fn json_exhausts_the_reader_after_a_decode_error() {
        let mut malformed = reader(&["not-json\n{\"text\":\"ignored\"}\n"]);

        assert!(malformed.json::<Message>().await.unwrap_err().is_decode());
        assert_eq!(malformed.json::<Message>().await.unwrap(), None);

        let mut empty = reader(&["\n"]);
        assert!(empty.json::<Message>().await.unwrap_err().is_decode());
        assert_eq!(empty.json::<Message>().await.unwrap(), None);
    }
}