tez 0.0.3

A lightweight, ergonomic, and fast high-level HTTP client for Rust
Documentation
use std::{error::Error as StdError, io};

use bytes::Bytes;
use futures_util::{Stream, StreamExt as _};
use serde::Serialize;

use crate::{Body, Error, Result};

pub(crate) const CONTENT_TYPE: &str = "application/x-ndjson";
pub(crate) const DOCUMENT_CONTENT_TYPE: &str = "application/json";

pub(crate) fn encode<S, T, E>(stream: S, item_limit: usize) -> Body
where
    S: Stream<Item = std::result::Result<T, E>> + Send + 'static,
    T: Serialize,
    E: StdError + Send + Sync + 'static,
{
    Body::stream(stream.map(move |item| {
        item.map_err(|_| Error::body()).and_then(|value| {
            let value_limit = item_limit.checked_sub(1).ok_or_else(Error::body)?;
            let mut encoded = encode_value(&value, value_limit)?;
            encoded.push(b'\n');
            Ok(Bytes::from(encoded))
        })
    }))
}

pub(crate) fn encode_array<S, T, E>(stream: S, item_limit: usize) -> Body
where
    S: Stream<Item = std::result::Result<T, E>> + Send + 'static,
    T: Serialize,
    E: StdError + Send + Sync + 'static,
{
    let opening = futures_util::stream::once(async { Ok::<_, Error>(Bytes::from_static(b"[")) });
    let values = stream.enumerate().map(move |(index, item)| {
        item.map_err(|_| Error::body()).and_then(|value| {
            let value = encode_value(&value, item_limit)?;
            if index == 0 {
                return Ok(Bytes::from(value));
            }
            let mut encoded = Vec::with_capacity(value.len() + 1);
            encoded.push(b',');
            encoded.extend_from_slice(&value);
            Ok(Bytes::from(encoded))
        })
    });
    let closing = futures_util::stream::once(async { Ok::<_, Error>(Bytes::from_static(b"]")) });
    Body::stream(opening.chain(values).chain(closing))
}

fn encode_value<T>(value: &T, limit: usize) -> Result<Vec<u8>>
where
    T: Serialize,
{
    let mut encoded = LimitedVec::new(limit);
    serde_json::to_writer(&mut encoded, value).map_err(|_| Error::body())?;
    Ok(encoded.into_inner())
}

struct LimitedVec {
    bytes: Vec<u8>,
    limit: usize,
}

impl LimitedVec {
    fn new(limit: usize) -> Self {
        Self {
            bytes: Vec::with_capacity(limit.min(1024)),
            limit,
        }
    }

    fn into_inner(self) -> Vec<u8> {
        self.bytes
    }
}

impl io::Write for LimitedVec {
    fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
        let length = self
            .bytes
            .len()
            .checked_add(bytes.len())
            .filter(|length| *length <= self.limit)
            .ok_or_else(|| io::Error::other("JSON value exceeds the configured stream limit"))?;
        self.bytes.reserve(length.saturating_sub(self.bytes.len()));
        self.bytes.extend_from_slice(bytes);
        Ok(bytes.len())
    }

    fn flush(&mut self) -> io::Result<()> {
        Ok(())
    }
}

#[derive(Debug)]
pub(super) struct Document {
    data: Vec<u8>,
    limit: usize,
}

impl Document {
    pub(super) fn new(limit: usize) -> Self {
        Self {
            data: Vec::new(),
            limit,
        }
    }

    pub(super) fn push_one(&mut self, bytes: &[u8], offset: &mut usize) -> Result<Option<Bytes>> {
        let data = &bytes[*offset..];
        let new_len = self
            .data
            .len()
            .checked_add(data.len())
            .filter(|length| *length <= self.limit)
            .ok_or_else(|| Error::decode("JSON document exceeds the configured stream limit"))?;
        self.data.reserve(new_len.saturating_sub(self.data.len()));
        self.data.extend_from_slice(data);
        *offset = bytes.len();
        Ok(None)
    }

    pub(super) fn finish(&mut self) -> Option<Bytes> {
        Some(Bytes::from(std::mem::take(&mut self.data)))
    }
}

#[derive(Debug)]
pub(super) struct Lines {
    record: Vec<u8>,
    limit: usize,
}

impl Lines {
    pub(super) fn new(limit: usize) -> Self {
        Self {
            record: Vec::new(),
            limit,
        }
    }

    pub(super) fn push_one(&mut self, bytes: &[u8], offset: &mut usize) -> Result<Option<Bytes>> {
        let remaining = &bytes[*offset..];
        if let Some(newline) = remaining.iter().position(|byte| *byte == b'\n') {
            self.record.extend_from_slice(&remaining[..newline]);
            *offset += newline + 1;
            self.ensure_record_fits()?;
            return self.finish_record().map(Some);
        }

        self.record.extend_from_slice(remaining);
        *offset = bytes.len();
        self.ensure_record_fits()?;
        Ok(None)
    }

    pub(super) fn finish(&mut self) -> Result<Option<Bytes>> {
        if self.record.is_empty() {
            return Ok(None);
        }
        self.finish_record().map(Some)
    }

    fn finish_record(&mut self) -> Result<Bytes> {
        if self.record.last() == Some(&b'\r') {
            self.record.truncate(self.record.len() - 1);
        }
        if self.record.is_empty() {
            return Err(Error::decode("NDJSON record is empty"));
        }
        if self.record.len() > self.limit {
            return Err(Error::decode(
                "NDJSON record exceeds the configured stream limit",
            ));
        }
        Ok(Bytes::from(std::mem::take(&mut self.record)))
    }

    fn ensure_record_fits(&self) -> Result<()> {
        let may_end_with_cr =
            self.record.len() == self.limit + 1 && self.record.last() == Some(&b'\r');
        if self.record.len() > self.limit && !may_end_with_cr {
            return Err(Error::decode(
                "NDJSON record exceeds the configured stream limit",
            ));
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use http_body_util::BodyExt as _;
    use std::convert::Infallible;

    fn decode_document(decoder: &mut Document, chunks: &[&[u8]]) -> Result<Bytes> {
        for chunk in chunks {
            let mut offset = 0;
            assert!(decoder.push_one(chunk, &mut offset)?.is_none());
            assert_eq!(offset, chunk.len());
        }
        decoder
            .finish()
            .ok_or_else(|| Error::decode("JSON decoder did not emit its document"))
    }

    fn decode_lines(decoder: &mut Lines, chunks: &[&[u8]]) -> Result<Vec<Bytes>> {
        let mut records = Vec::new();
        for chunk in chunks {
            let mut offset = 0;
            while offset < chunk.len() {
                if let Some(record) = decoder.push_one(chunk, &mut offset)? {
                    records.push(record);
                }
            }
        }
        if let Some(record) = decoder.finish()? {
            records.push(record);
        }
        Ok(records)
    }

    #[test]
    fn document_decoder_preserves_json_split_across_body_frames() {
        let document =
            decode_document(&mut Document::new(1024), &[br#"{"text":"fir"#, br#"st"}"#]).unwrap();
        assert_eq!(document, r#"{"text":"first"}"#);
    }

    #[test]
    fn line_decoder_preserves_records_split_across_body_frames() {
        let records = decode_lines(
            &mut Lines::new(1024),
            &[
                br#"{"text":"fir"#,
                b"st\"}\n{\"text\":\"second\"}\r",
                b"\n{\"text\":\"third\"}",
            ],
        )
        .unwrap();

        assert_eq!(
            records,
            [
                Bytes::from_static(br#"{"text":"first"}"#),
                Bytes::from_static(br#"{"text":"second"}"#),
                Bytes::from_static(br#"{"text":"third"}"#),
            ]
        );
    }

    #[test]
    fn decoders_enforce_the_item_limit() {
        assert_eq!(
            decode_document(&mut Document::new(3), &[b"123"]).unwrap(),
            "123"
        );
        assert!(
            decode_document(&mut Document::new(2), &[b"123"])
                .unwrap_err()
                .is_decode()
        );
        assert_eq!(
            decode_lines(&mut Lines::new(3), &[b"123\r\n"]).unwrap()[0],
            "123"
        );
        assert!(
            decode_lines(&mut Lines::new(2), &[b"123\n"])
                .unwrap_err()
                .is_decode()
        );
    }

    #[tokio::test]
    async fn encoder_serializes_one_value_per_body_frame() {
        let stream = futures_util::stream::iter([
            Ok::<_, Infallible>(serde_json::json!({ "id": 1 })),
            Ok(serde_json::json!({ "id": 2 })),
        ]);
        let frames = encode(stream, 1024).collect().await.unwrap().to_bytes();

        assert_eq!(frames, "{\"id\":1}\n{\"id\":2}\n");
    }

    #[tokio::test]
    async fn array_encoder_frames_a_valid_document_without_collecting_the_sequence() {
        let stream = futures_util::stream::iter([
            Ok::<_, Infallible>(serde_json::json!({ "id": 1 })),
            Ok(serde_json::json!({ "id": 2 })),
        ]);
        let frames = encode_array(stream, 1024)
            .collect()
            .await
            .unwrap()
            .to_bytes();

        assert_eq!(frames, "[{\"id\":1},{\"id\":2}]");
    }

    #[tokio::test]
    async fn encoders_reject_one_value_beyond_the_item_limit() {
        let record = futures_util::stream::iter([Ok::<_, Infallible>("1234")]);
        assert!(
            encode(record, 3)
                .frame()
                .await
                .unwrap()
                .unwrap_err()
                .is_body()
        );

        let value = futures_util::stream::iter([Ok::<_, Infallible>("1234")]);
        let mut body = encode_array(value, 3);
        assert!(body.frame().await.unwrap().is_ok());
        assert!(body.frame().await.unwrap().unwrap_err().is_body());

        let record = futures_util::stream::iter([Ok::<_, Infallible>(1)]);
        assert!(
            encode(record, 1)
                .frame()
                .await
                .unwrap()
                .unwrap_err()
                .is_body()
        );
    }
}