use bytes::BytesMut;
use core::marker::PhantomData;
use tokio::io::{AsyncRead, AsyncReadExt as _};
use tokio_util::codec::{Decoder, FramedRead};
use crate::de::{ParserConfig, from_slice_with_config};
use crate::error::{Error, Result};
pub async fn from_async_reader<R, T>(reader: &mut R) -> Result<T>
where
R: AsyncRead + Unpin,
T: serde_core::de::DeserializeOwned + 'static,
{
from_async_reader_with_config(reader, &ParserConfig::default()).await
}
pub async fn from_async_reader_with_config<R, T>(reader: &mut R, config: &ParserConfig) -> Result<T>
where
R: AsyncRead + Unpin,
T: serde_core::de::DeserializeOwned + 'static,
{
let buf = drain_bounded(reader, config.max_document_length, "max_document_length").await?;
let buf = strip_bom_owned(buf);
from_slice_with_config(&buf, config)
}
pub async fn from_async_reader_multi<R, T>(reader: &mut R) -> Result<Vec<T>>
where
R: AsyncRead + Unpin,
T: serde_core::de::DeserializeOwned + 'static,
{
from_async_reader_multi_with_config(reader, &ParserConfig::default()).await
}
pub async fn from_async_reader_multi_with_config<R, T>(
reader: &mut R,
config: &ParserConfig,
) -> Result<Vec<T>>
where
R: AsyncRead + Unpin,
T: serde_core::de::DeserializeOwned + 'static,
{
let buf = drain_bounded(reader, config.max_stream_bytes, "max_stream_bytes").await?;
let buf = strip_bom_owned(buf);
let text = core::str::from_utf8(&buf)
.map_err(|e| Error::from(std::io::Error::new(std::io::ErrorKind::InvalidData, e)))?;
let docs = crate::doc_boundary::split_documents_checked(text, config.max_documents)?;
let mut results = Vec::with_capacity(docs.len());
for doc in docs {
results.push(crate::from_str_with_config::<T>(doc, config)?);
}
Ok(results)
}
async fn drain_bounded<R>(reader: &mut R, max_bytes: usize, limit_name: &str) -> Result<Vec<u8>>
where
R: AsyncRead + Unpin,
{
let probe_bytes = max_bytes.saturating_add(1);
let initial_capacity = max_bytes.min(16 * 1024);
let mut buf = Vec::with_capacity(initial_capacity);
let take = u64::try_from(probe_bytes).unwrap_or(u64::MAX);
let mut limited = reader.take(take);
let _ = limited.read_to_end(&mut buf).await.map_err(Error::from)?;
if buf.len() > max_bytes {
return Err(Error::from(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!(
"noyalib async reader: input {} > {limit_name} {}",
buf.len(),
max_bytes
),
)));
}
Ok(buf)
}
fn strip_bom_owned(mut buf: Vec<u8>) -> Vec<u8> {
if crate::doc_boundary::strip_bom(&buf) == 3 {
let _ = buf.drain(..3);
}
buf
}
#[derive(Debug, Clone)]
pub struct YamlDecoder<T> {
config: ParserConfig,
max_frame_size: Option<usize>,
_marker: PhantomData<fn() -> T>,
}
pub type AsyncYamlStream<R, T> = FramedRead<R, YamlDecoder<T>>;
#[must_use]
pub fn async_yaml_stream<R, T>(reader: R) -> AsyncYamlStream<R, T> {
FramedRead::new(reader, YamlDecoder::new())
}
#[must_use]
pub fn async_yaml_stream_with_config<R, T>(
reader: R,
config: ParserConfig,
) -> AsyncYamlStream<R, T> {
FramedRead::new(reader, YamlDecoder::with_config(config))
}
impl<T> Default for YamlDecoder<T> {
fn default() -> Self {
Self::new()
}
}
impl<T> YamlDecoder<T> {
#[must_use]
pub fn new() -> Self {
let config = ParserConfig::default();
let max_frame_size = Some(config.max_document_length);
Self {
config,
max_frame_size,
_marker: PhantomData,
}
}
#[must_use]
pub fn with_config(config: ParserConfig) -> Self {
let max_frame_size = Some(config.max_document_length);
Self {
config,
max_frame_size,
_marker: PhantomData,
}
}
#[must_use]
pub fn max_frame_size(mut self, max: usize) -> Self {
self.max_frame_size = Some(max);
self
}
}
impl<T> Decoder for YamlDecoder<T>
where
T: serde_core::de::DeserializeOwned + 'static,
{
type Item = T;
type Error = Error;
fn decode(&mut self, src: &mut BytesMut) -> core::result::Result<Option<T>, Error> {
loop {
let bytes: &[u8] = src.as_ref();
let boundary = find_doc_boundary(bytes);
if let Some(max) = self.max_frame_size {
let frame_len = boundary.unwrap_or(bytes.len());
if frame_len > max {
return Err(Error::from(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("noyalib YamlDecoder: frame {frame_len} > max_frame_size {max}"),
)));
}
}
let Some(end) = boundary else {
return Ok(None);
};
let doc = src.split_to(end);
if doc.iter().all(u8::is_ascii_whitespace) {
continue;
}
let parsed = from_slice_with_config::<T>(&doc, &self.config)?;
return Ok(Some(parsed));
}
}
fn decode_eof(&mut self, src: &mut BytesMut) -> core::result::Result<Option<T>, Error> {
if src.is_empty() {
return Ok(None);
}
if let Some(v) = self.decode(src)? {
return Ok(Some(v));
}
if src.iter().all(u8::is_ascii_whitespace) {
src.clear();
return Ok(None);
}
let doc = src.split();
let parsed = from_slice_with_config::<T>(&doc, &self.config)?;
Ok(Some(parsed))
}
}
fn find_doc_boundary(bytes: &[u8]) -> Option<usize> {
crate::doc_boundary::next_marker_after(bytes, 0)
}
#[cfg(test)]
mod tests {
use super::*;
use bytes::BytesMut;
use tokio::io::BufReader;
#[derive(Debug, serde::Deserialize, PartialEq)]
struct Pkg {
name: String,
version: String,
}
#[tokio::test]
async fn reader_parses_single_document() {
let mut r = BufReader::new(&b"name: noyalib\nversion: 0.0.6\n"[..]);
let p: Pkg = from_async_reader(&mut r).await.unwrap();
assert_eq!(
p,
Pkg {
name: "noyalib".into(),
version: "0.0.6".into(),
}
);
}
#[tokio::test]
async fn reader_multi_parses_each_document() {
let yaml = b"---\nname: a\nversion: '1'\n---\nname: b\nversion: '2'\n";
let mut r = BufReader::new(&yaml[..]);
let docs: Vec<Pkg> = from_async_reader_multi(&mut r).await.unwrap();
assert_eq!(docs.len(), 2);
assert_eq!(docs[0].name, "a");
assert_eq!(docs[1].name, "b");
}
#[test]
fn decoder_emits_first_complete_document() {
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::new();
let mut buf = BytesMut::from(&b"name: a\nversion: '1'\n---\nname: b\nversion: '2'\n"[..]);
let first = decoder.decode(&mut buf).unwrap().unwrap();
assert_eq!(first.name, "a");
let second = decoder.decode_eof(&mut buf).unwrap().unwrap();
assert_eq!(second.name, "b");
}
#[test]
fn decoder_returns_none_on_incomplete_buffer() {
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::new();
let mut buf = BytesMut::from(&b"name: a\n"[..]);
assert!(decoder.decode(&mut buf).unwrap().is_none());
}
#[tokio::test]
async fn reader_with_config_respects_overrides() {
let mut r = BufReader::new(&b"name: x\nversion: '1'\n"[..]);
let cfg = ParserConfig::default();
let p: Pkg = from_async_reader_with_config(&mut r, &cfg).await.unwrap();
assert_eq!(p.name, "x");
}
#[test]
fn decoder_with_config_constructor() {
let cfg = ParserConfig::default();
let _d: YamlDecoder<Pkg> = YamlDecoder::with_config(cfg);
let _d2: YamlDecoder<Pkg> = YamlDecoder::default();
let _printed = format!("{:?}", YamlDecoder::<Pkg>::new());
}
#[test]
fn decoder_eof_on_empty_buffer_returns_none() {
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::new();
let mut buf = BytesMut::new();
assert!(decoder.decode_eof(&mut buf).unwrap().is_none());
}
#[test]
fn decoder_skips_whitespace_only_preamble() {
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::new();
let mut buf = BytesMut::from(&b"\n\n---\nname: q\nversion: '2'\n"[..]);
let p = decoder.decode_eof(&mut buf).unwrap().unwrap();
assert_eq!(p.name, "q");
}
#[test]
fn decoder_eof_drains_trailing_whitespace() {
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::new();
let mut buf = BytesMut::from(&b"name: r\nversion: '3'\n\n\n"[..]);
let p = decoder.decode_eof(&mut buf).unwrap().unwrap();
assert_eq!(p.name, "r");
}
#[tokio::test]
async fn reader_multi_handles_invalid_utf8() {
let mut r = BufReader::new(&[0xFFu8, 0xFE, 0xFD][..]);
let res: Result<Vec<Pkg>> = from_async_reader_multi(&mut r).await;
assert!(res.is_err());
}
#[test]
fn find_doc_boundary_handles_short_input() {
assert!(find_doc_boundary(b"").is_none());
assert!(find_doc_boundary(b"abc").is_none());
}
#[tokio::test]
async fn reader_strips_leading_bom() {
let mut r = BufReader::new(&b"\xEF\xBB\xBFname: x\nversion: '1'\n"[..]);
let p: Pkg = from_async_reader(&mut r).await.unwrap();
assert_eq!(p.name, "x");
}
#[tokio::test]
async fn reader_multi_accepts_crlf() {
let yaml = b"---\r\nname: a\r\nversion: '1'\r\n---\r\nname: b\r\nversion: '2'\r\n";
let mut r = BufReader::new(&yaml[..]);
let docs: Vec<Pkg> = from_async_reader_multi(&mut r).await.unwrap();
assert_eq!(docs.len(), 2);
assert_eq!(docs[0].name, "a");
assert_eq!(docs[1].name, "b");
}
#[tokio::test]
async fn reader_caps_at_max_document_length() {
let valid_prefix = "name: x\nversion: '1'\n";
let yaml = format!("{valid_prefix}ignored: true\n");
let cfg = ParserConfig {
max_document_length: valid_prefix.len(),
..ParserConfig::default()
};
let mut r = BufReader::new(yaml.as_bytes());
let err = from_async_reader_with_config::<_, Pkg>(&mut r, &cfg)
.await
.expect_err("over-limit input must be rejected, not truncated");
assert!(err.to_string().contains("max_document_length"));
}
#[tokio::test]
async fn reader_accepts_input_exactly_at_limit() {
let yaml = "name: x\nversion: '1'\n";
let cfg = ParserConfig {
max_document_length: yaml.len(),
..ParserConfig::default()
};
let mut r = BufReader::new(yaml.as_bytes());
let pkg = from_async_reader_with_config::<_, Pkg>(&mut r, &cfg)
.await
.unwrap();
assert_eq!(pkg.name, "x");
}
#[tokio::test]
async fn reader_zero_limit_rejects_nonempty_input() {
let cfg = ParserConfig {
max_document_length: 0,
..ParserConfig::default()
};
let mut r = BufReader::new(&b"null\n"[..]);
let err = from_async_reader_with_config::<_, crate::Value>(&mut r, &cfg)
.await
.expect_err("a zero limit must reject nonempty input");
assert!(err.to_string().contains("max_document_length 0"));
}
#[test]
fn decoder_rejects_oversize_frame() {
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::new().max_frame_size(16);
let mut buf = BytesMut::from(&b"name: long-name-no-marker-yet-need-more-bytes"[..]);
let err = decoder.decode(&mut buf).err().unwrap();
assert!(err.to_string().contains("max_frame_size"));
}
#[test]
fn decoder_accepts_multiple_buffered_documents_beyond_frame_cap() {
let cfg = ParserConfig {
max_document_length: 28,
..ParserConfig::default()
};
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::with_config(cfg);
let mut buf = BytesMut::from(&b"name: a\nversion: '1'\n---\nname: b\nversion: '2'\n"[..]);
assert!(buf.len() > 28);
let first = decoder.decode(&mut buf).unwrap().unwrap();
assert_eq!(first.name, "a");
let second = decoder.decode_eof(&mut buf).unwrap().unwrap();
assert_eq!(second.name, "b");
}
#[test]
fn stream_constructors_preserve_decoder_configuration() {
let default_stream = async_yaml_stream::<_, Pkg>(&b""[..]);
assert_eq!(
default_stream.decoder().max_frame_size,
Some(ParserConfig::default().max_document_length)
);
let cfg = ParserConfig {
max_document_length: 17,
..ParserConfig::default()
};
let configured_stream = async_yaml_stream_with_config::<_, Pkg>(&b""[..], cfg);
assert_eq!(configured_stream.decoder().max_frame_size, Some(17));
}
#[test]
fn decoder_derives_frame_cap_from_parser_config() {
let cfg = ParserConfig {
max_document_length: 16,
..ParserConfig::default()
};
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::with_config(cfg);
let mut buf = BytesMut::from(&b"name: long-name-without-boundary"[..]);
let err = decoder.decode(&mut buf).unwrap_err();
assert!(err.to_string().contains("max_frame_size 16"));
}
#[test]
fn decoder_eof_preserves_parser_config() {
let cfg = ParserConfig::strict();
let mut decoder: YamlDecoder<crate::Value> = YamlDecoder::with_config(cfg);
let mut buf = BytesMut::from(&b"a: 1\na: 2\n"[..]);
let err = decoder
.decode_eof(&mut buf)
.expect_err("strict duplicate-key policy must apply at EOF");
assert!(matches!(
err.kind(),
crate::error::ErrorKind::DuplicateKey | crate::error::ErrorKind::KeyCollision
));
}
#[test]
fn decoder_accepts_crlf_boundary() {
let mut decoder: YamlDecoder<Pkg> = YamlDecoder::new();
let mut buf =
BytesMut::from(&b"name: a\r\nversion: '1'\r\n---\r\nname: b\r\nversion: '2'\r\n"[..]);
let first = decoder.decode(&mut buf).unwrap().unwrap();
assert_eq!(first.name, "a");
}
#[tokio::test]
async fn reader_multi_with_config_routes_through() {
let yaml = b"---\nname: a\nversion: '1'\n---\nname: b\nversion: '2'\n";
let mut r = BufReader::new(&yaml[..]);
let cfg = ParserConfig::default();
let docs: Vec<Pkg> = from_async_reader_multi_with_config(&mut r, &cfg)
.await
.unwrap();
assert_eq!(docs.len(), 2);
}
#[tokio::test]
async fn multi_reader_separates_stream_and_document_byte_limits() {
let yaml = b"---\nname: a\nversion: '1'\n---\nname: b\nversion: '2'\n";
let cfg = ParserConfig::default()
.max_document_length(32)
.max_stream_bytes(yaml.len());
let mut reader = BufReader::new(&yaml[..]);
let docs: Vec<Pkg> = from_async_reader_multi_with_config(&mut reader, &cfg)
.await
.unwrap();
assert_eq!(docs.len(), 2);
}
#[tokio::test]
async fn multi_reader_rejects_stream_byte_overflow() {
let yaml = b"---\nname: a\nversion: '1'\n---\nname: b\nversion: '2'\n";
let cfg = ParserConfig::default().max_stream_bytes(yaml.len() - 1);
let mut reader = BufReader::new(&yaml[..]);
let error = from_async_reader_multi_with_config::<_, Pkg>(&mut reader, &cfg)
.await
.unwrap_err();
assert!(error.to_string().contains("max_stream_bytes"));
}
#[test]
fn find_doc_boundary_skips_leading_marker() {
assert!(find_doc_boundary(b"---\na: 1\n").is_none());
let bs = b"---\na: 1\n---\nb: 2\n";
let at = find_doc_boundary(bs).unwrap();
assert_eq!(at, 9);
}
}