1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
//! NDJSON (newline-delimited JSON) stream decoder.
use bytes::Bytes;
use futures::Stream;
use pin_project_lite::pin_project;
use serde::de::DeserializeOwned;
use std::pin::Pin;
use std::task::{Context, Poll};
use thiserror::Error;
/// Default upper bound on a single buffered NDJSON line's size (1 MiB).
const DEFAULT_MAX_LINE_SIZE: usize = 1024 * 1024;
#[derive(Debug, Error)]
pub enum NdjsonError {
#[error("utf-8 error: {0}")]
Utf8(#[from] std::string::FromUtf8Error),
#[error("json error: {0}")]
Json(#[from] serde_json::Error),
#[error("stream error: {0}")]
Stream(String),
#[error("NDJSON line exceeded maximum allowed size")]
LineTooLarge,
}
pin_project! {
/// Decodes a byte stream into parsed NDJSON values.
///
/// Bytes are accumulated in a raw `Vec<u8>` buffer and split on `\n`. Each
/// complete line is strictly UTF-8 decoded only *after* reassembly, so a
/// multi-byte character split across two network chunks is never corrupted
/// into replacement chars — unlike a per-chunk
/// `String::from_utf8_lossy` decode, which would emit a `U+FFFD` on each
/// side of the split.
pub struct NdjsonStream<S, T, E> {
#[pin]
inner: S,
buffer: Vec<u8>,
max_line_size: usize,
_marker: std::marker::PhantomData<(T, E)>,
}
}
impl<S, T, E> NdjsonStream<S, T, E>
where
S: Stream<Item = Result<Bytes, E>> + Unpin,
T: DeserializeOwned,
{
pub fn new(stream: S) -> Self {
Self::with_max_line_size(stream, DEFAULT_MAX_LINE_SIZE)
}
/// Create an [`NdjsonStream`] with a custom per-line size limit. A line
/// longer than `max_line_size` bytes (excluding the `\n`) yields
/// [`NdjsonError::LineTooLarge`], as does a buffer that grows past the
/// limit while waiting for a newline.
pub fn with_max_line_size(stream: S, max_line_size: usize) -> Self {
Self {
inner: stream,
buffer: Vec::new(),
max_line_size,
_marker: std::marker::PhantomData,
}
}
}
impl<S, T, E> Stream for NdjsonStream<S, T, E>
where
S: Stream<Item = Result<Bytes, E>> + Unpin,
T: DeserializeOwned,
E: std::fmt::Display,
{
type Item = Result<T, NdjsonError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.as_mut().get_mut();
loop {
// Try to parse a complete JSON line.
if let Some(pos) = this.buffer.iter().position(|&b| b == b'\n') {
if pos > this.max_line_size {
// Drop the oversized line so a retried poll makes progress.
this.buffer.drain(..pos + 1);
return Poll::Ready(Some(Err(NdjsonError::LineTooLarge)));
}
// Extract the line bytes (without the newline) and decode
// strictly. Decoding the fully reassembled line (rather than
// `from_utf8_lossy` per chunk) preserves code points split
// across chunks and surfaces invalid UTF-8 as an error.
let line_bytes: Vec<u8> = this.buffer.drain(..pos).collect();
this.buffer.drain(..1); // the newline
let line = match String::from_utf8(line_bytes) {
Ok(s) => s,
Err(e) => return Poll::Ready(Some(Err(NdjsonError::Utf8(e)))),
};
let line = line.trim();
if line.is_empty() {
continue;
}
match serde_json::from_str::<T>(line) {
Ok(value) => return Poll::Ready(Some(Ok(value))),
Err(e) => return Poll::Ready(Some(Err(NdjsonError::Json(e)))),
}
}
// No complete line yet. Guard against unbounded buffer growth when
// a newline never arrives. The partial can never form a valid line
// under the limit, so drop it — a retried poll then makes progress
// instead of looping on the same oversized buffer.
if this.buffer.len() > this.max_line_size {
this.buffer.clear();
return Poll::Ready(Some(Err(NdjsonError::LineTooLarge)));
}
// Read more bytes.
match Pin::new(&mut this.inner).poll_next(cx) {
Poll::Ready(Some(Ok(bytes))) => {
this.buffer.extend_from_slice(&bytes);
}
Poll::Ready(Some(Err(e))) => {
return Poll::Ready(Some(Err(NdjsonError::Stream(e.to_string()))));
}
Poll::Ready(None) => {
// Stream ended — try to parse any remaining buffer.
if this.buffer.len() > this.max_line_size {
return Poll::Ready(Some(Err(NdjsonError::LineTooLarge)));
}
let remaining_bytes = std::mem::take(&mut this.buffer);
let remaining = match String::from_utf8(remaining_bytes) {
Ok(s) => s,
Err(e) => return Poll::Ready(Some(Err(NdjsonError::Utf8(e)))),
};
let remaining = remaining.trim();
if remaining.is_empty() {
return Poll::Ready(None);
}
match serde_json::from_str::<T>(remaining) {
Ok(value) => return Poll::Ready(Some(Ok(value))),
Err(e) => return Poll::Ready(Some(Err(NdjsonError::Json(e)))),
}
}
Poll::Pending => return Poll::Pending,
}
}
}
}