1#![doc = include_str!("../README.md")]
2
3use std::fmt;
4
5use tokio::io::{AsyncRead, AsyncReadExt};
6use winnow::error::ErrMode;
7pub use winnow::{ModalResult, Partial};
8
9pub const CHUNK_SIZE: usize = 8 * 1024;
15
16#[derive(Debug)]
18pub enum Error<E> {
19 Io {
21 offset: usize,
23 source: std::io::Error,
25 },
26 Parse {
28 offset: usize,
30 detail: String,
32 },
33 Callback(E),
35}
36
37impl<E: fmt::Display> fmt::Display for Error<E> {
38 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
39 match self {
40 Self::Io { offset, source } => {
41 write!(formatter, "I/O error at offset {offset}: {source}")
42 }
43 Self::Parse { offset, detail } => {
44 write!(formatter, "parse error at offset {offset}: {detail}")
45 }
46 Self::Callback(error) => write!(formatter, "item callback failed: {error}"),
47 }
48 }
49}
50
51impl<E: fmt::Debug + fmt::Display> std::error::Error for Error<E> {
52 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
53 match self {
54 Self::Io { source, .. } => Some(source),
55 Self::Parse { .. } | Self::Callback(_) => None,
56 }
57 }
58}
59
60pub async fn parse_stream<R, P, Item, F, Fut, E>(
87 reader: R,
88 mut parser: P,
89 mut on_item: F,
90) -> Result<(), Error<E>>
91where
92 R: AsyncRead + Unpin,
93 P: for<'a> FnMut(&mut Partial<&'a [u8]>) -> ModalResult<Option<Item>>,
94 F: FnMut(Item) -> Fut,
95 Fut: Future<Output = Result<(), E>>,
96 E: fmt::Debug + fmt::Display,
97{
98 let mut reader = reader;
99 let mut buffer: Vec<u8> = Vec::with_capacity(CHUNK_SIZE * 2);
100 let mut stream_offset: usize = 0;
101 let mut eof = false;
102
103 loop {
104 if !eof {
105 let start = buffer.len();
106 buffer.resize(start + CHUNK_SIZE, 0);
107 let read = reader
108 .read(&mut buffer[start..])
109 .await
110 .map_err(|source| Error::Io {
111 offset: stream_offset,
112 source,
113 })?;
114 buffer.truncate(start + read);
115 if read == 0 {
116 eof = true;
117 }
118 }
119
120 let mut consumed_this_round: usize = 0;
121 loop {
122 if consumed_this_round >= buffer.len() {
123 break;
124 }
125 let slice = &buffer[consumed_this_round..];
126 let mut partial = Partial::new(slice);
127 let before_len = slice.len();
128 match parser(&mut partial) {
129 Ok(item_opt) => {
130 let consumed = before_len - partial.len();
131 if consumed == 0 {
132 return Err(Error::Parse {
133 offset: stream_offset + consumed_this_round,
134 detail: "parser returned Ok without consuming input".to_owned(),
135 });
136 }
137 consumed_this_round += consumed;
138 if let Some(item) = item_opt {
139 on_item(item).await.map_err(Error::Callback)?;
140 }
141 }
142 Err(ErrMode::Incomplete(_)) => {
143 if eof {
144 return Err(Error::Parse {
145 offset: stream_offset + consumed_this_round,
146 detail: format!(
147 "unexpected end of input: parser needs more bytes (remaining {} bytes)",
148 slice.len()
149 ),
150 });
151 }
152 break;
153 }
154 Err(ErrMode::Backtrack(error) | ErrMode::Cut(error)) => {
155 return Err(Error::Parse {
156 offset: stream_offset + consumed_this_round,
157 detail: render_context_error(&error),
158 });
159 }
160 }
161 }
162
163 if consumed_this_round > 0 {
164 buffer.drain(..consumed_this_round);
165 stream_offset += consumed_this_round;
166 }
167
168 if eof {
169 if buffer.is_empty() {
170 return Ok(());
171 }
172 if consumed_this_round == 0 {
173 return Err(Error::Parse {
174 offset: stream_offset,
175 detail: format!(
176 "trailing bytes at end of input: {} bytes remain",
177 buffer.len()
178 ),
179 });
180 }
181 }
182 }
183}
184
185fn render_context_error(error: &winnow::error::ContextError) -> String {
186 error.to_string().replace('\n', "; ")
187}
188
189#[cfg(test)]
190mod tests {
191 use std::io::Cursor;
192
193 use winnow::Parser;
194 use winnow::ascii::digit1;
195 use winnow::error::ContextError;
196 use winnow::token::{one_of, take_till};
197
198 use super::*;
199
200 fn int_line(input: &mut Partial<&[u8]>) -> ModalResult<Option<Vec<u8>>> {
201 let digits: &[u8] = digit1::<_, ErrMode<ContextError>>.parse_next(input)?;
202 let digits = digits.to_vec();
203 b'\n'.parse_next(input)?;
204 Ok(Some(digits))
205 }
206
207 fn comment_or_int_line(input: &mut Partial<&[u8]>) -> ModalResult<Option<Vec<u8>>> {
208 let slice: &[u8] = input;
209 if let Some(&b'#') = slice.first() {
210 one_of::<_, _, ErrMode<ContextError>>(b"#").parse_next(input)?;
211 take_till::<_, _, ErrMode<ContextError>>(0.., b'\n').parse_next(input)?;
212 b'\n'.parse_next(input)?;
213 return Ok(None);
214 }
215 int_line(input)
216 }
217
218 #[tokio::test]
219 async fn emits_items_across_chunk_boundaries() {
220 let input = b"12\n34\n567\n8\n".to_vec();
221 let mut items = Vec::new();
222 let reader = ChunkedReader::new(input, 3);
223 parse_stream(reader, int_line, |item| {
224 items.push(item);
225 async { Ok::<(), String>(()) }
226 })
227 .await
228 .unwrap();
229 assert_eq!(
230 items,
231 vec![
232 b"12".to_vec(),
233 b"34".to_vec(),
234 b"567".to_vec(),
235 b"8".to_vec()
236 ]
237 );
238 }
239
240 #[tokio::test]
241 async fn reports_error_on_midstream_eof() {
242 let reader = Cursor::new(b"12\n3".to_vec());
243 let error = parse_stream(reader, int_line, |_item| async { Ok::<(), String>(()) })
244 .await
245 .unwrap_err();
246 assert!(error.to_string().contains("unexpected end of input"));
247 }
248
249 #[tokio::test]
250 async fn reports_syntax_error_with_offset() {
251 let reader = Cursor::new(b"12\nXX\n".to_vec());
252 let mut items = Vec::new();
253 let error = parse_stream(reader, int_line, |item| {
254 items.push(item);
255 async { Ok::<(), String>(()) }
256 })
257 .await
258 .unwrap_err();
259 assert!(error.to_string().contains("offset 3"));
260 assert_eq!(items, vec![b"12"]);
261 }
262
263 #[tokio::test]
264 async fn none_items_consume_without_emit() {
265 let input = b"# top comment\n42\n# another\n7\n".to_vec();
266 let mut items = Vec::new();
267 let reader = ChunkedReader::new(input, 5);
268 parse_stream(reader, comment_or_int_line, |item| {
269 items.push(item);
270 async { Ok::<(), String>(()) }
271 })
272 .await
273 .unwrap();
274 assert_eq!(items, vec![b"42".to_vec(), b"7".to_vec()]);
275 }
276
277 #[tokio::test]
278 async fn empty_input_is_ok() {
279 let mut items = Vec::new();
280 parse_stream(&[][..], int_line, |item| {
281 items.push(item);
282 async { Ok::<(), String>(()) }
283 })
284 .await
285 .unwrap();
286 assert_eq!(items, Vec::<Vec<u8>>::new());
287 }
288
289 #[tokio::test]
290 async fn callback_error_is_preserved() {
291 let reader = Cursor::new(b"1\n2\n".to_vec());
292 let mut count = 0;
293 let error = parse_stream(reader, int_line, |_item| {
294 count += 1;
295 let result = if count == 2 {
296 Err("downstream rejected item".to_owned())
297 } else {
298 Ok(())
299 };
300 async move { result }
301 })
302 .await
303 .unwrap_err();
304 assert!(matches!(error, Error::Callback(message) if message == "downstream rejected item"));
305 }
306
307 #[tokio::test]
308 async fn items_larger_than_chunk_still_parse() {
309 let line: Vec<u8> = (0..100).map(|_| b'5').collect();
310 let mut input = Vec::new();
311 input.extend_from_slice(&line);
312 input.push(b'\n');
313 input.extend_from_slice(&line);
314 input.push(b'\n');
315 let reader = ChunkedReader::new(input, 7);
316 let mut items = Vec::new();
317 parse_stream(reader, int_line, |item| {
318 items.push(item);
319 async { Ok::<(), String>(()) }
320 })
321 .await
322 .unwrap();
323 assert_eq!(items.len(), 2);
324 assert_eq!(items[0].len(), 100);
325 assert_eq!(items[1].len(), 100);
326 }
327
328 struct ChunkedReader {
329 data: Vec<u8>,
330 position: usize,
331 chunk: usize,
332 }
333
334 impl ChunkedReader {
335 fn new(data: Vec<u8>, chunk: usize) -> Self {
336 Self {
337 data,
338 position: 0,
339 chunk,
340 }
341 }
342 }
343
344 impl tokio::io::AsyncRead for ChunkedReader {
345 fn poll_read(
346 mut self: std::pin::Pin<&mut Self>,
347 _context: &mut std::task::Context<'_>,
348 buffer: &mut tokio::io::ReadBuf<'_>,
349 ) -> std::task::Poll<std::io::Result<()>> {
350 let remaining = self.data.len() - self.position;
351 if remaining == 0 {
352 return std::task::Poll::Ready(Ok(()));
353 }
354 let amount = buffer.remaining().min(self.chunk).min(remaining);
355 let position = self.position;
356 buffer.put_slice(&self.data[position..position + amount]);
357 self.position += amount;
358 std::task::Poll::Ready(Ok(()))
359 }
360 }
361}