Skip to main content

winnow_tokio/
lib.rs

1#![doc = include_str!("../README.md")]
2
3use std::fmt;
4
5use tokio::io::{AsyncRead, AsyncReadExt};
6use winnow::error::ErrMode;
7pub use winnow::{ModalResult, Partial};
8
9/// Size of a single read from the underlying [`AsyncRead`].
10///
11/// The internal buffer can grow larger than this while one item spans
12/// multiple chunks. Its memory use is bounded by the longest item plus one
13/// chunk of read-ahead.
14pub const CHUNK_SIZE: usize = 8 * 1024;
15
16/// Errors reported by [`parse_stream`].
17#[derive(Debug)]
18pub enum Error<E> {
19    /// The underlying reader failed at the given byte offset.
20    Io {
21        /// Number of input bytes successfully consumed before the failure.
22        offset: usize,
23        /// The reader error.
24        source: std::io::Error,
25    },
26    /// The parser could not make valid progress.
27    Parse {
28        /// Byte offset at which parsing stopped.
29        offset: usize,
30        /// Human-readable parser diagnostic.
31        detail: String,
32    },
33    /// The item callback rejected an emitted item.
34    Callback(E),
35}
36
37impl<E: fmt::Display> fmt::Display for Error<E> {
38    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
39        match self {
40            Self::Io { offset, source } => {
41                write!(formatter, "I/O error at offset {offset}: {source}")
42            }
43            Self::Parse { offset, detail } => {
44                write!(formatter, "parse error at offset {offset}: {detail}")
45            }
46            Self::Callback(error) => write!(formatter, "item callback failed: {error}"),
47        }
48    }
49}
50
51impl<E: fmt::Debug + fmt::Display> std::error::Error for Error<E> {
52    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
53        match self {
54            Self::Io { source, .. } => Some(source),
55            Self::Parse { .. } | Self::Callback(_) => None,
56        }
57    }
58}
59
60/// Drive a winnow parser over a chunked [`AsyncRead`], invoking `on_item`
61/// for each item the parser yields.
62///
63/// The parser receives a [`Partial`] byte slice and must return
64/// [`ModalResult<Option<Item>>`]:
65///
66/// - `Ok(Some(item))` emits an item and must consume input.
67/// - `Ok(None)` consumes input without emitting an item, for example when
68///   skipping whitespace or comments.
69/// - `Err(ErrMode::Incomplete(_))` asks the feeder to read more input.
70/// - Any other parser error terminates the feeder with an offset-aware
71///   [`Error::Parse`].
72///
73/// At EOF, an incomplete parser result, a non-consuming successful result, or
74/// unconsumed trailing bytes is reported as a parse error. An empty input is
75/// valid and returns `Ok(())`.
76///
77/// The feeder drains consumed bytes after each round. Memory therefore scales
78/// with the longest item that has not completed, rather than with the total
79/// input size.
80///
81/// # Errors
82///
83/// Returns [`Error::Io`] when reading fails, [`Error::Parse`] when the parser
84/// cannot make progress or rejects the input, and [`Error::Callback`] when the
85/// item callback returns an error.
86pub async fn parse_stream<R, P, Item, F, Fut, E>(
87    reader: R,
88    mut parser: P,
89    mut on_item: F,
90) -> Result<(), Error<E>>
91where
92    R: AsyncRead + Unpin,
93    P: for<'a> FnMut(&mut Partial<&'a [u8]>) -> ModalResult<Option<Item>>,
94    F: FnMut(Item) -> Fut,
95    Fut: Future<Output = Result<(), E>>,
96    E: fmt::Debug + fmt::Display,
97{
98    let mut reader = reader;
99    let mut buffer: Vec<u8> = Vec::with_capacity(CHUNK_SIZE * 2);
100    let mut stream_offset: usize = 0;
101    let mut eof = false;
102
103    loop {
104        if !eof {
105            let start = buffer.len();
106            buffer.resize(start + CHUNK_SIZE, 0);
107            let read = reader
108                .read(&mut buffer[start..])
109                .await
110                .map_err(|source| Error::Io {
111                    offset: stream_offset,
112                    source,
113                })?;
114            buffer.truncate(start + read);
115            if read == 0 {
116                eof = true;
117            }
118        }
119
120        let mut consumed_this_round: usize = 0;
121        loop {
122            if consumed_this_round >= buffer.len() {
123                break;
124            }
125            let slice = &buffer[consumed_this_round..];
126            let mut partial = Partial::new(slice);
127            let before_len = slice.len();
128            match parser(&mut partial) {
129                Ok(item_opt) => {
130                    let consumed = before_len - partial.len();
131                    if consumed == 0 {
132                        return Err(Error::Parse {
133                            offset: stream_offset + consumed_this_round,
134                            detail: "parser returned Ok without consuming input".to_owned(),
135                        });
136                    }
137                    consumed_this_round += consumed;
138                    if let Some(item) = item_opt {
139                        on_item(item).await.map_err(Error::Callback)?;
140                    }
141                }
142                Err(ErrMode::Incomplete(_)) => {
143                    if eof {
144                        return Err(Error::Parse {
145                            offset: stream_offset + consumed_this_round,
146                            detail: format!(
147                                "unexpected end of input: parser needs more bytes (remaining {} bytes)",
148                                slice.len()
149                            ),
150                        });
151                    }
152                    break;
153                }
154                Err(ErrMode::Backtrack(error) | ErrMode::Cut(error)) => {
155                    return Err(Error::Parse {
156                        offset: stream_offset + consumed_this_round,
157                        detail: render_context_error(&error),
158                    });
159                }
160            }
161        }
162
163        if consumed_this_round > 0 {
164            buffer.drain(..consumed_this_round);
165            stream_offset += consumed_this_round;
166        }
167
168        if eof {
169            if buffer.is_empty() {
170                return Ok(());
171            }
172            if consumed_this_round == 0 {
173                return Err(Error::Parse {
174                    offset: stream_offset,
175                    detail: format!(
176                        "trailing bytes at end of input: {} bytes remain",
177                        buffer.len()
178                    ),
179                });
180            }
181        }
182    }
183}
184
185fn render_context_error(error: &winnow::error::ContextError) -> String {
186    error.to_string().replace('\n', "; ")
187}
188
189#[cfg(test)]
190mod tests {
191    use std::io::Cursor;
192
193    use winnow::Parser;
194    use winnow::ascii::digit1;
195    use winnow::error::ContextError;
196    use winnow::token::{one_of, take_till};
197
198    use super::*;
199
200    fn int_line(input: &mut Partial<&[u8]>) -> ModalResult<Option<Vec<u8>>> {
201        let digits: &[u8] = digit1::<_, ErrMode<ContextError>>.parse_next(input)?;
202        let digits = digits.to_vec();
203        b'\n'.parse_next(input)?;
204        Ok(Some(digits))
205    }
206
207    fn comment_or_int_line(input: &mut Partial<&[u8]>) -> ModalResult<Option<Vec<u8>>> {
208        let slice: &[u8] = input;
209        if let Some(&b'#') = slice.first() {
210            one_of::<_, _, ErrMode<ContextError>>(b"#").parse_next(input)?;
211            take_till::<_, _, ErrMode<ContextError>>(0.., b'\n').parse_next(input)?;
212            b'\n'.parse_next(input)?;
213            return Ok(None);
214        }
215        int_line(input)
216    }
217
218    #[tokio::test]
219    async fn emits_items_across_chunk_boundaries() {
220        let input = b"12\n34\n567\n8\n".to_vec();
221        let mut items = Vec::new();
222        let reader = ChunkedReader::new(input, 3);
223        parse_stream(reader, int_line, |item| {
224            items.push(item);
225            async { Ok::<(), String>(()) }
226        })
227        .await
228        .unwrap();
229        assert_eq!(
230            items,
231            vec![
232                b"12".to_vec(),
233                b"34".to_vec(),
234                b"567".to_vec(),
235                b"8".to_vec()
236            ]
237        );
238    }
239
240    #[tokio::test]
241    async fn reports_error_on_midstream_eof() {
242        let reader = Cursor::new(b"12\n3".to_vec());
243        let error = parse_stream(reader, int_line, |_item| async { Ok::<(), String>(()) })
244            .await
245            .unwrap_err();
246        assert!(error.to_string().contains("unexpected end of input"));
247    }
248
249    #[tokio::test]
250    async fn reports_syntax_error_with_offset() {
251        let reader = Cursor::new(b"12\nXX\n".to_vec());
252        let mut items = Vec::new();
253        let error = parse_stream(reader, int_line, |item| {
254            items.push(item);
255            async { Ok::<(), String>(()) }
256        })
257        .await
258        .unwrap_err();
259        assert!(error.to_string().contains("offset 3"));
260        assert_eq!(items, vec![b"12"]);
261    }
262
263    #[tokio::test]
264    async fn none_items_consume_without_emit() {
265        let input = b"# top comment\n42\n# another\n7\n".to_vec();
266        let mut items = Vec::new();
267        let reader = ChunkedReader::new(input, 5);
268        parse_stream(reader, comment_or_int_line, |item| {
269            items.push(item);
270            async { Ok::<(), String>(()) }
271        })
272        .await
273        .unwrap();
274        assert_eq!(items, vec![b"42".to_vec(), b"7".to_vec()]);
275    }
276
277    #[tokio::test]
278    async fn empty_input_is_ok() {
279        let mut items = Vec::new();
280        parse_stream(&[][..], int_line, |item| {
281            items.push(item);
282            async { Ok::<(), String>(()) }
283        })
284        .await
285        .unwrap();
286        assert_eq!(items, Vec::<Vec<u8>>::new());
287    }
288
289    #[tokio::test]
290    async fn callback_error_is_preserved() {
291        let reader = Cursor::new(b"1\n2\n".to_vec());
292        let mut count = 0;
293        let error = parse_stream(reader, int_line, |_item| {
294            count += 1;
295            let result = if count == 2 {
296                Err("downstream rejected item".to_owned())
297            } else {
298                Ok(())
299            };
300            async move { result }
301        })
302        .await
303        .unwrap_err();
304        assert!(matches!(error, Error::Callback(message) if message == "downstream rejected item"));
305    }
306
307    #[tokio::test]
308    async fn items_larger_than_chunk_still_parse() {
309        let line: Vec<u8> = (0..100).map(|_| b'5').collect();
310        let mut input = Vec::new();
311        input.extend_from_slice(&line);
312        input.push(b'\n');
313        input.extend_from_slice(&line);
314        input.push(b'\n');
315        let reader = ChunkedReader::new(input, 7);
316        let mut items = Vec::new();
317        parse_stream(reader, int_line, |item| {
318            items.push(item);
319            async { Ok::<(), String>(()) }
320        })
321        .await
322        .unwrap();
323        assert_eq!(items.len(), 2);
324        assert_eq!(items[0].len(), 100);
325        assert_eq!(items[1].len(), 100);
326    }
327
328    struct ChunkedReader {
329        data: Vec<u8>,
330        position: usize,
331        chunk: usize,
332    }
333
334    impl ChunkedReader {
335        fn new(data: Vec<u8>, chunk: usize) -> Self {
336            Self {
337                data,
338                position: 0,
339                chunk,
340            }
341        }
342    }
343
344    impl tokio::io::AsyncRead for ChunkedReader {
345        fn poll_read(
346            mut self: std::pin::Pin<&mut Self>,
347            _context: &mut std::task::Context<'_>,
348            buffer: &mut tokio::io::ReadBuf<'_>,
349        ) -> std::task::Poll<std::io::Result<()>> {
350            let remaining = self.data.len() - self.position;
351            if remaining == 0 {
352                return std::task::Poll::Ready(Ok(()));
353            }
354            let amount = buffer.remaining().min(self.chunk).min(remaining);
355            let position = self.position;
356            buffer.put_slice(&self.data[position..position + amount]);
357            self.position += amount;
358            std::task::Poll::Ready(Ok(()))
359        }
360    }
361}