yap_streaming/
lib.rs

1/*!
2This crate builds on the interfaces from [`yap`](https://crates.io/crates/yap) to allow simple parsing of streams.
3
4# Why
5
6There already exist [many](https://github.com/rosetta-rs/parse-rosetta-rs) crates that intend to help with parsing.
7Of that list `nom`, `winnow`, `chumsky`, `combine` support parsing streams of values.
8
9`nom`:
10- No obvious way to signal the end of a stream to a parser.
11- The user of the library has to implement a streaming parser noticeably differently from a non-streaming parser.
12- Parsing occurs on chunks. Parsing dynamically sized chunks can require re-parsing the chunk from scratch and redoing work.
13
14`winnow`:
15- Parsing occurs on chunks. Parsing dynamically sized chunks can require re-parsing the chunk from scratch and redoing work.
16
17`chumsky` is not designed for speed.
18
19`combine` is complicated.
20
21This crate allows using an already written [`yap`](https://crates.io/crates/yap) parser by simply changing the initial tokens declaration.
22
23```rust
24# #[cfg(feature = "alloc")] {
25use std::{
26    fs::File,
27    io::{self, BufReader, Read},
28};
29use yap_streaming::{
30    // Allows you to use `.into_tokens()` on strings and slices,
31    // to get an instance of the above:
32    IntoTokens,
33    // Allows you to get an instance of `Tokens` that supports streams:
34    StrStreamTokens,
35    // This trait has all of the parsing methods on it:
36    Tokens,
37};
38
39// Write parser
40// =========================================
41
42#[derive(PartialEq, Debug)]
43enum Op {
44    Plus,
45    Minus,
46    Multiply,
47}
48#[derive(PartialEq, Debug)]
49enum OpOrDigit {
50    Op(Op),
51    Digit(u32),
52}
53
54// The `Tokens` trait builds on `Iterator`, so we get a `next` method.
55fn parse_op(t: &mut impl Tokens<Item = char>) -> Option<Op> {
56    let loc = t.location();
57    match t.next()? {
58        '-' => Some(Op::Minus),
59        '+' => Some(Op::Plus),
60        'x' => Some(Op::Multiply),
61        _ => {
62            t.set_location(loc);
63            None
64        }
65    }
66}
67
68// We also get other useful functions..
69fn parse_digits(t: &mut impl Tokens<Item = char>) -> Option<u32> {
70    t.take_while(|c| c.is_digit(10)).parse::<u32, String>().ok()
71}
72
73fn parse_all(t: &mut impl Tokens<Item = char>) -> impl Tokens<Item = OpOrDigit> + '_ {
74    // As well as combinator functions like `sep_by_all` and `surrounded_by`..
75    t.sep_by_all(
76        |t| {
77            t.surrounded_by(
78                |t| parse_digits(t).map(OpOrDigit::Digit),
79                |t| {
80                    t.skip_while(|c| c.is_ascii_whitespace());
81                },
82            )
83        },
84        |t| parse_op(t).map(OpOrDigit::Op),
85    )
86}
87
88// Now we've parsed our input into OpOrDigits, let's calculate the result..
89fn eval(t: &mut impl Tokens<Item = char>) -> u32 {
90    let op_or_digit = parse_all(t);
91    let mut current_op = Op::Plus;
92    let mut current_digit = 0;
93    for d in op_or_digit.into_iter() {
94        match d {
95            OpOrDigit::Op(op) => current_op = op,
96            OpOrDigit::Digit(n) => match current_op {
97                Op::Plus => current_digit += n,
98                Op::Minus => current_digit -= n,
99                Op::Multiply => current_digit *= n,
100            },
101        }
102    }
103    current_digit
104}
105
106// Use parser
107// =========================================
108
109// Get our input and convert into something implementing `Tokens`
110let mut tokens = "10 + 2 x 12-4,foobar".into_tokens();
111// Parse
112assert_eq!(eval(&mut tokens), 140);
113
114// Instead of parsing an in-memory buffer we can use `yap_streaming` to parse a stream.
115// While we could [`std::io::Read::read_to_end()`] here, what if the file was too large
116// to fit in memory? What if we were parsing from a network socket?
117let mut io_err = None;
118let file_chars = BufReader::new(File::open("examples/opOrDigit.txt").expect("open file"))
119    .bytes()
120    .map_while(|x| {
121        match x {
122            Ok(x) => {
123                if x.is_ascii() {
124                    Some(x as char)
125                } else {
126                    io_err = Some(io::ErrorKind::InvalidData.into());
127                    // Don't parse any further if non-ascii input.
128                    // This simple example parser only makes sense with ascii values.
129                    None
130                }
131            }
132            Err(e) => {
133                io_err = Some(e);
134                // Don't parse any further if io error.
135                // Alternatively could panic, retry the byte,
136                // or include as an error variant and parse Result<char, ParseError> instead.
137                None
138            }
139        }
140    });
141
142// Convert to something implementing `Tokens`.
143// If parsing a stream not of `char` use [`yap_streaming::StreamTokens`] instead.
144let mut tokens = StrStreamTokens::new(file_chars);
145// Parse
146assert_eq!(eval(&mut tokens), 140);
147// Check that parse encountered no io errors.
148assert!(io_err.is_none());
149# }
150```
151*/
152#![deny(
153    missing_copy_implementations,
154    missing_debug_implementations,
155    missing_docs
156)]
157#![cfg_attr(not(feature = "std"), no_std)]
158#[cfg(feature = "alloc")]
159extern crate alloc;
160
161#[cfg(feature = "alloc")]
162mod stream_tokens;
163#[cfg(feature = "alloc")]
164pub use stream_tokens::{str_stream_tokens::StrStreamTokens, StreamTokens, StreamTokensLocation};
165pub use yap::{IntoTokens, TokenLocation, Tokens};
yap_streaming/lib.rs

yap_streaming/
lib.rs
