jjpwrgem_parse/
tokens.rs

1pub(crate) mod lexical;
2mod number;
3mod stream;
4mod string;
5
6use core::{fmt::Display, iter::Peekable, range::Range};
7
8pub use lexical::{JsonByte, JsonChar};
9pub use stream::TokenStream;
10
11#[repr(u8)]
12#[derive(Debug, PartialEq, Eq, Clone, Copy)]
13pub enum Token {
14    OpenCurlyBrace,
15    ClosedCurlyBrace,
16    Colon,
17    Comma,
18    OpenSquareBracket,
19    ClosedSquareBracket,
20    String,
21    Mantissa,
22    Exponent,
23    Null,
24    True,
25    False,
26}
27
28impl Token {
29    pub(crate) fn is_start_of_value(self) -> bool {
30        matches!(
31            self,
32            Token::OpenCurlyBrace
33                | Token::OpenSquareBracket
34                | Token::String
35                | Token::Null
36                | Token::True
37                | Token::False
38                | Token::Mantissa
39        )
40    }
41
42    pub(crate) fn is_scalar(self) -> bool {
43        matches!(
44            self,
45            Token::String | Token::Null | Token::True | Token::False | Token::Mantissa
46        )
47    }
48}
49
50impl Display for Token {
51    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
52        let val = match self {
53            Token::OpenCurlyBrace => "{",
54            Token::ClosedCurlyBrace => "}",
55            Token::Colon => ":",
56            Token::Comma => ",",
57            Token::OpenSquareBracket => "[",
58            Token::ClosedSquareBracket => "]",
59            Token::String => "string",
60            Token::Mantissa => "mantissa",
61            Token::Exponent => "exponent",
62            Token::Null => NULL,
63            Token::True => TRUE,
64            Token::False => FALSE,
65        };
66        write!(f, "`{val}`")
67    }
68}
69
70const NO_SIGNIFICANT_CHARACTERS: &str = "no significant characters";
71
72#[derive(Debug, PartialEq, Eq, Clone)]
73pub struct ErrorToken {
74    pub(crate) tag: Token,
75    content: Box<str>,
76}
77
78impl ErrorToken {
79    pub(crate) fn new(tag: Token, range: Range<usize>, source: &str) -> Self {
80        Self {
81            tag,
82            content: source[range].into(),
83        }
84    }
85
86    pub fn content(&self) -> &str {
87        &self.content
88    }
89}
90
91impl Display for ErrorToken {
92    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
93        match self.tag {
94            Token::String | Token::Mantissa => write!(f, "`{}`", self.content),
95            Token::Exponent => write!(f, "`e{}`", self.content),
96            t => write!(f, "{t}"),
97        }
98    }
99}
100
101#[derive(Debug, PartialEq, Eq, Clone)]
102pub struct TokenOption(pub(crate) Option<ErrorToken>);
103
104impl Display for TokenOption {
105    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
106        match &self.0 {
107            Some(t) => write!(f, "{t}"),
108            None => write!(f, "{NO_SIGNIFICANT_CHARACTERS}"),
109        }
110    }
111}
112
113impl From<Option<Token>> for TokenOption {
114    fn from(value: Option<Token>) -> Self {
115        Self(value.map(|t| ErrorToken {
116            tag: t,
117            content: "".into(),
118        }))
119    }
120}
121
122#[derive(Debug, PartialEq, Eq, Clone, Copy)]
123pub struct JsonCharOption(pub Option<JsonChar>);
124
125impl Display for JsonCharOption {
126    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
127        let val = match &self.0 {
128            Some(x) => format!("`{x}`"),
129            None => NO_SIGNIFICANT_CHARACTERS.to_owned(),
130        };
131        write!(f, "{val}")
132    }
133}
134
135impl From<Option<JsonChar>> for JsonCharOption {
136    fn from(value: Option<JsonChar>) -> Self {
137        Self(value)
138    }
139}
140
141#[derive(Debug, Clone, Copy, PartialEq, Eq)]
142pub struct TokenWithContext {
143    pub token: Token,
144    pub range: Range<usize>,
145}
146
147impl TokenWithContext {
148    pub(crate) fn new(token: Token, range: Range<usize>) -> Self {
149        Self { token, range }
150    }
151
152    pub(crate) fn content_range(&self) -> Range<usize> {
153        match self.token {
154            Token::String => self.range.start + 1..self.range.end - 1,
155            _ => self.range,
156        }
157    }
158}
159
160pub(crate) const NULL: &str = "null";
161pub(crate) const FALSE: &str = "false";
162pub(crate) const TRUE: &str = "true";
163
164#[derive(Debug, Clone, Copy, PartialEq, Eq)]
165pub struct CharWithContext(pub Range<usize>, pub JsonChar);
166
167impl From<(usize, char)> for CharWithContext {
168    fn from((i, c): (usize, char)) -> Self {
169        Self(i..i + c.len_utf8(), c.into())
170    }
171}
172
173#[derive(Debug, Clone, Copy, PartialEq, Eq)]
174pub struct ByteWithContext(pub usize, pub JsonByte);
175
176impl From<(usize, u8)> for ByteWithContext {
177    fn from((i, b): (usize, u8)) -> Self {
178        Self(i, b.into())
179    }
180}
181
182impl ByteWithContext {
183    pub(crate) fn range(&self) -> Range<usize> {
184        self.0..self.0 + 1
185    }
186
187    pub(crate) fn as_token_with_context(&self) -> Option<TokenWithContext> {
188        Some(TokenWithContext::new(self.1.as_token()?, self.range()))
189    }
190
191    pub(crate) fn as_byte(&self) -> u8 {
192        self.1.0
193    }
194}
195
196#[derive(Debug, Clone)]
197pub(crate) struct BytesWithContext<'a> {
198    input: &'a str,
199    pos: usize,
200}
201
202impl<'a> BytesWithContext<'a> {
203    pub(crate) fn new(input: &'a str, pos: usize) -> Self {
204        Self { input, pos }
205    }
206}
207
208impl Iterator for BytesWithContext<'_> {
209    type Item = ByteWithContext;
210
211    fn next(&mut self) -> Option<Self::Item> {
212        self.input.as_bytes().get(self.pos).copied().map(|byte| {
213            let start = self.pos;
214            self.pos += 1;
215            (start, byte).into()
216        })
217    }
218}
219
220pub(crate) fn current_byte_pos(
221    bytes: &mut Peekable<impl Iterator<Item = ByteWithContext>>,
222    input: &str,
223) -> usize {
224    bytes
225        .peek()
226        .map_or(input.len(), |ByteWithContext(start, _)| *start)
227}
228
229impl From<bool> for Token {
230    fn from(value: bool) -> Self {
231        if value { Token::True } else { Token::False }
232    }
233}
234
235#[cfg(test)]
236mod tests {
237    use super::*;
238    use crate::{Error, ErrorKind, Result};
239
240    fn str_to_tokens(s: &str) -> Result<Vec<TokenWithContext>> {
241        stream::TokenStream::new(s).collect()
242    }
243
244    fn ctx(token: Token, range: Range<usize>) -> TokenWithContext {
245        TokenWithContext::new(token, range)
246    }
247
248    #[test]
249    fn should_parse_single_key_object() {
250        assert_eq!(
251            str_to_tokens(r#"{"rust": "is a must"}"#).unwrap(),
252            [
253                ctx(Token::OpenCurlyBrace, 0..1),
254                ctx(Token::String, 1..7),
255                ctx(Token::Colon, 7..8),
256                ctx(Token::String, 9..20),
257                ctx(Token::ClosedCurlyBrace, 20..21),
258            ]
259        );
260    }
261
262    #[rstest_reuse::template]
263    #[rstest::rstest]
264    #[case("null", Token::Null)]
265    #[case("true", Token::True)]
266    #[case("false", Token::False)]
267    #[case("\"burger\"", Token::String)]
268    #[case(r#""\"burger\"""#, Token::String)]
269    fn primitive_template(#[case] json: &str, #[case] expected: Token) {}
270
271    #[rstest_reuse::apply(primitive_template)]
272    fn primitives(#[case] json: &str, #[case] expected: Token) {
273        assert_eq!(str_to_tokens(json), Ok(vec![ctx(expected, 0..json.len())]));
274    }
275
276    #[rstest_reuse::apply(primitive_template)]
277    fn primitive_object_value(#[case] primitive: &str, #[case] expected: Token) {
278        let json = format!(
279            r#"{{
280                "rust": {primitive}
281            }}"#
282        );
283        assert_eq!(
284            str_to_tokens(&json).unwrap(),
285            [
286                ctx(Token::OpenCurlyBrace, 0..1),
287                ctx(Token::String, 18..24),
288                ctx(Token::Colon, 24..25),
289                ctx(expected, 26..(26 + primitive.len())),
290                ctx(Token::ClosedCurlyBrace, (json.len() - 1)..json.len()),
291            ]
292        );
293    }
294
295    fn json_to_json_and_error(
296        json: &'static str,
297        kind: ErrorKind,
298        range: Option<Range<usize>>,
299    ) -> (&'static str, Error) {
300        let error = match range {
301            Some(range) => Error::new(kind, range, json),
302            None => Error::from_unterminated(kind, json),
303        };
304        (json, error)
305    }
306
307    #[rstest::rstest]
308    #[case(json_to_json_and_error(
309        "a",
310        ErrorKind::UnexpectedCharacter('a'.into()),
311        Some(0..1)
312    ))]
313    #[case(json_to_json_and_error(
314        "n",
315        ErrorKind::UnexpectedCharacter('n'.into()),
316        Some(0..1)
317    ))]
318    #[case(json_to_json_and_error(
319        r#""
320    
321    ""#,
322        ErrorKind::UnexpectedControlCharacterInString('\n'.into()),
323        Some(1..2)
324    ))]
325    fn should_not_parse_invalid_syntax(#[case] (json, error): (&str, Error)) {
326        assert_eq!(str_to_tokens(json), Err(error));
327    }
328
329    #[test]
330    fn multiple_keys() {
331        assert_eq!(
332            str_to_tokens(
333                r#"{
334                "rust": "is a must",
335                "name": "ferris"
336            }"#
337            )
338            .unwrap(),
339            [
340                ctx(Token::OpenCurlyBrace, 0..1),
341                ctx(Token::String, 18..24),
342                ctx(Token::Colon, 24..25),
343                ctx(Token::String, 26..37),
344                ctx(Token::Comma, 37..38),
345                ctx(Token::String, 55..61),
346                ctx(Token::Colon, 61..62),
347                ctx(Token::String, 63..71),
348                ctx(Token::ClosedCurlyBrace, 84..85),
349            ]
350        );
351    }
352
353    #[test]
354    fn array_brackets() {
355        assert_eq!(
356            str_to_tokens("[]").unwrap(),
357            [
358                ctx(Token::OpenSquareBracket, 0..1),
359                ctx(Token::ClosedSquareBracket, 1..2),
360            ]
361        );
362    }
363
364    #[test]
365    fn token_with_context_size() {
366        assert_eq!(core::mem::size_of::<Token>(), 1);
367        assert_eq!(core::mem::size_of::<TokenWithContext>(), 24);
368    }
369}
jjpwrgem_parse/tokens.rs

jjpwrgem_parse/
tokens.rs
