musli_json/parser/
token.rs

1#![allow(clippy::identity_op, clippy::just_underscores_and_digits)]
2
3use core::fmt;
4
5/// Tokens.
6#[derive(Debug, Clone, Copy)]
7#[repr(u8)]
8pub enum Token {
9    /// `,`.
10    Comma = CM,
11    /// `:`.
12    Colon = CL,
13    /// Whitespace.
14    Whitespace = WS,
15    /// `{`.
16    OpenBrace = OB,
17    /// `}`.
18    CloseBrace = CB,
19    /// `[`.
20    OpenBracket = OA,
21    /// `]`.
22    CloseBracket = CA,
23    /// A string.
24    String = ST,
25    /// A simple number.
26    Number = NM,
27    /// `null` literal.
28    Null = NU,
29    /// `true` literal.
30    True = TR,
31    /// `false` literal.
32    False = FL,
33    /// Error.
34    Error = __,
35    /// End-of-file.
36    Eof = EF,
37}
38
39impl Token {
40    /// Construct a token from a single byte.
41    ///
42    /// Note that this should optimize into a no-op beyond the lookup into the
43    /// `MAP` table.
44    #[inline]
45    pub(crate) fn from_byte(b: u8) -> Token {
46        match MAP[b as usize] {
47            WS => Token::Whitespace,
48            OA => Token::OpenBrace,
49            CA => Token::CloseBrace,
50            OB => Token::OpenBracket,
51            CB => Token::CloseBracket,
52            CM => Token::Comma,
53            CL => Token::Colon,
54            ST => Token::String,
55            NM => Token::Number,
56            NU => Token::Null,
57            TR => Token::True,
58            FL => Token::False,
59            __ => Token::Error,
60            _ => unreachable!(),
61        }
62    }
63
64    #[inline]
65    pub(crate) fn is_value(&self) -> bool {
66        (*self as u8) & VAL_BIT != 0
67    }
68
69    #[inline]
70    pub(crate) fn is_null(&self) -> bool {
71        matches!(self, Token::Null)
72    }
73}
74
75impl fmt::Display for Token {
76    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
77        match self {
78            Token::Comma => write!(f, "`,`"),
79            Token::Colon => write!(f, "`:`"),
80            Token::Whitespace => write!(f, "<whitespace>"),
81            Token::OpenBrace => write!(f, "`{{`"),
82            Token::CloseBrace => write!(f, "`}}`"),
83            Token::OpenBracket => write!(f, "`[`"),
84            Token::CloseBracket => write!(f, "`]`"),
85            Token::String => write!(f, "`\"`"),
86            Token::Number => write!(f, "<number>"),
87            Token::Null => write!(f, "null"),
88            Token::True => write!(f, "true"),
89            Token::False => write!(f, "false"),
90            Token::Error => write!(f, "<error>"),
91            Token::Eof => write!(f, "<eof>"),
92        }
93    }
94}
95
96const VAL_BIT: u8 = 0b0000_0001;
97const CTL_BIT: u8 = 0b0000_0010;
98
99const WS: u8 = 0b0000_0000;
100const OA: u8 = 0b0001_0000 | VAL_BIT;
101const CA: u8 = 0b0010_0000;
102const OB: u8 = 0b0011_0000 | VAL_BIT;
103const CB: u8 = 0b0100_0000;
104const NM: u8 = 0b0101_0000 | VAL_BIT;
105const ST: u8 = 0b0111_0000 | VAL_BIT;
106const NU: u8 = 0b1000_0000 | VAL_BIT;
107const TR: u8 = 0b1001_0000 | VAL_BIT;
108const FL: u8 = 0b1010_0000 | VAL_BIT;
109const CM: u8 = 0b0000_0000 | CTL_BIT;
110const CL: u8 = 0b0001_0000 | CTL_BIT;
111const EF: u8 = 0b1110_0000 | CTL_BIT;
112const __: u8 = 0b1111_0000 | CTL_BIT;
113
114static MAP: [u8; 256] = [
115    //  1   2   3   4   5   6   7   8   9   a   b   c   d   e   f
116    __, __, __, __, __, __, __, __, __, WS, WS, __, WS, WS, __, __, // 0
117    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 1
118    WS, __, ST, __, __, __, __, __, __, __, __, __, CM, NM, __, __, // 2
119    NM, NM, NM, NM, NM, NM, NM, NM, NM, NM, CL, __, __, __, __, __, // 3
120    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 4
121    __, __, __, __, __, __, __, __, __, __, __, OB, __, CB, __, __, // 5
122    __, __, __, __, __, __, FL, __, __, __, __, __, __, __, NU, __, // 6
123    __, __, __, __, TR, __, __, __, __, __, __, OA, __, CA, __, __, // 7
124    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 8
125    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 9
126    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // a
127    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // b
128    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // c
129    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // d
130    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // e
131    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // f
132];
musli_json/parser/token.rs

musli_json/parser/
token.rs
