noa_parser/bytes/
token.rs

1//! Classic tokens
2
3use crate::bytes::matchers::{match_char, match_pattern};
4use crate::matcher::{Match, MatchSize};
5
6#[derive(Copy, Clone)]
7/// The token type
8#[derive(PartialEq, Debug)]
9pub enum Token {
10    /// The "(" character
11    OpenParen,
12    /// The `)` character
13    CloseParen,
14    /// The `,` character
15    Comma,
16    /// The `;` character
17    Semicolon,
18    /// The `:` character
19    Colon,
20    /// The whitespace character
21    Whitespace,
22    /// The `>` character
23    GreaterThan,
24    /// The `<` character
25    LessThan,
26    /// The `!` character
27    Exclamation,
28    /// The `'` character
29    Quote,
30    /// The `"` character
31    DoubleQuote,
32    /// The `=` character
33    Equal,
34    /// The `+` character
35    Plus,
36    /// The `-` character
37    Dash,
38    /// The `/` character
39    Slash,
40    /// The `*` character
41    Star,
42    /// The `%` character
43    Percent,
44    /// The `&` character
45    Ampersand,
46    /// The `|` character
47    Pipe,
48    /// The `^` character
49    Caret,
50    /// The `~` character
51    Tilde,
52    /// The `.` character
53    Dot,
54    /// The `?` character
55    Question,
56    /// The `@` character
57    At,
58    /// The `#` character
59    Hash,
60    /// The `$` character
61    Dollar,
62    /// The `\\` character
63    Backslash,
64    /// The `_` character
65    Underscore,
66    /// The `#` character
67    Sharp,
68    /// The `\n` character
69    Ln,
70    /// The `\r` character
71    Cr,
72    /// The `\t` character
73    Tab,
74    /// The `\r\n` character
75    CrLn,
76}
77
78impl Match<u8> for Token {
79    fn matcher(&self, data: &[u8]) -> (bool, usize) {
80        match self {
81            Token::OpenParen => match_char('(', data),
82            Token::CloseParen => match_char(')', data),
83            Token::Comma => match_char(',', data),
84            Token::Semicolon => match_char(';', data),
85            Token::Colon => match_char(':', data),
86            Token::Whitespace => match_char(' ', data),
87            Token::GreaterThan => match_char('>', data),
88            Token::LessThan => match_char('<', data),
89            Token::Exclamation => match_char('!', data),
90            Token::Quote => match_char('\'', data),
91            Token::DoubleQuote => match_char('"', data),
92            Token::Equal => match_char('=', data),
93            Token::Plus => match_char('+', data),
94            Token::Dash => match_char('-', data),
95            Token::Slash => match_char('/', data),
96            Token::Star => match_char('*', data),
97            Token::Percent => match_char('%', data),
98            Token::Ampersand => match_char('&', data),
99            Token::Pipe => match_char('|', data),
100            Token::Caret => match_char('^', data),
101            Token::Tilde => match_char('~', data),
102            Token::Dot => match_char('.', data),
103            Token::Question => match_char('?', data),
104            Token::At => match_char('@', data),
105            Token::Hash => match_char('#', data),
106            Token::Dollar => match_char('$', data),
107            Token::Backslash => match_char('\\', data),
108            Token::Underscore => match_char('_', data),
109            Token::Sharp => match_char('#', data),
110            Token::Ln => match_char('\n', data),
111            Token::Cr => match_char('\r', data),
112            Token::Tab => match_char('\t', data),
113            Token::CrLn => match_pattern(b"\r\n", data),
114        }
115    }
116}
117
118impl MatchSize for Token {
119    fn size(&self) -> usize {
120        match self {
121            Token::OpenParen => 1,
122            Token::CloseParen => 1,
123            Token::Comma => 1,
124            Token::Semicolon => 1,
125            Token::Colon => 1,
126            Token::Whitespace => 1,
127            Token::GreaterThan => 1,
128            Token::LessThan => 1,
129            Token::Exclamation => 1,
130            Token::Quote => 1,
131            Token::DoubleQuote => 1,
132            Token::Equal => 1,
133            Token::Plus => 1,
134            Token::Dash => 1,
135            Token::Slash => 1,
136            Token::Star => 1,
137            Token::Percent => 1,
138            Token::Ampersand => 1,
139            Token::Pipe => 1,
140            Token::Caret => 1,
141            Token::Tilde => 1,
142            Token::Dot => 1,
143            Token::Question => 1,
144            Token::At => 1,
145            Token::Hash => 1,
146            Token::Dollar => 1,
147            Token::Backslash => 1,
148            Token::Underscore => 1,
149            Token::Sharp => 1,
150            Token::Ln => 1,
151            Token::Cr => 1,
152            Token::Tab => 1,
153            Token::CrLn => 2,
154        }
155    }
156}
noa_parser/bytes/token.rs

noa_parser/bytes/
token.rs
