pomsky_syntax/lexer/
token.rs

1use super::LexErrorMsg;
2
3/// A token encountered while lexing a pomsky expression.
4#[derive(Debug, Eq, PartialEq, Copy, Clone)]
5#[non_exhaustive]
6pub enum Token {
7    /// `^` (start boundary)
8    Caret,
9    /// `$` (end boundary)
10    Dollar,
11    /// `%` (`\b` boundary)
12    Percent,
13    /// `<` (word start)
14    AngleLeft,
15    /// `>` (word end)
16    AngleRight,
17
18    /// `*` (`*?` repetition)
19    Star,
20    /// `+` (`+?` repetition)
21    Plus,
22    /// `?` (`??` repetition)
23    QuestionMark,
24
25    /// `|` (or)
26    Pipe,
27    /// `&` (and)
28    Ampersand,
29
30    /// `:` (capturing group start)
31    Colon,
32    /// `(` (open group)
33    OpenParen,
34    /// `)` (close group)
35    CloseParen,
36
37    /// `{` (open repetition)
38    OpenBrace,
39    /// `}` (close repetition)
40    CloseBrace,
41    /// `,` (comma in repetition)
42    Comma,
43
44    /// `!` (negation)
45    Not,
46
47    /// `[` (open character class)
48    OpenBracket,
49    /// `]` (close character class)
50    CloseBracket,
51
52    /// `-` (unicode range)
53    Dash,
54    /// `.` (any code point except newline)
55    Dot,
56
57    /// `>>` (positive lookahead)
58    LookAhead,
59    /// `<<` (positive lookbehind)
60    LookBehind,
61
62    /// `::` (back reference)
63    DoubleColon,
64
65    /// `;` (delimits modifiers)
66    Semicolon,
67
68    /// `=` (for assignments)
69    Equals,
70
71    /// `"Hello"` or `'Hello'` (`Hello`)
72    String,
73
74    /// `U+FFF03` (Unicode code point)
75    CodePoint,
76
77    /// `12` (number in repetition)
78    Number,
79
80    /// `hello` (capturing group name)
81    Identifier,
82
83    /// `lazy` (reserved name)
84    ReservedName,
85
86    /// Illegal token for which we want to show a better error message
87    ErrorMsg(LexErrorMsg),
88
89    /// Token representing an unknown character
90    Error,
91}
92
93impl core::fmt::Display for Token {
94    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
95        f.write_str(match self {
96            Token::Caret => "`^`",
97            Token::Dollar => "`$`",
98            Token::Percent => "`%`",
99            Token::Star => "`*`",
100            Token::Plus => "`+`",
101            Token::QuestionMark => "`?`",
102            Token::Pipe => "`|`",
103            Token::Ampersand => "`&`",
104            Token::Colon => "`:`",
105            Token::OpenParen => "`(`",
106            Token::CloseParen => "`)`",
107            Token::OpenBrace => "`{`",
108            Token::CloseBrace => "`}`",
109            Token::Comma => "`,`",
110            Token::LookAhead => "`>>`",
111            Token::LookBehind => "`<<`",
112            Token::AngleLeft => "`<`",
113            Token::AngleRight => "`>`",
114            Token::DoubleColon => "`::`",
115            Token::Not => "`!`",
116            Token::OpenBracket => "`[`",
117            Token::Dash => "`-`",
118            Token::CloseBracket => "`]`",
119            Token::Dot => "`.`",
120            Token::Semicolon => "`;`",
121            Token::Equals => "`=`",
122            Token::String => "string",
123            Token::CodePoint => "code point",
124            Token::Number => "number",
125            Token::Identifier => "identifier",
126            Token::ReservedName => "reserved name",
127            Token::ErrorMsg(_) | Token::Error => "error",
128        })
129    }
130}
pomsky_syntax/lexer/token.rs

pomsky_syntax/lexer/
token.rs
