1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

use super::LexErrorMsg;

/// A token encountered while lexing a pomsky expression.
#[derive(Debug, Eq, PartialEq, Copy, Clone)]
#[non_exhaustive]
pub enum Token {
    /// `^` (start boundary)
    Caret,
    /// `$` (end boundary)
    Dollar,
    /// `%` (`\b` boundary)
    BWord,

    /// `*` (`*?` repetition)
    Star,
    /// `+` (`+?` repetition)
    Plus,
    /// `?` (`??` repetition)
    QuestionMark,

    /// `|` (or)
    Pipe,

    /// `:` (capturing group start)
    Colon,
    /// `(` (open group)
    OpenParen,
    /// `)` (close group)
    CloseParen,

    /// `{` (open repetition)
    OpenBrace,
    /// `}` (close repetition)
    CloseBrace,
    /// `,` (comma in repetition)
    Comma,

    /// `!` (negation)
    Not,

    /// `[` (open character class)
    OpenBracket,

    /// `-` (unicode range)
    Dash,

    /// `]` (close character class)
    CloseBracket,

    /// `.` (any code point except newline)
    Dot,

    /// `>>` (positive lookahead)
    LookAhead,

    /// `<<` (positive lookbehind)
    LookBehind,

    /// `::` (back reference)
    DoubleColon,

    /// `;` (delimits modifiers)
    Semicolon,

    /// `=` (for assignments)
    Equals,

    /// `"Hello"` or `'Hello'` (`Hello`)
    String,

    /// `U+FFF03` (Unicode code point)
    CodePoint,

    /// `12` (number in repetition)
    Number,

    /// `hello` (capturing group name)
    Identifier,

    /// `lazy` (reserved name)
    ReservedName,

    /// Illegal token for which we want to show a better error message
    ErrorMsg(LexErrorMsg),

    /// Token representing an unknown character
    Error,
}

impl core::fmt::Display for Token {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.write_str(match self {
            Token::Caret => "`^`",
            Token::Dollar => "`$`",
            Token::BWord => "`%`",
            Token::Star => "`*`",
            Token::Plus => "`+`",
            Token::QuestionMark => "`?`",
            Token::Pipe => "`|`",
            Token::Colon => "`:`",
            Token::OpenParen => "`(`",
            Token::CloseParen => "`)`",
            Token::OpenBrace => "`{`",
            Token::CloseBrace => "`}`",
            Token::Comma => "`,`",
            Token::LookAhead => "`>>`",
            Token::LookBehind => "`<<`",
            Token::DoubleColon => "`::`",
            Token::Not => "`!`",
            Token::OpenBracket => "`[`",
            Token::Dash => "`-`",
            Token::CloseBracket => "`]`",
            Token::Dot => "`.`",
            Token::Semicolon => "`;`",
            Token::Equals => "`=`",
            Token::String => "string",
            Token::CodePoint => "code point",
            Token::Number => "number",
            Token::Identifier => "identifier",
            Token::ReservedName => "reserved name",
            Token::ErrorMsg(_) | Token::Error => "error",
        })
    }
}