oak_tcl/lexer/
token_type.rs

1use oak_core::{Source, Token, TokenType, UniversalElementRole, UniversalTokenRole};
2
3/// A token type alias for Tcl tokens.
4pub type TclToken = Token<TclTokenType>;
5
6impl TokenType for TclTokenType {
7    type Role = UniversalTokenRole;
8    const END_OF_STREAM: Self = Self::Error;
9
10    fn is_ignored(&self) -> bool {
11        false
12    }
13
14    fn role(&self) -> Self::Role {
15        match self {
16            _ => UniversalTokenRole::None,
17        }
18    }
19}
20
21/// Represents the different types of tokens in the Tcl language.
22#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
23#[repr(u8)]
24#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
25pub enum TclTokenType {
26    /// Root node of the AST.
27    Root,
28    /// A command node.
29    Command,
30    /// A word node.
31    Word,
32    /// A simple word node.
33    SimpleWord,
34    /// A variable word node.
35    VariableWord,
36    /// A script word node.
37    ScriptWord,
38    /// A braced word node.
39    BracedWord,
40
41    /// A numeric literal.
42    Number,
43    /// A string literal.
44    StringLiteral,
45    /// An identifier.
46    Identifier,
47
48    /// The `if` keyword.
49    If,
50    /// The `else` keyword.
51    Else,
52    /// The `elseif` keyword.
53    ElseIf,
54    /// The `for` keyword.
55    For,
56    /// The `while` keyword.
57    While,
58    /// The `foreach` keyword.
59    ForEach,
60    /// The `proc` keyword.
61    Proc,
62    /// The `return` keyword.
63    Return,
64    /// The `break` keyword.
65    Break,
66    /// The `continue` keyword.
67    Continue,
68    /// The `set` keyword.
69    Set,
70    /// The `unset` keyword.
71    Unset,
72    /// The `global` keyword.
73    Global,
74    /// The `upvar` keyword.
75    Upvar,
76    /// The `variable` keyword.
77    Variable,
78
79    /// The `+` operator.
80    Plus,
81    /// The `-` operator.
82    Minus,
83    /// The `*` operator.
84    Star,
85    /// The `/` operator.
86    Slash,
87    /// The `%` operator.
88    Percent,
89    /// The `==` operator.
90    Equal,
91    /// The `!=` operator.
92    NotEqual,
93    /// The `<` operator.
94    Less,
95    /// The `>` operator.
96    Greater,
97    /// The `<=` operator.
98    LessEqual,
99    /// The `>=` operator.
100    GreaterEqual,
101    /// The `&` operator.
102    Ampersand,
103    /// The `&&` operator.
104    AmpersandAmpersand,
105    /// The `|` operator.
106    Pipe,
107    /// The `||` operator.
108    PipePipe,
109    /// The `!` operator.
110    Exclamation,
111
112    /// The `(` punctuation.
113    LeftParen,
114    /// The `)` punctuation.
115    RightParen,
116    /// The `[` punctuation.
117    LeftBracket,
118    /// The `]` punctuation.
119    RightBracket,
120    /// The `{` punctuation.
121    LeftBrace,
122    /// The `}` punctuation.
123    RightBrace,
124    /// The `;` punctuation.
125    Semicolon,
126    /// The `,` punctuation.
127    Comma,
128    /// The `$` punctuation.
129    Dollar,
130
131    /// Whitespace token.
132    Whitespace,
133    /// Newline token.
134    Newline,
135    /// Comment token.
136    Comment,
137    /// Error token.
138    Error,
139    /// End of file token.
140    Eof,
141}
142
143impl From<TclTokenType> for UniversalElementRole {
144    fn from(kind: TclTokenType) -> Self {
145        match kind {
146            TclTokenType::Root => UniversalElementRole::Root,
147            TclTokenType::Command => UniversalElementRole::Expression,
148            TclTokenType::Word | TclTokenType::SimpleWord | TclTokenType::VariableWord | TclTokenType::ScriptWord | TclTokenType::BracedWord => UniversalElementRole::Expression,
149            TclTokenType::Identifier => UniversalElementRole::Name,
150            TclTokenType::Number | TclTokenType::StringLiteral => UniversalElementRole::Value,
151            _ => UniversalElementRole::None,
152        }
153    }
154}
155
156impl From<crate::parser::element_type::TclElementType> for TclTokenType {
157    fn from(element: crate::parser::element_type::TclElementType) -> Self {
158        unsafe { std::mem::transmute(element) }
159    }
160}
oak_tcl/lexer/token_type.rs

oak_tcl/lexer/
token_type.rs
