rslua_march1917/
tokens.rs

1use crate::types::{FloatType, IntType, Source};
2
3#[derive(Debug, Clone, Copy, PartialEq)]
4pub enum TokenType {
5    And,
6    Break,
7    Do,
8    Else,
9    ElseIf,
10    End,
11    False,
12    For,
13    Function,
14    Goto,
15    // '//'
16    IDiv,
17    If,
18    In,
19    Local,
20    Nil,
21    Not,
22    Or,
23    Repeat,
24    Return,
25    Then,
26    True,
27    Until,
28    While,
29    // ..
30    Concat,
31    // ...
32    Dots,
33    // ==
34    Eq,
35    // >=
36    Ge,
37    // <=
38    Le,
39    // ~=
40    Ne,
41    // <<
42    Shl,
43    // >>
44    Shr,
45    // ::
46    DbColon,
47    Eos,
48    // float number
49    Flt,
50    // int number
51    Int,
52    // name
53    Name,
54    // string literal
55    String,
56    // () [] {}
57    Lp,
58    Rp,
59    Ls,
60    Rs,
61    Lb,
62    Rb,
63    // + - * / % ^ #
64    Add,
65    Minus,
66    Mul,
67    Div,
68    Mod,
69    Pow,
70    Len,
71    // =
72    Assign,
73    // < >
74    Lt,
75    Gt,
76    // & | ~
77    BAnd,
78    BOr,
79    BXor,
80    // : , ;
81    Colon,
82    Comma,
83    Semi,
84    // .
85    Attr,
86    // single line comment
87    SComment,
88    // multi-line comment
89    MComment,
90}
91
92impl TokenType {
93    // convert keyword to token type.
94    pub fn from_keyword(word: &str) -> Option<TokenType> {
95        match word {
96            "and" => Some(TokenType::And),
97            "break" => Some(TokenType::Break),
98            "do" => Some(TokenType::Do),
99            "else" => Some(TokenType::Else),
100            "elseif" => Some(TokenType::ElseIf),
101            "end" => Some(TokenType::End),
102            "false" => Some(TokenType::False),
103            "for" => Some(TokenType::For),
104            "function" => Some(TokenType::Function),
105            "goto" => Some(TokenType::Goto),
106            "if" => Some(TokenType::If),
107            "in" => Some(TokenType::In),
108            "local" => Some(TokenType::Local),
109            "nil" => Some(TokenType::Nil),
110            "not" => Some(TokenType::Not),
111            "or" => Some(TokenType::Or),
112            "repeat" => Some(TokenType::Repeat),
113            "return" => Some(TokenType::Return),
114            "then" => Some(TokenType::Then),
115            "true" => Some(TokenType::True),
116            "until" => Some(TokenType::Until),
117            "while" => Some(TokenType::While),
118            _ => None,
119        }
120    }
121}
122
123#[derive(Debug, PartialEq)]
124pub enum TokenValue {
125    None,
126    Float(FloatType),
127    Int(IntType),
128    Str(String),
129}
130
131#[derive(Debug, PartialEq)]
132pub struct Token {
133    pub t: TokenType,
134    pub value: TokenValue,
135    pub source: Source,
136    pub offset: usize,
137    pub comment_offset: usize,
138}
139
140impl Token {
141    pub fn get_float(&self) -> FloatType {
142        match self.value {
143            TokenValue::Float(f) => f,
144            _ => unreachable!(),
145        }
146    }
147    pub fn get_int(&self) -> IntType {
148        match self.value {
149            TokenValue::Int(i) => i,
150            _ => unreachable!(),
151        }
152    }
153    pub fn get_string(&self) -> String {
154        match &self.value {
155            TokenValue::Str(s) => s.clone(),
156            _ => unreachable!(),
157        }
158    }
159    pub fn is_comment(&self) -> bool {
160        self.t == TokenType::SComment || self.t == TokenType::MComment
161    }
162
163    pub fn comments<'a>(&self, tokens: &'a Vec<Token>) -> &'a [Token] {
164        &tokens[self.comment_offset..self.offset]
165    }
166}
rslua_march1917/tokens.rs

rslua_march1917/
tokens.rs
