aether/
token.rs

1// src/token.rs
2//! Token definitions for the Aether lexer
3
4/// Position information for tokens
5#[derive(Debug, Clone, Copy, PartialEq)]
6pub struct Position {
7    pub line: usize,
8    pub column: usize,
9}
10
11impl Position {
12    pub fn new(line: usize, column: usize) -> Self {
13        Position { line, column }
14    }
15}
16
17impl std::fmt::Display for Position {
18    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
19        write!(f, "line {}, column {}", self.line, self.column)
20    }
21}
22
23/// Token with position information
24#[derive(Debug, Clone, PartialEq)]
25pub struct TokenWithPos {
26    pub token: Token,
27    pub pos: Position,
28}
29
30impl TokenWithPos {
31    pub fn new(token: Token, line: usize, column: usize) -> Self {
32        TokenWithPos {
33            token,
34            pos: Position::new(line, column),
35        }
36    }
37}
38
39/// Represents all possible tokens in the Aether language
40#[derive(Debug, Clone, PartialEq)]
41pub enum Token {
42    // Keywords - 首字母大写
43    Set,
44    Func,
45    Generator,
46    Lazy,
47    If,
48    Elif,
49    Else,
50    While,
51    For,
52    In,
53    Switch,
54    Case,
55    Default,
56    Return,
57    Yield,
58    Break,
59    Continue,
60    Import,
61    From,
62    As,
63    Export,
64    Throw,
65
66    // Identifiers and literals - 全大写标识符
67    Identifier(String),
68    Number(f64),
69    BigInteger(String), // 大整数字面量，保留原始字符串
70    String(String),
71    Boolean(bool),
72    Null,
73
74    // Operators
75    Plus,     // +
76    Minus,    // -
77    Multiply, // *
78    Divide,   // /
79    Modulo,   // %
80
81    // Comparison
82    Equal,        // ==
83    NotEqual,     // !=
84    Less,         // <
85    LessEqual,    // <=
86    Greater,      // >
87    GreaterEqual, // >=
88
89    // Logical
90    And, // &&
91    Or,  // ||
92    Not, // !
93
94    // Assignment
95    Assign, // = (但 Aether 中用 Set，这里可能不需要)
96
97    // Delimiters
98    LeftParen,    // (
99    RightParen,   // )
100    LeftBrace,    // {
101    RightBrace,   // }
102    LeftBracket,  // [
103    RightBracket, // ]
104    Comma,        // ,
105    Colon,        // :
106    Semicolon,    // ;
107    Newline,      // \n (语句分隔符)
108
109    // Special
110    Arrow, // ->
111    Illegal(char),
112    EOF,
113}
114
115impl Token {
116    /// Check if a string is a keyword
117    pub fn lookup_keyword(ident: &str) -> Token {
118        match ident {
119            // Keywords
120            "Set" => Token::Set,
121            "Func" => Token::Func,
122            "Generator" => Token::Generator,
123            "Lazy" => Token::Lazy,
124            "If" => Token::If,
125            "Elif" => Token::Elif,
126            "Else" => Token::Else,
127            "While" => Token::While,
128            "For" => Token::For,
129            "In" => Token::In,
130            "Switch" => Token::Switch,
131            "Case" => Token::Case,
132            "Default" => Token::Default,
133            "Return" => Token::Return,
134            "Yield" => Token::Yield,
135            "Break" => Token::Break,
136            "Continue" => Token::Continue,
137            "Import" => Token::Import,
138            "From" => Token::From,
139            "as" => Token::As,
140            "Export" => Token::Export,
141            "Throw" => Token::Throw,
142
143            // Logical operators as keywords
144            "And" => Token::And,
145            "Or" => Token::Or,
146            "Not" => Token::Not,
147
148            // Boolean literals (uppercase per Aether design)
149            "True" => Token::Boolean(true),
150            "False" => Token::Boolean(false),
151
152            // Null
153            "Null" => Token::Null,
154
155            // Not a keyword, return identifier
156            _ => Token::Identifier(ident.to_string()),
157        }
158    }
159
160    /// Get a human-readable representation of the token
161    pub fn token_type(&self) -> &str {
162        match self {
163            Token::Set => "Set",
164            Token::Func => "Func",
165            Token::Generator => "Generator",
166            Token::Lazy => "Lazy",
167            Token::If => "If",
168            Token::Elif => "Elif",
169            Token::Else => "Else",
170            Token::While => "While",
171            Token::For => "For",
172            Token::In => "In",
173            Token::Switch => "Switch",
174            Token::Case => "Case",
175            Token::Default => "Default",
176            Token::Return => "Return",
177            Token::Yield => "Yield",
178            Token::Break => "Break",
179            Token::Continue => "Continue",
180            Token::Import => "Import",
181            Token::From => "From",
182            Token::As => "as",
183            Token::Export => "Export",
184            Token::Throw => "Throw",
185            Token::Identifier(_) => "Identifier",
186            Token::Number(_) => "Number",
187            Token::BigInteger(_) => "BigInteger",
188            Token::String(_) => "String",
189            Token::Boolean(_) => "Boolean",
190            Token::Null => "nil",
191            Token::Plus => "+",
192            Token::Minus => "-",
193            Token::Multiply => "*",
194            Token::Divide => "/",
195            Token::Modulo => "%",
196            Token::Equal => "==",
197            Token::NotEqual => "!=",
198            Token::Less => "<",
199            Token::LessEqual => "<=",
200            Token::Greater => ">",
201            Token::GreaterEqual => ">=",
202            Token::And => "&&",
203            Token::Or => "||",
204            Token::Not => "!",
205            Token::Assign => "=",
206            Token::LeftParen => "(",
207            Token::RightParen => ")",
208            Token::LeftBrace => "{",
209            Token::RightBrace => "}",
210            Token::LeftBracket => "[",
211            Token::RightBracket => "]",
212            Token::Comma => ",",
213            Token::Colon => ":",
214            Token::Semicolon => ";",
215            Token::Newline => "\\n",
216            Token::Arrow => "->",
217            Token::Illegal(_) => "Illegal",
218            Token::EOF => "EOF",
219        }
220    }
221}
222
223#[cfg(test)]
224mod tests {
225    use super::*;
226
227    #[test]
228    fn test_keyword_lookup() {
229        assert_eq!(Token::lookup_keyword("Set"), Token::Set);
230        assert_eq!(Token::lookup_keyword("Func"), Token::Func);
231        assert_eq!(Token::lookup_keyword("If"), Token::If);
232        assert_eq!(Token::lookup_keyword("True"), Token::Boolean(true));
233        assert_eq!(Token::lookup_keyword("False"), Token::Boolean(false));
234        assert_eq!(Token::lookup_keyword("Null"), Token::Null);
235
236        // Non-keyword should return identifier
237        match Token::lookup_keyword("MY_VAR") {
238            Token::Identifier(s) => assert_eq!(s, "MY_VAR"),
239            _ => panic!("Expected identifier"),
240        }
241    }
242
243    #[test]
244    fn test_token_type() {
245        assert_eq!(Token::Set.token_type(), "Set");
246        assert_eq!(Token::Plus.token_type(), "+");
247        assert_eq!(Token::Equal.token_type(), "==");
248        assert_eq!(Token::LeftParen.token_type(), "(");
249        assert_eq!(Token::EOF.token_type(), "EOF");
250    }
251}
aether/token.rs

aether/
token.rs
