aether/
token.rs

1// src/token.rs
2//! Token definitions for the Aether lexer
3
4/// Position information for tokens
5#[derive(Debug, Clone, Copy, PartialEq)]
6pub struct Position {
7    pub line: usize,
8    pub column: usize,
9}
10
11impl Position {
12    pub fn new(line: usize, column: usize) -> Self {
13        Position { line, column }
14    }
15}
16
17impl std::fmt::Display for Position {
18    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
19        write!(f, "line {}, column {}", self.line, self.column)
20    }
21}
22
23/// Token with position information
24#[derive(Debug, Clone, PartialEq)]
25pub struct TokenWithPos {
26    pub token: Token,
27    pub pos: Position,
28}
29
30impl TokenWithPos {
31    pub fn new(token: Token, line: usize, column: usize) -> Self {
32        TokenWithPos {
33            token,
34            pos: Position::new(line, column),
35        }
36    }
37}
38
39/// Represents all possible tokens in the Aether language
40#[derive(Debug, Clone, PartialEq)]
41pub enum Token {
42    // Keywords - 首字母大写
43    Set,
44    Func,
45    Lambda,
46    Generator,
47    Lazy,
48    If,
49    Elif,
50    Else,
51    While,
52    For,
53    In,
54    Switch,
55    Case,
56    Default,
57    Return,
58    Yield,
59    Break,
60    Continue,
61    Import,
62    From,
63    As,
64    Export,
65    Throw,
66
67    // Identifiers and literals - 全大写标识符
68    Identifier(String),
69    Number(f64),
70    BigInteger(String), // 大整数字面量，保留原始字符串
71    String(String),
72    Boolean(bool),
73    Null,
74
75    // Operators
76    Plus,     // +
77    Minus,    // -
78    Multiply, // *
79    Divide,   // /
80    Modulo,   // %
81
82    // Comparison
83    Equal,        // ==
84    NotEqual,     // !=
85    Less,         // <
86    LessEqual,    // <=
87    Greater,      // >
88    GreaterEqual, // >=
89
90    // Logical
91    And, // &&
92    Or,  // ||
93    Not, // !
94
95    // Assignment
96    Assign, // = (但 Aether 中用 Set，这里可能不需要)
97
98    // Delimiters
99    LeftParen,    // (
100    RightParen,   // )
101    LeftBrace,    // {
102    RightBrace,   // }
103    LeftBracket,  // [
104    RightBracket, // ]
105    Comma,        // ,
106    Colon,        // :
107    Semicolon,    // ;
108    Newline,      // \n (语句分隔符)
109
110    // Special
111    Arrow, // ->
112    Illegal(char),
113    EOF,
114}
115
116impl Token {
117    /// Check if a string is a keyword
118    pub fn lookup_keyword(ident: &str) -> Token {
119        match ident {
120            // Keywords
121            "Set" => Token::Set,
122            "Func" => Token::Func,
123            "Lambda" => Token::Lambda,
124            "Generator" => Token::Generator,
125            "Lazy" => Token::Lazy,
126            "If" => Token::If,
127            "Elif" => Token::Elif,
128            "Else" => Token::Else,
129            "While" => Token::While,
130            "For" => Token::For,
131            "In" => Token::In,
132            "Switch" => Token::Switch,
133            "Case" => Token::Case,
134            "Default" => Token::Default,
135            "Return" => Token::Return,
136            "Yield" => Token::Yield,
137            "Break" => Token::Break,
138            "Continue" => Token::Continue,
139            "Import" => Token::Import,
140            "From" => Token::From,
141            "as" => Token::As,
142            "Export" => Token::Export,
143            "Throw" => Token::Throw,
144
145            // Logical operators as keywords
146            "And" => Token::And,
147            "Or" => Token::Or,
148            "Not" => Token::Not,
149
150            // Boolean literals (uppercase per Aether design)
151            "True" => Token::Boolean(true),
152            "False" => Token::Boolean(false),
153
154            // Null
155            "Null" => Token::Null,
156
157            // Not a keyword, return identifier
158            _ => Token::Identifier(ident.to_string()),
159        }
160    }
161
162    /// Get a human-readable representation of the token
163    pub fn token_type(&self) -> &str {
164        match self {
165            Token::Set => "Set",
166            Token::Func => "Func",
167            Token::Lambda => "Lambda",
168            Token::Generator => "Generator",
169            Token::Lazy => "Lazy",
170            Token::If => "If",
171            Token::Elif => "Elif",
172            Token::Else => "Else",
173            Token::While => "While",
174            Token::For => "For",
175            Token::In => "In",
176            Token::Switch => "Switch",
177            Token::Case => "Case",
178            Token::Default => "Default",
179            Token::Return => "Return",
180            Token::Yield => "Yield",
181            Token::Break => "Break",
182            Token::Continue => "Continue",
183            Token::Import => "Import",
184            Token::From => "From",
185            Token::As => "as",
186            Token::Export => "Export",
187            Token::Throw => "Throw",
188            Token::Identifier(_) => "Identifier",
189            Token::Number(_) => "Number",
190            Token::BigInteger(_) => "BigInteger",
191            Token::String(_) => "String",
192            Token::Boolean(_) => "Boolean",
193            Token::Null => "nil",
194            Token::Plus => "+",
195            Token::Minus => "-",
196            Token::Multiply => "*",
197            Token::Divide => "/",
198            Token::Modulo => "%",
199            Token::Equal => "==",
200            Token::NotEqual => "!=",
201            Token::Less => "<",
202            Token::LessEqual => "<=",
203            Token::Greater => ">",
204            Token::GreaterEqual => ">=",
205            Token::And => "&&",
206            Token::Or => "||",
207            Token::Not => "!",
208            Token::Assign => "=",
209            Token::LeftParen => "(",
210            Token::RightParen => ")",
211            Token::LeftBrace => "{",
212            Token::RightBrace => "}",
213            Token::LeftBracket => "[",
214            Token::RightBracket => "]",
215            Token::Comma => ",",
216            Token::Colon => ":",
217            Token::Semicolon => ";",
218            Token::Newline => "\\n",
219            Token::Arrow => "->",
220            Token::Illegal(_) => "Illegal",
221            Token::EOF => "EOF",
222        }
223    }
224}
225
226#[cfg(test)]
227mod tests {
228    use super::*;
229
230    #[test]
231    fn test_keyword_lookup() {
232        assert_eq!(Token::lookup_keyword("Set"), Token::Set);
233        assert_eq!(Token::lookup_keyword("Func"), Token::Func);
234        assert_eq!(Token::lookup_keyword("If"), Token::If);
235        assert_eq!(Token::lookup_keyword("True"), Token::Boolean(true));
236        assert_eq!(Token::lookup_keyword("False"), Token::Boolean(false));
237        assert_eq!(Token::lookup_keyword("Null"), Token::Null);
238
239        // Non-keyword should return identifier
240        match Token::lookup_keyword("MY_VAR") {
241            Token::Identifier(s) => assert_eq!(s, "MY_VAR"),
242            _ => panic!("Expected identifier"),
243        }
244    }
245
246    #[test]
247    fn test_token_type() {
248        assert_eq!(Token::Set.token_type(), "Set");
249        assert_eq!(Token::Plus.token_type(), "+");
250        assert_eq!(Token::Equal.token_type(), "==");
251        assert_eq!(Token::LeftParen.token_type(), "(");
252        assert_eq!(Token::EOF.token_type(), "EOF");
253    }
254}
aether/token.rs

aether/
token.rs
