aether/
token.rs

1// src/token.rs
2//! Token definitions for the Aether lexer
3
4/// Position information for tokens
5#[derive(Debug, Clone, Copy, PartialEq)]
6pub struct Position {
7    pub line: usize,
8    pub column: usize,
9}
10
11impl Position {
12    pub fn new(line: usize, column: usize) -> Self {
13        Position { line, column }
14    }
15}
16
17impl std::fmt::Display for Position {
18    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
19        write!(f, "line {}, column {}", self.line, self.column)
20    }
21}
22
23/// Token with position information
24#[derive(Debug, Clone, PartialEq)]
25pub struct TokenWithPos {
26    pub token: Token,
27    pub pos: Position,
28}
29
30impl TokenWithPos {
31    pub fn new(token: Token, line: usize, column: usize) -> Self {
32        TokenWithPos {
33            token,
34            pos: Position::new(line, column),
35        }
36    }
37}
38
39/// Represents all possible tokens in the Aether language
40#[derive(Debug, Clone, PartialEq)]
41pub enum Token {
42    // Keywords - 首字母大写
43    Set,
44    Func,
45    Lambda,
46    Generator,
47    Lazy,
48    If,
49    Elif,
50    Else,
51    While,
52    For,
53    In,
54    Switch,
55    Case,
56    Default,
57    Return,
58    Yield,
59    Break,
60    Continue,
61    Import,
62    From,
63    As,
64    Export,
65    Throw,
66
67    // Identifiers and literals - 全大写标识符
68    Identifier(String),
69    Number(f64),
70    BigInteger(String), // 大整数字面量，保留原始字符串
71    String(String),
72    Boolean(bool),
73    Null,
74
75    // Operators
76    Plus,     // +
77    Minus,    // -
78    Multiply, // *
79    Divide,   // /
80    Modulo,   // %
81
82    // Comparison
83    Equal,        // ==
84    NotEqual,     // !=
85    Less,         // <
86    LessEqual,    // <=
87    Greater,      // >
88    GreaterEqual, // >=
89
90    // Logical
91    And, // &&
92    Or,  // ||
93    Not, // !
94
95    // Assignment
96    Assign, // = (但 Aether 中用 Set，这里可能不需要)
97
98    // Delimiters
99    LeftParen,    // (
100    RightParen,   // )
101    LeftBrace,    // {
102    RightBrace,   // }
103    LeftBracket,  // [
104    RightBracket, // ]
105    Comma,        // ,
106    Colon,        // :
107    Semicolon,    // ;
108    Newline,      // \n (语句分隔符)
109
110    // Special
111    Arrow, // ->
112    Illegal(char),
113    EOF,
114}
115
116impl Token {
117    /// Check if a string is a keyword
118    pub fn lookup_keyword(ident: &str) -> Token {
119        match ident {
120            // Keywords
121            "Set" => Token::Set,
122            "Func" => Token::Func,
123            "Lambda" => Token::Lambda,
124            "Generator" => Token::Generator,
125            "Lazy" => Token::Lazy,
126            "If" => Token::If,
127            "Elif" => Token::Elif,
128            "Else" => Token::Else,
129            "While" => Token::While,
130            "For" => Token::For,
131            "In" => Token::In,
132            "Switch" => Token::Switch,
133            "Case" => Token::Case,
134            "Default" => Token::Default,
135            "Return" => Token::Return,
136            "Yield" => Token::Yield,
137            "Break" => Token::Break,
138            "Continue" => Token::Continue,
139            "Import" => Token::Import,
140            "From" => Token::From,
141            "As" => Token::As,
142            "as" => Token::As,
143            "Export" => Token::Export,
144            "Throw" => Token::Throw,
145
146            // Logical operators as keywords
147            "And" => Token::And,
148            "Or" => Token::Or,
149            "Not" => Token::Not,
150
151            // Boolean literals (uppercase per Aether design)
152            "True" => Token::Boolean(true),
153            "False" => Token::Boolean(false),
154
155            // Null
156            "Null" => Token::Null,
157
158            // Not a keyword, return identifier
159            _ => Token::Identifier(ident.to_string()),
160        }
161    }
162
163    /// Get a human-readable representation of the token
164    pub fn token_type(&self) -> &str {
165        match self {
166            Token::Set => "Set",
167            Token::Func => "Func",
168            Token::Lambda => "Lambda",
169            Token::Generator => "Generator",
170            Token::Lazy => "Lazy",
171            Token::If => "If",
172            Token::Elif => "Elif",
173            Token::Else => "Else",
174            Token::While => "While",
175            Token::For => "For",
176            Token::In => "In",
177            Token::Switch => "Switch",
178            Token::Case => "Case",
179            Token::Default => "Default",
180            Token::Return => "Return",
181            Token::Yield => "Yield",
182            Token::Break => "Break",
183            Token::Continue => "Continue",
184            Token::Import => "Import",
185            Token::From => "From",
186            Token::As => "As",
187            Token::Export => "Export",
188            Token::Throw => "Throw",
189            Token::Identifier(_) => "Identifier",
190            Token::Number(_) => "Number",
191            Token::BigInteger(_) => "BigInteger",
192            Token::String(_) => "String",
193            Token::Boolean(_) => "Boolean",
194            Token::Null => "nil",
195            Token::Plus => "+",
196            Token::Minus => "-",
197            Token::Multiply => "*",
198            Token::Divide => "/",
199            Token::Modulo => "%",
200            Token::Equal => "==",
201            Token::NotEqual => "!=",
202            Token::Less => "<",
203            Token::LessEqual => "<=",
204            Token::Greater => ">",
205            Token::GreaterEqual => ">=",
206            Token::And => "&&",
207            Token::Or => "||",
208            Token::Not => "!",
209            Token::Assign => "=",
210            Token::LeftParen => "(",
211            Token::RightParen => ")",
212            Token::LeftBrace => "{",
213            Token::RightBrace => "}",
214            Token::LeftBracket => "[",
215            Token::RightBracket => "]",
216            Token::Comma => ",",
217            Token::Colon => ":",
218            Token::Semicolon => ";",
219            Token::Newline => "\\n",
220            Token::Arrow => "->",
221            Token::Illegal(_) => "Illegal",
222            Token::EOF => "EOF",
223        }
224    }
225}
aether/token.rs

aether/
token.rs
