neutron_engine/iris/
lexer.rs

1/// Iris Lexer - Tokenizer
2/// 
3/// Converts raw source code into a stream of tokens.
4
5#[derive(Debug, Clone, PartialEq)]
6pub enum TokenType {
7    // Literals
8    Null,
9    Bool(bool),
10    Number(f64),
11    String(String),
12    Identifier(String),
13
14    // Keywords
15    Let,
16    Const,
17    Fn,
18    If,
19    Else,
20    While,
21    For,
22    In,
23    Return,
24    Break,
25    Continue,
26    True,
27    False,
28    And,
29    Or,
30    Not,
31    Import,
32
33    // Operators
34    Plus,      // +
35    Minus,     // -
36    Star,      // *
37    Slash,     // /
38    Percent,   // %
39    Eq,        // ==
40    Neq,       // !=
41    Lt,        // <
42    Gt,        // >
43    Lte,       // <=
44    Gte,       // >=
45    Assign,    // =
46    PlusAssign,  // +=
47    MinusAssign, // -=
48    StarAssign,  // *=
49    SlashAssign, // /=
50
51    // Delimiters
52    LParen,    // (
53    RParen,    // )
54    LBrace,    // {
55    RBrace,    // }
56    LBracket,  // [
57    RBracket,  // ]
58    Semicolon, // ;
59    Colon,     // :
60    Comma,     // ,
61    Dot,       // .
62    Arrow,     // =>
63
64    // Special
65    Comment(String),
66    Newline,
67    Eof,
68}
69
70#[derive(Debug, Clone)]
71pub struct Token {
72    pub token_type: TokenType,
73    pub line: usize,
74    pub column: usize,
75}
76
77pub fn tokenize(source: &str) -> Result<Vec<Token>, String> {
78    let mut tokens = Vec::new();
79    let mut chars = source.chars().peekable();
80    let mut line = 1;
81    let mut column = 1;
82
83    while let Some(&ch) = chars.peek() {
84        let start_col = column;
85
86        match ch {
87            // Whitespace
88            ' ' | '\t' | '\r' => {
89                chars.next();
90                column += 1;
91            }
92            '\n' => {
93                chars.next();
94                tokens.push(Token { token_type: TokenType::Newline, line, column });
95                line += 1;
96                column = 1;
97            }
98
99            // Comments
100            '/' if chars.clone().nth(1) == Some('/') => {
101                chars.next(); chars.next();
102                column += 2;
103                let mut comment = String::new();
104                while let Some(&c) = chars.peek() {
105                    if c == '\n' { break; }
106                    comment.push(c);
107                    chars.next();
108                    column += 1;
109                }
110                tokens.push(Token { token_type: TokenType::Comment(comment), line, column: start_col });
111            }
112            '/' if chars.clone().nth(1) == Some('*') => {
113                chars.next(); chars.next();
114                column += 2;
115                let mut comment = String::new();
116                let mut depth = 1;
117                while depth > 0 {
118                    match chars.next() {
119                        Some('*') if chars.peek() == Some(&'/') => {
120                            chars.next();
121                            depth -= 1;
122                        }
123                        Some('/') if chars.peek() == Some(&'*') => {
124                            chars.next();
125                            depth += 1;
126                        }
127                        Some('\n') => { line += 1; column = 1; }
128                        Some(c) => comment.push(c),
129                        None => return Err(format!("Unterminated block comment at line {}", line)),
130                    }
131                }
132                tokens.push(Token { token_type: TokenType::Comment(comment), line, column: start_col });
133            }
134
135            // Strings
136            '"' => {
137                chars.next();
138                column += 1;
139                let mut string = String::new();
140                while let Some(&c) = chars.peek() {
141                    if c == '"' { break; }
142                    if c == '\\' {
143                        chars.next();
144                        column += 1;
145                        match chars.peek() {
146                            Some(&'n') => string.push('\n'),
147                            Some(&'t') => string.push('\t'),
148                            Some(&'r') => string.push('\r'),
149                            Some(&'\\') => string.push('\\'),
150                            Some(&'"') => string.push('"'),
151                            Some(&c) => string.push(c),
152                            None => return Err(format!("Unterminated string escape at line {}", line)),
153                        }
154                    } else {
155                        string.push(c);
156                    }
157                    chars.next();
158                    column += 1;
159                }
160                if chars.peek() != Some(&'"') {
161                    return Err(format!("Unterminated string at line {}", line));
162                }
163                chars.next();
164                column += 1;
165                tokens.push(Token { token_type: TokenType::String(string), line, column: start_col });
166            }
167
168            // Numbers
169            '0'..='9' => {
170                let mut num_str = String::new();
171                while let Some(&c) = chars.peek() {
172                    if c.is_ascii_digit() || c == '.' {
173                        num_str.push(c);
174                        chars.next();
175                        column += 1;
176                    } else {
177                        break;
178                    }
179                }
180                match num_str.parse::<f64>() {
181                    Ok(n) => tokens.push(Token { token_type: TokenType::Number(n), line, column: start_col }),
182                    Err(_) => return Err(format!("Invalid number '{}' at line {}", num_str, line)),
183                }
184            }
185
186            // Identifiers and Keywords
187            'a'..='z' | 'A'..='Z' | '_' => {
188                let mut ident = String::new();
189                while let Some(&c) = chars.peek() {
190                    if c.is_alphanumeric() || c == '_' {
191                        ident.push(c);
192                        chars.next();
193                        column += 1;
194                    } else {
195                        break;
196                    }
197                }
198                let token_type = match ident.as_str() {
199                    "null" => TokenType::Null,
200                    "true" => TokenType::Bool(true),
201                    "false" => TokenType::Bool(false),
202                    "let" => TokenType::Let,
203                    "const" => TokenType::Const,
204                    "fn" => TokenType::Fn,
205                    "if" => TokenType::If,
206                    "else" => TokenType::Else,
207                    "while" => TokenType::While,
208                    "for" => TokenType::For,
209                    "in" => TokenType::In,
210                    "return" => TokenType::Return,
211                    "break" => TokenType::Break,
212                    "continue" => TokenType::Continue,
213                    "and" => TokenType::And,
214                    "or" => TokenType::Or,
215                    "not" => TokenType::Not,
216                    "import" => TokenType::Import,
217                    _ => TokenType::Identifier(ident),
218                };
219                tokens.push(Token { token_type, line, column: start_col });
220            }
221
222            // Multi-char operators
223            '+' if chars.clone().nth(1) == Some('=') => {
224                chars.next(); chars.next();
225                column += 2;
226                tokens.push(Token { token_type: TokenType::PlusAssign, line, column: start_col });
227            }
228            '-' if chars.clone().nth(1) == Some('=') => {
229                chars.next(); chars.next();
230                column += 2;
231                tokens.push(Token { token_type: TokenType::MinusAssign, line, column: start_col });
232            }
233            '*' if chars.clone().nth(1) == Some('=') => {
234                chars.next(); chars.next();
235                column += 2;
236                tokens.push(Token { token_type: TokenType::StarAssign, line, column: start_col });
237            }
238            '/' if chars.clone().nth(1) == Some('=') => {
239                chars.next(); chars.next();
240                column += 2;
241                tokens.push(Token { token_type: TokenType::SlashAssign, line, column: start_col });
242            }
243            '=' if chars.clone().nth(1) == Some('=') => {
244                chars.next(); chars.next();
245                column += 2;
246                tokens.push(Token { token_type: TokenType::Eq, line, column: start_col });
247            }
248            '!' if chars.clone().nth(1) == Some('=') => {
249                chars.next(); chars.next();
250                column += 2;
251                tokens.push(Token { token_type: TokenType::Neq, line, column: start_col });
252            }
253            '<' if chars.clone().nth(1) == Some('=') => {
254                chars.next(); chars.next();
255                column += 2;
256                tokens.push(Token { token_type: TokenType::Lte, line, column: start_col });
257            }
258            '>' if chars.clone().nth(1) == Some('=') => {
259                chars.next(); chars.next();
260                column += 2;
261                tokens.push(Token { token_type: TokenType::Gte, line, column: start_col });
262            }
263            '=' if chars.clone().nth(1) == Some('>') => {
264                chars.next(); chars.next();
265                column += 2;
266                tokens.push(Token { token_type: TokenType::Arrow, line, column: start_col });
267            }
268
269            // Single-char operators
270            '+' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Plus, line, column: start_col }); }
271            '-' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Minus, line, column: start_col }); }
272            '*' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Star, line, column: start_col }); }
273            '/' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Slash, line, column: start_col }); }
274            '%' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Percent, line, column: start_col }); }
275            '=' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Assign, line, column: start_col }); }
276            '<' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Lt, line, column: start_col }); }
277            '>' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Gt, line, column: start_col }); }
278
279            // Delimiters
280            '(' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::LParen, line, column: start_col }); }
281            ')' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::RParen, line, column: start_col }); }
282            '{' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::LBrace, line, column: start_col }); }
283            '}' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::RBrace, line, column: start_col }); }
284            '[' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::LBracket, line, column: start_col }); }
285            ']' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::RBracket, line, column: start_col }); }
286            ';' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Semicolon, line, column: start_col }); }
287            ':' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Colon, line, column: start_col }); }
288            ',' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Comma, line, column: start_col }); }
289            '.' => { chars.next(); column += 1; tokens.push(Token { token_type: TokenType::Dot, line, column: start_col }); }
290
291            _ => return Err(format!("Unexpected character '{}' at line {}, column {}", ch, line, column)),
292        }
293    }
294
295    tokens.push(Token { token_type: TokenType::Eof, line, column });
296    Ok(tokens)
297}
298
neutron_engine/iris/lexer.rs

neutron_engine/iris/
lexer.rs
