neutron_engine/iris/
lexer.rs

1use std::result::Result;
2use super::token::TokenType;
3
4
5#[derive(Debug, Clone)]
6pub struct Token {
7    pub token_type: TokenType,
8    pub line: usize,
9    pub column: usize,
10    pub lexeme: String,
11}
12
13pub fn tokenize(source: &str) -> Result<Vec<Token>, String> {
14    let mut tokens = Vec::new();
15    let chars: Vec<char> = source.chars().collect();
16    let mut i = 0;
17    let mut line = 1;
18    let mut col = 1;
19    
20    while i < chars.len() {
21        let ch = chars[i];
22        let start_col = col;
23        match ch {
24            ' ' | '\t' | '\r' => {
25                i += 1;
26                col += 1;
27            }
28            '\n' => {
29                i += 1;
30                line += 1;
31                col = 1;
32            }
33            '/' => {
34                if i + 1 < chars.len() && chars[i + 1] == '/' {
35                    // Skip comment
36                    i += 2;
37                    while i < chars.len() && chars[i] != '\n' {
38                        i += 1;
39                        col += 1;
40                    }
41                    continue;
42                }
43                tokens.push(Token {
44                    token_type: TokenType::Slash,
45                    line,
46                    column: start_col,
47                    lexeme: "/".to_string(),
48                });
49                i += 1;
50                col += 1;
51            }
52            '"' => {
53                i += 1;
54                col += 1;
55                let mut s = String::new();
56                while i < chars.len() && chars[i] != '"' {
57                    s.push(chars[i]);
58                    i += 1;
59                    col += 1;
60                }
61                if i < chars.len() {
62                    i += 1;
63                    col += 1;
64                }
65                let s_clone = s.clone();
66                tokens.push(Token {
67                    token_type: TokenType::String(s),
68                    line,
69                    column: start_col,
70                    lexeme: format!("\"{}\"", s_clone),
71                });
72
73            }
74            '0'..='9' => {
75                let mut num = String::new();
76                while i < chars.len() && chars[i].is_ascii_digit() {
77                    num.push(chars[i]);
78                    i += 1;
79                    col += 1;
80                }
81                let n: f64 = num.parse().map_err(|_| "Invalid number".to_string())?;
82                tokens.push(Token {
83                    token_type: TokenType::Number(n),
84                    line,
85                    column: start_col,
86                    lexeme: num,
87                });
88            }
89            'a'..='z' | 'A'..='Z' | '_' => {
90                let mut ident = String::new();
91                while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_') {
92                    ident.push(chars[i]);
93                    i += 1;
94                    col += 1;
95                }
96                let ttype = match ident.as_str() {
97                    "let" => TokenType::Let,
98                    "const" => TokenType::Const,
99                    "fn" => TokenType::Fn,
100                    "if" => TokenType::If,
101                    "else" => TokenType::Else,
102                    "elif" => TokenType::Elif,
103                    "while" => TokenType::While,
104                    "for" => TokenType::For,
105"in" => TokenType::In,
106                    "return" => TokenType::Return,
107"true" => TokenType::True,
108"false" => TokenType::False,
109                    "null" => TokenType::Null,
110                    "break" => TokenType::Break,
111                    "continue" => TokenType::Continue,
112                    "system" => TokenType::System,
113                    "import" => TokenType::Import,
114_ => TokenType::Identifier(ident.clone()),
115                };
116                tokens.push(Token {
117                    token_type: ttype,
118                    line,
119                    column: start_col,
120                    lexeme: ident,
121                });
122            }
123            '!' => {
124                i += 1;
125                col += 1;
126                tokens.push(Token {
127                    token_type: TokenType::Bang,
128                    line,
129                    column: start_col,
130                    lexeme: "!".to_string(),
131                });
132            }
133            '+' => {
134                tokens.push(Token { token_type: TokenType::Plus, line, column: start_col, lexeme: "+".to_string() });
135                i += 1;
136                col += 1;
137            }
138            '-' => {
139                tokens.push(Token { token_type: TokenType::Minus, line, column: start_col, lexeme: "-".to_string() });
140                i += 1;
141                col += 1;
142            }
143            '*' => {
144                tokens.push(Token { token_type: TokenType::Star, line, column: start_col, lexeme: "*".to_string() });
145                i += 1;
146                col += 1;
147            }
148            '=' => {
149                if i + 1 < chars.len() && chars[i + 1] == '=' {
150                    i += 2;
151                    col += 2;
152                    tokens.push(Token { token_type: TokenType::EqualEqual, line, column: start_col, lexeme: "==".to_string() });
153                } else {
154                    i += 1;
155                    col += 1;
156                    tokens.push(Token { token_type: TokenType::Assign, line, column: start_col, lexeme: "=".to_string() });
157                }
158            }
159            '<' => {
160                if i + 1 < chars.len() && chars[i + 1] == '=' {
161                    i += 2;
162                    col += 2;
163                    tokens.push(Token { token_type: TokenType::Lte, line, column: start_col, lexeme: "<=".to_string() });
164                } else {
165                    i += 1;
166                    col += 1;
167                    tokens.push(Token { token_type: TokenType::Lt, line, column: start_col, lexeme: "<".to_string() });
168                }
169            }
170            '>' => {
171                if i + 1 < chars.len() && chars[i + 1] == '=' {
172                    i += 2;
173                    col += 2;
174                    tokens.push(Token { token_type: TokenType::Gte, line, column: start_col, lexeme: ">=".to_string() });
175                } else {
176                    i += 1;
177                    col += 1;
178                    tokens.push(Token { token_type: TokenType::Gt, line, column: start_col, lexeme: ">".to_string() });
179                }
180            }
181            '(' => {
182                tokens.push(Token { token_type: TokenType::LeftParen, line, column: start_col, lexeme: "(".to_string() });
183                i += 1;
184                col += 1;
185            }
186            ')' => {
187                tokens.push(Token { token_type: TokenType::RightParen, line, column: start_col, lexeme: ")".to_string() });
188                i += 1;
189                col += 1;
190            }
191            '{' => {
192                tokens.push(Token { token_type: TokenType::LeftBrace, line, column: start_col, lexeme: "{".to_string() });
193                i += 1;
194                col += 1;
195            }
196            '}' => {
197                tokens.push(Token { token_type: TokenType::RightBrace, line, column: start_col, lexeme: "}".to_string() });
198                i += 1;
199                col += 1;
200            }
201            ';' => {
202                tokens.push(Token { token_type: TokenType::Semicolon, line, column: start_col, lexeme: ";".to_string() });
203                i += 1;
204                col += 1;
205            }
206            ',' => {
207                tokens.push(Token { token_type: TokenType::Comma, line, column: start_col, lexeme: ",".to_string() });
208                i += 1;
209                col += 1;
210            }
211            '[' => {
212                tokens.push(Token { token_type: TokenType::LBracket, line, column: start_col, lexeme: "[".to_string() });
213                i += 1;
214                col += 1;
215            }
216            ']' => {
217                tokens.push(Token { token_type: TokenType::RBracket, line, column: start_col, lexeme: "]".to_string() });
218                i += 1;
219                col += 1;
220            }
221            ':' => {
222                if i + 1 < chars.len() && chars[i + 1] == ':' {
223                    tokens.push(Token { token_type: TokenType::Scope, line, column: start_col, lexeme: "::".to_string() });
224                    i += 2;
225                    col += 2;
226                } else {
227                    tokens.push(Token { token_type: TokenType::Colon, line, column: start_col, lexeme: ":".to_string() });
228                    i += 1;
229                    col += 1;
230                }
231            }
232            '%' => {
233                tokens.push(Token { token_type: TokenType::Percent, line, column: start_col, lexeme: "%".to_string() });
234                i += 1;
235                col += 1;
236            }
237            _ => {
238                i += 1;
239                col += 1;
240            }
241        }
242    }
243    
244    tokens.push(Token {
245        token_type: TokenType::Eof,
246        line,
247        column: col,
248        lexeme: "".to_string(),
249    });
250    
251    Ok(tokens)
252}
253
neutron_engine/iris/lexer.rs

neutron_engine/iris/
lexer.rs
