use std::result::Result;
use super::token::TokenType;
#[derive(Debug, Clone)]
pub struct Token {
pub token_type: TokenType,
pub line: usize,
pub column: usize,
pub lexeme: String,
}
pub fn tokenize(source: &str) -> Result<Vec<Token>, String> {
let mut tokens = Vec::new();
let chars: Vec<char> = source.chars().collect();
let mut i = 0;
let mut line = 1;
let mut col = 1;
while i < chars.len() {
let ch = chars[i];
let start_col = col;
match ch {
' ' | '\t' | '\r' => {
i += 1;
col += 1;
}
'\n' => {
i += 1;
line += 1;
col = 1;
}
'/' => {
if i + 1 < chars.len() && chars[i + 1] == '/' {
i += 2;
while i < chars.len() && chars[i] != '\n' {
i += 1;
col += 1;
}
continue;
}
tokens.push(Token {
token_type: TokenType::Slash,
line,
column: start_col,
lexeme: "/".to_string(),
});
i += 1;
col += 1;
}
'"' => {
i += 1;
col += 1;
let mut s = String::new();
while i < chars.len() && chars[i] != '"' {
s.push(chars[i]);
i += 1;
col += 1;
}
if i < chars.len() {
i += 1;
col += 1;
}
let s_clone = s.clone();
tokens.push(Token {
token_type: TokenType::String(s),
line,
column: start_col,
lexeme: format!("\"{}\"", s_clone),
});
}
'0'..='9' => {
let mut num = String::new();
while i < chars.len() && chars[i].is_ascii_digit() {
num.push(chars[i]);
i += 1;
col += 1;
}
let n: f64 = num.parse().map_err(|_| "Invalid number".to_string())?;
tokens.push(Token {
token_type: TokenType::Number(n),
line,
column: start_col,
lexeme: num,
});
}
'a'..='z' | 'A'..='Z' | '_' => {
let mut ident = String::new();
while i < chars.len() && (chars[i].is_alphanumeric() || chars[i] == '_') {
ident.push(chars[i]);
i += 1;
col += 1;
}
let ttype = match ident.as_str() {
"let" => TokenType::Let,
"const" => TokenType::Const,
"fn" => TokenType::Fn,
"if" => TokenType::If,
"else" => TokenType::Else,
"elif" => TokenType::Elif,
"while" => TokenType::While,
"for" => TokenType::For,
"in" => TokenType::In,
"return" => TokenType::Return,
"true" => TokenType::True,
"false" => TokenType::False,
"null" => TokenType::Null,
"break" => TokenType::Break,
"continue" => TokenType::Continue,
"system" => TokenType::System,
"import" => TokenType::Import,
_ => TokenType::Identifier(ident.clone()),
};
tokens.push(Token {
token_type: ttype,
line,
column: start_col,
lexeme: ident,
});
}
'!' => {
i += 1;
col += 1;
tokens.push(Token {
token_type: TokenType::Bang,
line,
column: start_col,
lexeme: "!".to_string(),
});
}
'+' => {
tokens.push(Token { token_type: TokenType::Plus, line, column: start_col, lexeme: "+".to_string() });
i += 1;
col += 1;
}
'-' => {
tokens.push(Token { token_type: TokenType::Minus, line, column: start_col, lexeme: "-".to_string() });
i += 1;
col += 1;
}
'*' => {
tokens.push(Token { token_type: TokenType::Star, line, column: start_col, lexeme: "*".to_string() });
i += 1;
col += 1;
}
'=' => {
if i + 1 < chars.len() && chars[i + 1] == '=' {
i += 2;
col += 2;
tokens.push(Token { token_type: TokenType::EqualEqual, line, column: start_col, lexeme: "==".to_string() });
} else {
i += 1;
col += 1;
tokens.push(Token { token_type: TokenType::Assign, line, column: start_col, lexeme: "=".to_string() });
}
}
'<' => {
if i + 1 < chars.len() && chars[i + 1] == '=' {
i += 2;
col += 2;
tokens.push(Token { token_type: TokenType::Lte, line, column: start_col, lexeme: "<=".to_string() });
} else {
i += 1;
col += 1;
tokens.push(Token { token_type: TokenType::Lt, line, column: start_col, lexeme: "<".to_string() });
}
}
'>' => {
if i + 1 < chars.len() && chars[i + 1] == '=' {
i += 2;
col += 2;
tokens.push(Token { token_type: TokenType::Gte, line, column: start_col, lexeme: ">=".to_string() });
} else {
i += 1;
col += 1;
tokens.push(Token { token_type: TokenType::Gt, line, column: start_col, lexeme: ">".to_string() });
}
}
'(' => {
tokens.push(Token { token_type: TokenType::LeftParen, line, column: start_col, lexeme: "(".to_string() });
i += 1;
col += 1;
}
')' => {
tokens.push(Token { token_type: TokenType::RightParen, line, column: start_col, lexeme: ")".to_string() });
i += 1;
col += 1;
}
'{' => {
tokens.push(Token { token_type: TokenType::LeftBrace, line, column: start_col, lexeme: "{".to_string() });
i += 1;
col += 1;
}
'}' => {
tokens.push(Token { token_type: TokenType::RightBrace, line, column: start_col, lexeme: "}".to_string() });
i += 1;
col += 1;
}
';' => {
tokens.push(Token { token_type: TokenType::Semicolon, line, column: start_col, lexeme: ";".to_string() });
i += 1;
col += 1;
}
',' => {
tokens.push(Token { token_type: TokenType::Comma, line, column: start_col, lexeme: ",".to_string() });
i += 1;
col += 1;
}
'[' => {
tokens.push(Token { token_type: TokenType::LBracket, line, column: start_col, lexeme: "[".to_string() });
i += 1;
col += 1;
}
']' => {
tokens.push(Token { token_type: TokenType::RBracket, line, column: start_col, lexeme: "]".to_string() });
i += 1;
col += 1;
}
':' => {
if i + 1 < chars.len() && chars[i + 1] == ':' {
tokens.push(Token { token_type: TokenType::Scope, line, column: start_col, lexeme: "::".to_string() });
i += 2;
col += 2;
} else {
tokens.push(Token { token_type: TokenType::Colon, line, column: start_col, lexeme: ":".to_string() });
i += 1;
col += 1;
}
}
'%' => {
tokens.push(Token { token_type: TokenType::Percent, line, column: start_col, lexeme: "%".to_string() });
i += 1;
col += 1;
}
_ => {
i += 1;
col += 1;
}
}
}
tokens.push(Token {
token_type: TokenType::Eof,
line,
column: col,
lexeme: "".to_string(),
});
Ok(tokens)
}