sml_lang/
lexer.rs

1use crate::error::{Error, ErrorType};
2
3#[derive(Debug)]
4pub enum Token {
5    Identifier(String),
6    Section(String),
7    Newline,
8    Assign,
9
10    Int(i64),
11    Float(f64),
12    StringVal(String),
13    Bool(bool),
14}
15
16pub fn generate_tokens(source: String) -> Result<Vec<Token>, Error> {
17    let mut tokens: Vec<Token> = Vec::new();
18    let chars: Vec<char> = source.chars().collect();
19    let mut index: usize = 0;
20
21    while index < chars.len() {
22        let current_character = chars[index];
23
24        if current_character == '\n' {
25            tokens.push(Token::Newline);
26            index += 1;
27        }
28
29        else if current_character == ' ' {
30            tokens.push(Token::Assign);
31            index += 1;
32        }
33
34        else if current_character.is_alphabetic() {
35            let mut buffer = String::new();
36
37            while index < chars.len()
38                && (chars[index].is_alphanumeric() || chars[index] == '_')
39            {
40                buffer.push(chars[index]);
41                index += 1;
42            }
43
44            tokens.push(Token::Identifier(buffer));
45        }
46
47        else if current_character.is_numeric() {
48            let mut number_string = String::new();
49
50            while index < chars.len() &&
51                  (chars[index].is_numeric() || chars[index] == '.') {
52
53                number_string.push(chars[index]);
54                index += 1;
55            }
56
57            let dot_count = number_string.chars()
58                .filter(|&c| c == '.')
59                .count();
60
61            if dot_count == 0 {
62                tokens.push(Token::Int(number_string.parse().unwrap()));
63            }
64            else if dot_count == 1 {
65                tokens.push(Token::Float(number_string.parse().unwrap()));
66            }
67            else {
68                return Err(Error {
69                    error_type: ErrorType::IllegalCharacterError,
70                    description: "Too many dots in float".to_string(),
71                });
72            }
73        }
74
75        else {
76            return Err(Error {
77                error_type: ErrorType::IllegalCharacterError,
78                description: format!("Illegal character: {}", current_character),
79            });
80        }
81    }
82
83    Ok(tokens)
84}
sml_lang/lexer.rs

sml_lang/
lexer.rs
