math_parser_rs/dsl/
lex.rs

1use super::types::*;
2
3pub fn tokenize(function: String) -> LexResult {
4    let mut rhs: Vec<Token> = vec![];
5    let chars = function.to_lowercase().chars().collect::<Vec<_>>();
6    let mut i = 0;
7
8    let simple_token = [
9        ('+', TokenType::Operator),
10        ('-', TokenType::Operator),
11        ('/', TokenType::Operator),
12        ('*', TokenType::Operator),
13        ('(', TokenType::ParaOpen),
14        (')', TokenType::ParaClose),
15        (',', TokenType::Comma),
16    ];
17
18    while i < chars.len() {
19        if chars[i].is_whitespace() {
20            i += 1;
21            continue;
22        }
23
24        if let Some((token, n)) = try_number(&chars, i) {
25            rhs.push(token);
26            i += n;
27            continue;
28        }
29
30        if let Some((token, n)) = try_identifiers(&chars, i) {
31            rhs.push(token);
32            i += n;
33            continue;
34        }
35
36        if let Some(t_type) = simple_token
37            .iter()
38            .find(|(ch, _)| *ch == chars[i])
39            .map(|p| p.1)
40        {
41            rhs.push(Token::new(t_type, chars[i].to_string(), i));
42            i += 1;
43            continue;
44        }
45
46        rhs.push(Token::new(
47            TokenType::Unimplemented,
48            chars[i].to_string(),
49            i,
50        ));
51        i += 1;
52    }
53
54    LexResult { rhs }
55}
56
57fn try_number(chars: &[char], mut i: usize) -> Option<(Token, usize)> {
58    let start = i;
59    if !chars[i].is_ascii_digit() {
60        return None;
61    }
62    while i < chars.len() && (chars[i].is_ascii_digit() || chars[i] == '.') {
63        i += 1;
64    }
65    let token = Token {
66        token_type: TokenType::Number,
67        text: chars[start..i].iter().collect(),
68        position: start,
69    };
70    Some((token, i - start))
71}
72
73fn try_identifiers(chars: &[char], mut i: usize) -> Option<(Token, usize)> {
74    let start = i;
75    if !chars[i].is_ascii_alphabetic() {
76        return None;
77    }
78    while i < chars.len() && chars[i].is_ascii_alphanumeric() {
79        i += 1;
80    }
81    let token = Token {
82        token_type: TokenType::Identifier,
83        text: chars[start..i].iter().collect(),
84        position: start,
85    };
86    Some((token, i - start))
87}
88
89#[cfg(test)]
90mod tests {
91    use super::*;
92
93    #[test]
94    fn test_try_number() {
95        assert_eq!(
96            try_number(&['1', '2'], 0),
97            Some((Token::new(TokenType::Number, "12".to_string(), 0), 2))
98        );
99        assert_eq!(
100            try_number(&['1', '.', '2'], 0),
101            Some((Token::new(TokenType::Number, "1.2".to_string(), 0), 3))
102        );
103        assert_eq!(
104            try_number(&['1', '1', '.', '2'], 0),
105            Some((Token::new(TokenType::Number, "11.2".to_string(), 0), 4))
106        );
107        assert_eq!(
108            try_number(&['1', '.', '2'], 2),
109            Some((Token::new(TokenType::Number, "2".to_string(), 2), 1))
110        );
111        assert_eq!(
112            try_number(&['1', '+', '2'], 0),
113            Some((Token::new(TokenType::Number, "1".to_string(), 0), 1))
114        );
115        assert_eq!(
116            try_number(&['1', ' ', '2'], 0),
117            Some((Token::new(TokenType::Number, "1".to_string(), 0), 1))
118        );
119    }
120}
math_parser_rs/dsl/lex.rs

math_parser_rs/dsl/
lex.rs
