spreadsheet_formulas/
tokenizer.rs

1/// Tokenizer — lexes formula strings into tokens.
2
3use crate::cellref::CellRef;
4use crate::range::Range;
5
6#[derive(Debug, Clone, PartialEq)]
7pub enum Token {
8    Number(f64),
9    String(String),
10    Ident(String),
11    CellRef(CellRef),
12    Range(Range),
13    LParen,
14    RParen,
15    Comma,
16    Plus,
17    Minus,
18    Star,
19    Slash,
20    Caret,
21    Percent,
22    Eq,
23    Neq,
24    Lt,
25    Gt,
26    Lte,
27    Gte,
28    Amp, // &
29}
30
31/// Tokenize a formula string (without the leading =).
32pub fn tokenize(input: &str) -> Result<Vec<Token>, String> {
33    let chars: Vec<char> = input.chars().collect();
34    let mut tokens = Vec::new();
35    let mut i = 0;
36
37    while i < chars.len() {
38        let c = chars[i];
39
40        // Skip whitespace
41        if c.is_whitespace() {
42            i += 1;
43            continue;
44        }
45
46        // String literal
47        if c == '"' {
48            let mut s = String::new();
49            i += 1;
50            while i < chars.len() && chars[i] != '"' {
51                s.push(chars[i]);
52                i += 1;
53            }
54            if i >= chars.len() {
55                return Err("Unterminated string literal".into());
56            }
57            i += 1; // skip closing "
58            tokens.push(Token::String(s));
59            continue;
60        }
61
62        // Number
63        if c.is_ascii_digit() || (c == '.' && i + 1 < chars.len() && chars[i + 1].is_ascii_digit()) {
64            let mut num_str = String::new();
65            while i < chars.len() && (chars[i].is_ascii_digit() || chars[i] == '.') {
66                num_str.push(chars[i]);
67                i += 1;
68            }
69            let val: f64 = num_str.parse().map_err(|_| format!("Invalid number: {}", num_str))?;
70            tokens.push(Token::Number(val));
71            continue;
72        }
73
74        // Comparison operators (multi-char)
75        if c == '<' || c == '>' || c == '!' || c == '=' {
76            if c == '<' && i + 1 < chars.len() && chars[i + 1] == '=' {
77                tokens.push(Token::Lte);
78                i += 2;
79                continue;
80            }
81            if c == '>' && i + 1 < chars.len() && chars[i + 1] == '=' {
82                tokens.push(Token::Gte);
83                i += 2;
84                continue;
85            }
86            if c == '<' {
87                tokens.push(Token::Lt);
88                i += 1;
89                continue;
90            }
91            if c == '>' {
92                tokens.push(Token::Gt);
93                i += 1;
94                continue;
95            }
96            if c == '=' {
97                tokens.push(Token::Eq);
98                i += 1;
99                continue;
100            }
101            if c == '!' && i + 1 < chars.len() && chars[i + 1] == '=' {
102                tokens.push(Token::Neq);
103                i += 2;
104                continue;
105            }
106        }
107
108        // Single char operators
109        match c {
110            '(' => { tokens.push(Token::LParen); i += 1; continue; }
111            ')' => { tokens.push(Token::RParen); i += 1; continue; }
112            ',' => { tokens.push(Token::Comma); i += 1; continue; }
113            '+' => { tokens.push(Token::Plus); i += 1; continue; }
114            '-' => { tokens.push(Token::Minus); i += 1; continue; }
115            '*' => { tokens.push(Token::Star); i += 1; continue; }
116            '/' => { tokens.push(Token::Slash); i += 1; continue; }
117            '^' => { tokens.push(Token::Caret); i += 1; continue; }
118            '%' => { tokens.push(Token::Percent); i += 1; continue; }
119            '&' => { tokens.push(Token::Amp); i += 1; continue; }
120            _ => {}
121        }
122
123        // Identifiers (may include $), cell refs, or ranges
124        if c.is_ascii_alphabetic() || c == '$' || c == '_' {
125            let mut ident = String::new();
126            let _start = i;
127            while i < chars.len() && (chars[i].is_ascii_alphanumeric() || chars[i] == '$' || chars[i] == '_') {
128                ident.push(chars[i]);
129                i += 1;
130            }
131
132            // Try to parse as range (contains :)
133            if i < chars.len() && chars[i] == ':' {
134                let mut range_str = ident.clone();
135                range_str.push(':');
136                i += 1;
137                let right_start = i;
138                while i < chars.len() && (chars[i].is_ascii_alphanumeric() || chars[i] == '$') {
139                    range_str.push(chars[i]);
140                    i += 1;
141                }
142                if let Some(r) = Range::parse(&range_str) {
143                    tokens.push(Token::Range(r));
144                    continue;
145                }
146                // Fallback: didn't parse as range
147                let _ = right_start;
148                return Err(format!("Failed to parse range: {}", range_str));
149            }
150
151            // Try cell ref
152            if let Some(cell) = CellRef::parse(&ident) {
153                tokens.push(Token::CellRef(cell));
154                continue;
155            }
156
157            // It's an identifier
158            tokens.push(Token::Ident(ident.to_uppercase()));
159            continue;
160        }
161
162        return Err(format!("Unexpected character: '{}'", c));
163    }
164
165    Ok(tokens)
166}
167
168#[cfg(test)]
169mod tests {
170    use super::*;
171
172    #[test]
173    fn tokenize_number() {
174        let tokens = tokenize("42.5").unwrap();
175        assert_eq!(tokens.len(), 1);
176        assert_eq!(tokens[0], Token::Number(42.5));
177    }
178
179    #[test]
180    fn tokenize_string() {
181        let tokens = tokenize("\"hello world\"").unwrap();
182        assert_eq!(tokens[0], Token::String("hello world".into()));
183    }
184
185    #[test]
186    fn tokenize_function_call() {
187        let tokens = tokenize("SUM(A1:A10)").unwrap();
188        assert_eq!(tokens.len(), 4);
189        assert_eq!(tokens[0], Token::Ident("SUM".into()));
190        assert_eq!(tokens[1], Token::LParen);
191        assert!(matches!(&tokens[2], Token::Range(_)));
192        assert_eq!(tokens[3], Token::RParen);
193    }
194
195    #[test]
196    fn tokenize_operators() {
197        let tokens = tokenize("1+2*3").unwrap();
198        assert_eq!(tokens.len(), 5);
199        assert_eq!(tokens[0], Token::Number(1.0));
200        assert_eq!(tokens[1], Token::Plus);
201        assert_eq!(tokens[2], Token::Number(2.0));
202        assert_eq!(tokens[3], Token::Star);
203        assert_eq!(tokens[4], Token::Number(3.0));
204    }
205
206    #[test]
207    fn tokenize_comparison() {
208        let tokens = tokenize("A1<=B1").unwrap();
209        assert!(matches!(&tokens[1], Token::Lte));
210    }
211
212    #[test]
213    fn tokenize_cell_ref() {
214        let tokens = tokenize("$A$1").unwrap();
215        assert_eq!(tokens.len(), 1);
216        if let Token::CellRef(c) = &tokens[0] {
217            assert_eq!(c.col, 0);
218            assert_eq!(c.row, 0);
219        } else {
220            panic!("Expected CellRef");
221        }
222    }
223}
spreadsheet_formulas/tokenizer.rs

spreadsheet_formulas/
tokenizer.rs
