fastn_resolved/evalexpr/token/
mod.rs

1use fastn_resolved::evalexpr::{
2    error::{EvalexprError, EvalexprResult},
3    value::{FloatType, IntType},
4};
5
6mod display;
7
8#[derive(Clone, PartialEq, Debug)]
9pub enum Token {
10    // Arithmetic
11    Plus,
12    Minus,
13    Star,
14    Slash,
15    Percent,
16    Hat,
17
18    // Logic
19    Eq,
20    Neq,
21    Gt,
22    Lt,
23    Geq,
24    Leq,
25    And,
26    Or,
27    Not,
28
29    // Precedence
30    LBrace,
31    RBrace,
32
33    // Assignment
34    Assign,
35    PlusAssign,
36    MinusAssign,
37    StarAssign,
38    SlashAssign,
39    PercentAssign,
40    HatAssign,
41    AndAssign,
42    OrAssign,
43
44    // Special
45    Comma,
46    Semicolon,
47
48    // Values, Variables and Functions
49    Identifier(String),
50    Float(FloatType),
51    Int(IntType),
52    Boolean(bool),
53    String(String),
54}
55
56/// A partial token is an input character whose meaning depends on the characters around it.
57#[derive(Clone, Debug, PartialEq)]
58pub enum PartialToken {
59    /// A partial token that unambiguously maps to a single token.
60    Token(Token),
61    /// A partial token that is a literal.
62    Literal(String),
63    /// A plus character '+'.
64    Plus,
65    /// A minus character '-'.
66    Minus,
67    /// A star character '*'.
68    Star,
69    /// A slash character '/'.
70    Slash,
71    /// A percent character '%'.
72    Percent,
73    /// A hat character '^'.
74    Hat,
75    /// A whitespace character, e.g. ' '.
76    Whitespace,
77    /// An equal-to character '='.
78    Eq,
79    /// An exclamation mark character '!'.
80    ExclamationMark,
81    /// A greater-than character '>'.
82    Gt,
83    /// A lower-than character '<'.
84    Lt,
85    /// An ampersand character '&'.
86    Ampersand,
87    /// A vertical bar character '|'.
88    VerticalBar,
89}
90
91// Make this a const fn as soon as is_whitespace and to_string get stable (issue #57563)
92fn char_to_partial_token(c: char) -> PartialToken {
93    match c {
94        '+' => PartialToken::Plus,
95        '-' => PartialToken::Minus,
96        '*' => PartialToken::Star,
97        '/' => PartialToken::Slash,
98        '%' => PartialToken::Percent,
99        '^' => PartialToken::Hat,
100
101        '(' => PartialToken::Token(Token::LBrace),
102        ')' => PartialToken::Token(Token::RBrace),
103
104        ',' => PartialToken::Token(Token::Comma),
105        ';' => PartialToken::Token(Token::Semicolon),
106
107        '=' => PartialToken::Eq,
108        '!' => PartialToken::ExclamationMark,
109        '>' => PartialToken::Gt,
110        '<' => PartialToken::Lt,
111        '&' => PartialToken::Ampersand,
112        '|' => PartialToken::VerticalBar,
113
114        c => {
115            if c.is_whitespace() {
116                PartialToken::Whitespace
117            } else {
118                PartialToken::Literal(c.to_string())
119            }
120        }
121    }
122}
123
124impl Token {
125    pub(crate) const fn is_leftsided_value(&self) -> bool {
126        match self {
127            Token::Plus => false,
128            Token::Minus => false,
129            Token::Star => false,
130            Token::Slash => false,
131            Token::Percent => false,
132            Token::Hat => false,
133
134            Token::Eq => false,
135            Token::Neq => false,
136            Token::Gt => false,
137            Token::Lt => false,
138            Token::Geq => false,
139            Token::Leq => false,
140            Token::And => false,
141            Token::Or => false,
142            Token::Not => false,
143
144            Token::LBrace => true,
145            Token::RBrace => false,
146
147            Token::Comma => false,
148            Token::Semicolon => false,
149
150            Token::Assign => false,
151            Token::PlusAssign => false,
152            Token::MinusAssign => false,
153            Token::StarAssign => false,
154            Token::SlashAssign => false,
155            Token::PercentAssign => false,
156            Token::HatAssign => false,
157            Token::AndAssign => false,
158            Token::OrAssign => false,
159
160            Token::Identifier(_) => true,
161            Token::Float(_) => true,
162            Token::Int(_) => true,
163            Token::Boolean(_) => true,
164            Token::String(_) => true,
165        }
166    }
167
168    pub(crate) const fn is_rightsided_value(&self) -> bool {
169        match self {
170            Token::Plus => false,
171            Token::Minus => false,
172            Token::Star => false,
173            Token::Slash => false,
174            Token::Percent => false,
175            Token::Hat => false,
176
177            Token::Eq => false,
178            Token::Neq => false,
179            Token::Gt => false,
180            Token::Lt => false,
181            Token::Geq => false,
182            Token::Leq => false,
183            Token::And => false,
184            Token::Or => false,
185            Token::Not => false,
186
187            Token::LBrace => false,
188            Token::RBrace => true,
189
190            Token::Comma => false,
191            Token::Semicolon => false,
192
193            Token::Assign => false,
194            Token::PlusAssign => false,
195            Token::MinusAssign => false,
196            Token::StarAssign => false,
197            Token::SlashAssign => false,
198            Token::PercentAssign => false,
199            Token::HatAssign => false,
200            Token::AndAssign => false,
201            Token::OrAssign => false,
202
203            Token::Identifier(_) => true,
204            Token::Float(_) => true,
205            Token::Int(_) => true,
206            Token::Boolean(_) => true,
207            Token::String(_) => true,
208        }
209    }
210
211    pub(crate) fn is_assignment(&self) -> bool {
212        use Token::*;
213        matches!(
214            self,
215            Assign
216                | PlusAssign
217                | MinusAssign
218                | StarAssign
219                | SlashAssign
220                | PercentAssign
221                | HatAssign
222                | AndAssign
223                | OrAssign
224        )
225    }
226}
227
228/// Parses an escape sequence within a string literal.
229fn parse_escape_sequence<Iter: Iterator<Item = char>>(iter: &mut Iter) -> EvalexprResult<char> {
230    match iter.next() {
231        Some('"') => Ok('"'),
232        Some('\\') => Ok('\\'),
233        Some(c) => Err(EvalexprError::IllegalEscapeSequence(format!("\\{}", c))),
234        None => Err(EvalexprError::IllegalEscapeSequence("\\".to_string())),
235    }
236}
237
238/// Parses a string value from the given character iterator.
239///
240/// The first character from the iterator is interpreted as first character of the string.
241/// The string is terminated by a double quote `"`.
242/// Occurrences of `"` within the string can be escaped with `\`.
243/// The backslash needs to be escaped with another backslash `\`.
244fn parse_string_literal<Iter: Iterator<Item = char>>(
245    mut iter: &mut Iter,
246) -> EvalexprResult<PartialToken> {
247    let mut result = String::new();
248
249    while let Some(c) = iter.next() {
250        match c {
251            '"' => break,
252            '\\' => result.push(parse_escape_sequence(&mut iter)?),
253            c => result.push(c),
254        }
255    }
256
257    Ok(PartialToken::Token(Token::String(result)))
258}
259
260/// Converts a string to a vector of partial tokens.
261fn str_to_partial_tokens(string: &str) -> EvalexprResult<Vec<PartialToken>> {
262    let mut result = Vec::new();
263    let mut iter = string.chars().peekable();
264
265    while let Some(c) = iter.next() {
266        if c == '"' {
267            result.push(parse_string_literal(&mut iter)?);
268        } else {
269            let mut partial_token = char_to_partial_token(c);
270            if let Some(PartialToken::Literal(..)) = result.last() {
271                if partial_token == PartialToken::Minus {
272                    partial_token = PartialToken::Literal('-'.to_string())
273                }
274            }
275
276            let if_let_successful =
277                if let (Some(PartialToken::Literal(last)), PartialToken::Literal(literal)) =
278                    (result.last_mut(), &partial_token)
279                {
280                    last.push_str(literal);
281                    true
282                } else {
283                    false
284                };
285
286            if !if_let_successful {
287                result.push(partial_token);
288            }
289        }
290    }
291    Ok(result)
292}
293
294/// Resolves all partial tokens by converting them to complex tokens.
295fn partial_tokens_to_tokens(mut tokens: &[PartialToken]) -> EvalexprResult<Vec<Token>> {
296    let mut result = Vec::new();
297    while !tokens.is_empty() {
298        let first = tokens[0].clone();
299        let second = tokens.get(1).cloned();
300        let third = tokens.get(2).cloned();
301        let mut cutoff = 2;
302
303        result.extend(
304            match first {
305                PartialToken::Token(token) => {
306                    cutoff = 1;
307                    Some(token)
308                }
309                PartialToken::Plus => match second {
310                    Some(PartialToken::Eq) => Some(Token::PlusAssign),
311                    _ => {
312                        cutoff = 1;
313                        Some(Token::Plus)
314                    }
315                },
316                PartialToken::Minus => match second {
317                    Some(PartialToken::Eq) => Some(Token::MinusAssign),
318                    _ => {
319                        cutoff = 1;
320                        Some(Token::Minus)
321                    }
322                },
323                PartialToken::Star => match second {
324                    Some(PartialToken::Eq) => Some(Token::StarAssign),
325                    _ => {
326                        cutoff = 1;
327                        Some(Token::Star)
328                    }
329                },
330                PartialToken::Slash => match second {
331                    Some(PartialToken::Eq) => Some(Token::SlashAssign),
332                    _ => {
333                        cutoff = 1;
334                        Some(Token::Slash)
335                    }
336                },
337                PartialToken::Percent => match second {
338                    Some(PartialToken::Eq) => Some(Token::PercentAssign),
339                    _ => {
340                        cutoff = 1;
341                        Some(Token::Percent)
342                    }
343                },
344                PartialToken::Hat => match second {
345                    Some(PartialToken::Eq) => Some(Token::HatAssign),
346                    _ => {
347                        cutoff = 1;
348                        Some(Token::Hat)
349                    }
350                },
351                PartialToken::Literal(literal) => {
352                    cutoff = 1;
353                    if let Ok(number) = literal.parse::<IntType>() {
354                        Some(Token::Int(number))
355                    } else if let Ok(number) = literal.parse::<FloatType>() {
356                        Some(Token::Float(number))
357                    } else if let Ok(boolean) = literal.parse::<bool>() {
358                        Some(Token::Boolean(boolean))
359                    } else {
360                        // If there are two tokens following this one, check if the next one is
361                        // a plus or a minus. If so, then attempt to parse all three tokens as a
362                        // scientific notation number of the form `<coefficient>e{+,-}<exponent>`,
363                        // for example [Literal("10e"), Minus, Literal("3")] => "1e-3".parse().
364                        match (second, third) {
365                            (Some(second), Some(third))
366                                if second == PartialToken::Minus
367                                    || second == PartialToken::Plus =>
368                            {
369                                if let Ok(number) =
370                                    format!("{}{}{}", literal, second, third).parse::<FloatType>()
371                                {
372                                    cutoff = 3;
373                                    Some(Token::Float(number))
374                                } else {
375                                    Some(Token::Identifier(literal.to_string()))
376                                }
377                            }
378                            _ => Some(Token::Identifier(literal.to_string())),
379                        }
380                    }
381                }
382                PartialToken::Whitespace => {
383                    cutoff = 1;
384                    None
385                }
386                PartialToken::Eq => match second {
387                    Some(PartialToken::Eq) => Some(Token::Eq),
388                    _ => {
389                        cutoff = 1;
390                        Some(Token::Assign)
391                    }
392                },
393                PartialToken::ExclamationMark => match second {
394                    Some(PartialToken::Eq) => Some(Token::Neq),
395                    _ => {
396                        cutoff = 1;
397                        Some(Token::Not)
398                    }
399                },
400                PartialToken::Gt => match second {
401                    Some(PartialToken::Eq) => Some(Token::Geq),
402                    _ => {
403                        cutoff = 1;
404                        Some(Token::Gt)
405                    }
406                },
407                PartialToken::Lt => match second {
408                    Some(PartialToken::Eq) => Some(Token::Leq),
409                    _ => {
410                        cutoff = 1;
411                        Some(Token::Lt)
412                    }
413                },
414                PartialToken::Ampersand => match second {
415                    Some(PartialToken::Ampersand) => match third {
416                        Some(PartialToken::Eq) => {
417                            cutoff = 3;
418                            Some(Token::AndAssign)
419                        }
420                        _ => Some(Token::And),
421                    },
422                    _ => return Err(EvalexprError::unmatched_partial_token(first, second)),
423                },
424                PartialToken::VerticalBar => match second {
425                    Some(PartialToken::VerticalBar) => match third {
426                        Some(PartialToken::Eq) => {
427                            cutoff = 3;
428                            Some(Token::OrAssign)
429                        }
430                        _ => Some(Token::Or),
431                    },
432                    _ => return Err(EvalexprError::unmatched_partial_token(first, second)),
433                },
434            }
435            .into_iter(),
436        );
437
438        tokens = &tokens[cutoff..];
439    }
440    Ok(result)
441}
442
443pub(crate) fn tokenize(string: &str) -> EvalexprResult<Vec<Token>> {
444    partial_tokens_to_tokens(&str_to_partial_tokens(string)?)
445}
446
447#[cfg(test)]
448mod tests {
449    use fastn_resolved::evalexpr::token::{char_to_partial_token, tokenize, Token};
450    use std::fmt::Write;
451
452    #[test]
453    fn test_partial_token_display() {
454        let chars = vec![
455            '+', '-', '*', '/', '%', '^', '(', ')', ',', ';', '=', '!', '>', '<', '&', '|', ' ',
456        ];
457
458        for char in chars {
459            assert_eq!(
460                format!("{}", char),
461                format!("{}", char_to_partial_token(char))
462            );
463        }
464    }
465
466    #[test]
467    fn test_token_display() {
468        let token_string =
469            "+ - * / % ^ == != > < >= <= && || ! ( ) = += -= *= /= %= ^= &&= ||= , ; ";
470        let tokens = tokenize(token_string).unwrap();
471        let mut result_string = String::new();
472
473        for token in tokens {
474            write!(result_string, "{} ", token).unwrap();
475        }
476
477        assert_eq!(token_string, result_string);
478    }
479
480    #[test]
481    fn assignment_lhs_is_identifier() {
482        let tokens = tokenize("a = 1").unwrap();
483        assert_eq!(
484            tokens.as_slice(),
485            [
486                Token::Identifier("a".to_string()),
487                Token::Assign,
488                Token::Int(1)
489            ]
490        );
491    }
492}
fastn_resolved/evalexpr/token/mod.rs

fastn_resolved/evalexpr/token/
mod.rs
