mate_rs/
token.rs

1//
2// Copyright 2022-present theiskaa. All rights reserved.
3// Use of this source code is governed by MIT license
4// that can be found in the LICENSE file.
5//
6
7use crate::utils::ChUtils;
8
9// The structure model for high level sub expression implementations.
10// That could hold the actual source: tokens, and transformation method.
11//
12// For example: in case of parentheses and combinable
13// operations(*, /, %) method have to be [PAREN].
14// Or, in case of absolute values the method have to be [ABS],
15// to let calculator know the approach it has to take to
16// return final result of tokens.
17#[derive(Clone, Debug, PartialEq)]
18pub struct Sub {
19    pub tokens: Vec<Token>,
20    pub method: SubMethod,
21}
22
23impl Sub {
24    // Generate a new sub structure data.
25    pub fn new(tokens: Vec<Token>, method: SubMethod) -> Self {
26        Self { tokens, method }
27    }
28
29    // Generates a empty representation of token's sub element.
30    pub fn empty() -> Self {
31        Self {
32            tokens: Vec::new(),
33            method: SubMethod::PAREN,
34        }
35    }
36}
37
38// The method type of sub expression -> [Sub].
39// Used to decide the final calculation method of sub expression tokens.
40// For example, in case of [PAREN] the result will be default result of calculated [tokens].
41// Or, in case of [ABS] the result always gonna be positive value.
42#[derive(Clone, Debug, PartialEq)]
43pub enum SubMethod {
44    PAREN,
45    ABS,
46}
47
48#[derive(Clone, Debug, PartialEq)]
49pub enum TokenType {
50    ILLEGAL,
51
52    // Internal
53    NUMBER,
54    IDENTIFIER,
55    ASSIGN,
56
57    // Sub related tokens
58    SUBEXP,
59    POINTER,
60    LPAREN,
61    RPAREN,
62    LABS,
63    RABS,
64
65    // Operations
66    PLUS,
67    MINUS,
68    PRODUCT,
69    DIVIDE,
70    PERCENTAGE,
71    POWER,
72    FACTORIAL,
73
74    // Math functions
75    SQRT,
76    SIN,
77    COS,
78    TAN,
79    LOG,
80    LN,
81    EXP,
82    FLOOR,
83    CEIL,
84    ROUND,
85}
86
87// The main structure of input's each parsed character.
88#[derive(Clone, Debug, PartialEq)]
89pub struct Token {
90    pub typ: TokenType,
91    pub literal: String,
92    pub sub: Sub,
93    // the index range of concrete token.
94    // [-1] represents the unknown index.
95    // left side is the starting point and right side is ending point.
96    pub index: (i32, i32),
97}
98
99impl TokenType {
100    // A function to get valid sub-method from token type.
101    //
102    // - [TokenType::ABS] is [SubMethod::ABS].
103    // - [TokenType::LPAREN] & [TokenType::RPAREN] is [SubMethod::PAREN].
104    pub fn to_submethod(&self) -> SubMethod {
105        match self {
106            TokenType::LABS => SubMethod::ABS,
107            TokenType::RABS => SubMethod::ABS,
108            _ => SubMethod::PAREN, // + LPAREN, RPAREN
109        }
110    }
111}
112
113impl Token {
114    // Define a new Token value by providing all fields.
115    pub fn new(typ: TokenType, literal: String, sub: Sub, index: (i32, i32)) -> Self {
116        Self {
117            typ,
118            literal,
119            sub,
120            index,
121        }
122    }
123
124    // Create a new sub token model with just sub tokens.
125    pub fn new_sub(tokens: Vec<Token>, method: SubMethod) -> Self {
126        Self {
127            typ: TokenType::SUBEXP,
128            literal: String::new(),
129            sub: Sub { tokens, method },
130            index: Token::unknown_index(),
131        }
132    }
133
134    // Creates a pointer token, that newer will be used
135    // at normal token result.
136    pub fn new_pointer(i: usize, method: SubMethod) -> Self {
137        Self {
138            typ: TokenType::POINTER,
139            literal: format!("{i}"),
140            sub: Sub::new(Vec::new(), method),
141            index: Token::unknown_index(),
142        }
143    }
144
145    // Create a new token model from a literal.
146    // The type is decided automatically by checking it.
147    pub fn from(mut literal: String, index: (i32, i32)) -> Self {
148        let typ = if literal.is_number() {
149            TokenType::NUMBER
150        } else {
151            match literal.trim().to_lowercase().as_str() {
152                "+" => TokenType::PLUS,
153                "-" => TokenType::MINUS,
154                "*" | "•" => TokenType::PRODUCT,
155                "/" | ":" => TokenType::DIVIDE,
156                "(" => TokenType::LPAREN,
157                ")" => TokenType::RPAREN,
158                "%" => TokenType::PERCENTAGE,
159                "^" => TokenType::POWER,
160                "!" => TokenType::FACTORIAL,
161                "[" => TokenType::LABS,
162                "]" => TokenType::RABS,
163                "sqrt" => TokenType::SQRT,
164                "sin" => TokenType::SIN,
165                "cos" => TokenType::COS,
166                "tan" => TokenType::TAN,
167                "log" => TokenType::LOG,
168                "ln" => TokenType::LN,
169                "exp" => TokenType::EXP,
170                "floor" => TokenType::FLOOR,
171                "ceil" => TokenType::CEIL,
172                "round" => TokenType::ROUND,
173                "=" => TokenType::ASSIGN,
174                _ => {
175                    // Check if it's a valid identifier (starts with letter, contains only alphanumeric)
176                    let trimmed = literal.trim();
177                    if !trimmed.is_empty() && trimmed.chars().next().unwrap().is_alphabetic()
178                        && trimmed.chars().all(|c| c.is_alphanumeric() || c == '_') {
179                        TokenType::IDENTIFIER
180                    } else {
181                        TokenType::ILLEGAL
182                    }
183                }
184            }
185        };
186
187        // Clear the white-spaces from literal.
188        literal.retain(|c| !c.is_whitespace());
189
190        Self {
191            typ,
192            literal,
193            sub: Sub::empty(),
194            index,
195        }
196    }
197
198    // Creates an empty Token model.
199    pub fn empty() -> Self {
200        Self {
201            typ: TokenType::ILLEGAL,
202            literal: String::new(),
203            sub: Sub::empty(),
204            index: (0, 0),
205        }
206    }
207
208    // A function to get valid sub-method from token.
209    //
210    // - If [self.typ] is [ABS] is [SubMethod::ABS].
211    // - If [self.typ] is [LPAREN] or [RPAREN] is [SubMethod::PAREN].
212    pub fn to_submethod(&self) -> SubMethod {
213        match &self.typ {
214            TokenType::LABS => SubMethod::ABS,
215            TokenType::RABS => SubMethod::ABS,
216            _ => SubMethod::PAREN, // + LPAREN, RPAREN
217        }
218    }
219
220    // Returns the default unknown index representation.
221    pub fn unknown_index() -> (i32, i32) {
222        (-1, -1)
223    }
224
225    // Takes the pointer's index as [usize].
226    // If current token is not an pointer token, returned option will be [None].
227    pub fn take_pointer_index(&self) -> Option<usize> {
228        if self.typ != TokenType::POINTER {
229            return None;
230        }
231
232        self.literal.as_str().parse::<usize>().ok()
233    }
234
235    pub fn is_illegal(&self) -> bool {
236        matches!(self.typ, TokenType::ILLEGAL)
237    }
238
239    pub fn is_number(&self) -> bool {
240        matches!(self.typ, TokenType::NUMBER)
241    }
242
243    pub fn is_lparen(&self) -> bool {
244        matches!(self.typ, TokenType::LPAREN)
245    }
246
247    pub fn is_rparen(&self) -> bool {
248        matches!(self.typ, TokenType::RPAREN)
249    }
250
251    pub fn is_pointer(&self) -> bool {
252        matches!(self.typ, TokenType::POINTER)
253    }
254
255    pub fn is_sub_exp(&self) -> bool {
256        matches!(self.typ, TokenType::SUBEXP)
257    }
258
259    pub fn is_power(&self) -> bool {
260        matches!(self.typ, TokenType::POWER)
261    }
262
263    pub fn is_labs(&self) -> bool {
264        matches!(self.typ, TokenType::LABS)
265    }
266
267    pub fn is_rabs(&self) -> bool {
268        matches!(self.typ, TokenType::RABS)
269    }
270
271    pub fn is_factorial(&self) -> bool {
272        matches!(self.typ, TokenType::FACTORIAL)
273    }
274
275    pub fn is_identifier(&self) -> bool {
276        matches!(self.typ, TokenType::IDENTIFIER)
277    }
278
279    pub fn is_assign(&self) -> bool {
280        matches!(self.typ, TokenType::ASSIGN)
281    }
282
283    pub fn is_function(&self) -> bool {
284        matches!(
285            self.typ,
286            TokenType::SQRT
287                | TokenType::SIN
288                | TokenType::COS
289                | TokenType::TAN
290                | TokenType::LOG
291                | TokenType::LN
292                | TokenType::EXP
293                | TokenType::FLOOR
294                | TokenType::CEIL
295                | TokenType::ROUND
296        )
297    }
298
299    // Checks the "parentheses" family tokens' matching to each other.
300    // So, if pointed(self) token is left-parentheses
301    // given token(t) should be right-parentheses, if not returns false.
302    pub fn matchto(&self, t: &Token) -> bool {
303        let m = match self.typ {
304            TokenType::LPAREN => TokenType::RPAREN,
305            TokenType::LABS => TokenType::RABS,
306            _ => TokenType::ILLEGAL,
307        };
308
309        m == t.typ
310    }
311}
312
313#[cfg(test)]
314mod tests {
315    use super::*;
316    use std::collections::HashMap;
317
318    #[test]
319    fn new_sub_struct() {
320        let test_data: Vec<Sub> = vec![
321            Sub {
322                tokens: Vec::new(),
323                method: SubMethod::PAREN,
324            },
325            Sub {
326                tokens: Vec::new(),
327                method: SubMethod::ABS,
328            },
329        ];
330
331        for sub in test_data {
332            let res = Sub::new(sub.clone().tokens, sub.clone().method);
333            assert_eq!(res, sub)
334        }
335    }
336
337    #[test]
338    fn empty() {
339        let test_data: Vec<Sub> = vec![Sub {
340            tokens: Vec::new(),
341            method: SubMethod::PAREN,
342        }];
343
344        for sub in test_data {
345            let res = Sub::empty();
346            assert_eq!(res, sub)
347        }
348    }
349
350    #[test]
351    fn to_submethod() {
352        assert_eq!(TokenType::LABS.to_submethod(), SubMethod::ABS);
353        assert_eq!(TokenType::RABS.to_submethod(), SubMethod::ABS);
354        assert_eq!(TokenType::LPAREN.to_submethod(), SubMethod::PAREN);
355        assert_eq!(TokenType::RPAREN.to_submethod(), SubMethod::PAREN);
356    }
357
358    #[test]
359    fn new() {
360        let test_data: Vec<Token> = vec![
361            Token {
362                typ: TokenType::PLUS,
363                literal: String::from("+"),
364                sub: Sub::empty(),
365                index: (0, 0),
366            },
367            Token {
368                typ: TokenType::MINUS,
369                literal: String::from("-"),
370                sub: Sub::empty(),
371                index: (1, 1),
372            },
373            Token {
374                typ: TokenType::DIVIDE,
375                literal: String::from("/"),
376                sub: Sub::empty(),
377                index: (2, 2),
378            },
379            Token {
380                typ: TokenType::SUBEXP,
381                literal: String::from(""),
382                sub: Sub::new(
383                    Vec::from([
384                        Token::from(String::from("2"), (0, 0)),
385                        Token::from(String::from("+"), (1, 1)),
386                        Token::from(String::from("5"), (2, 2)),
387                    ]),
388                    SubMethod::PAREN,
389                ),
390                index: (0, 2),
391            },
392        ];
393
394        for t in test_data {
395            let res = Token::new(
396                t.clone().typ,
397                t.clone().literal,
398                t.clone().sub,
399                t.clone().index,
400            );
401
402            assert_eq!(res.typ, t.clone().typ);
403            assert_eq!(res.literal, t.clone().literal);
404            assert_eq!(res.sub, t.clone().sub);
405            assert_eq!(res.index, t.clone().index);
406        }
407    }
408
409    #[test]
410    fn new_sub() {
411        let test_data: HashMap<Vec<String>, Token> = HashMap::from([
412            (
413                vec![String::from("4"), String::from("+"), String::from("2")],
414                Token {
415                    typ: TokenType::SUBEXP,
416                    literal: String::new(),
417                    sub: Sub::new(
418                        Vec::from([
419                            Token::from(String::from("4"), (0, 0)),
420                            Token::from(String::from("+"), (0, 0)),
421                            Token::from(String::from("2"), (0, 0)),
422                        ]),
423                        SubMethod::PAREN,
424                    ),
425                    index: Token::unknown_index(),
426                },
427            ),
428            (
429                vec![String::from("2"), String::from("+"), String::from("+")],
430                Token {
431                    typ: TokenType::SUBEXP,
432                    literal: String::new(),
433                    sub: Sub::new(
434                        Vec::from([
435                            Token::from(String::from("2"), (0, 0)),
436                            Token::from(String::from("+"), (0, 0)),
437                            Token::from(String::from("+"), (0, 0)),
438                        ]),
439                        SubMethod::PAREN,
440                    ),
441                    index: Token::unknown_index(),
442                },
443            ),
444        ]);
445
446        for (t, expected) in test_data {
447            let tokens = t.into_iter().map(|tt| Token::from(tt, (0, 0))).collect();
448            let res = Token::new_sub(tokens, SubMethod::PAREN);
449
450            assert_eq!(res.typ, expected.clone().typ);
451            assert_eq!(res.literal, expected.clone().literal);
452            assert_eq!(res.sub, expected.clone().sub);
453            assert_eq!(res.index, expected.clone().index);
454        }
455    }
456
457    #[test]
458    fn new_pointer() {
459        let test_data: HashMap<usize, Token> = HashMap::from([
460            (
461                0,
462                Token::new(
463                    TokenType::POINTER,
464                    String::from("0"),
465                    Sub::new(Vec::new(), SubMethod::PAREN),
466                    (-1, -1),
467                ),
468            ),
469            (
470                99,
471                Token::new(
472                    TokenType::POINTER,
473                    String::from("99"),
474                    Sub::new(Vec::new(), SubMethod::ABS),
475                    (-1, -1),
476                ),
477            ),
478        ]);
479
480        for (i, expected) in test_data {
481            let token: Token = Token::new_pointer(i, expected.clone().sub.method);
482            assert_eq!(token, expected);
483        }
484    }
485
486    #[test]
487    fn from() {
488        let test_data: HashMap<(String, (i32, i32)), Token> = HashMap::from([
489            (
490                (String::from("42"), (0, 1)),
491                Token::new(TokenType::NUMBER, String::from("42"), Sub::empty(), (0, 1)),
492            ),
493            (
494                (String::from("}"), (0, 0)),
495                Token::new(TokenType::ILLEGAL, String::from("}"), Sub::empty(), (0, 0)),
496            ),
497            (
498                (String::from("+"), (0, 0)),
499                Token::new(TokenType::PLUS, String::from("+"), Sub::empty(), (0, 0)),
500            ),
501            (
502                (String::from("-"), (0, 0)),
503                Token::new(TokenType::MINUS, String::from("-"), Sub::empty(), (0, 0)),
504            ),
505            (
506                (String::from("*"), (0, 0)),
507                Token::new(TokenType::PRODUCT, String::from("*"), Sub::empty(), (0, 0)),
508            ),
509            (
510                (String::from("•"), (0, 0)),
511                Token::new(TokenType::PRODUCT, String::from("•"), Sub::empty(), (0, 0)),
512            ),
513            (
514                (String::from("/"), (0, 0)),
515                Token::new(TokenType::DIVIDE, String::from("/"), Sub::empty(), (0, 0)),
516            ),
517            (
518                (String::from(":"), (0, 0)),
519                Token::new(TokenType::DIVIDE, String::from(":"), Sub::empty(), (0, 0)),
520            ),
521            (
522                (String::from("%"), (0, 0)),
523                Token::new(
524                    TokenType::PERCENTAGE,
525                    String::from("%"),
526                    Sub::empty(),
527                    (0, 0),
528                ),
529            ),
530        ]);
531
532        for (v, expected) in test_data {
533            let res = Token::from(v.0, v.1);
534            assert_eq!(res, expected);
535        }
536    }
537
538    #[test]
539    fn unknown_index() {
540        assert_eq!(Token::unknown_index(), (-1, -1));
541    }
542
543    #[test]
544    fn take_pointer_index() {
545        let test_data: HashMap<Option<usize>, Token> = HashMap::from([
546            (None, Token::from(String::from("25"), (0, 1))),
547            (None, Token::from(String::from("-"), (0, 0))),
548            (Some(0), Token::new_pointer(0, SubMethod::PAREN)),
549            (Some(9), Token::new_pointer(9, SubMethod::PAREN)),
550        ]);
551
552        for (expected, token) in test_data {
553            assert_eq!(expected, token.take_pointer_index());
554        }
555    }
556
557    #[test]
558    fn is_illegal() {
559        let test_data: HashMap<bool, Token> = HashMap::from([
560            (false, Token::from(String::from("-25"), (0, 1))),
561            (false, Token::from(String::from("-"), (0, 0))),
562            (true, Token::from(String::from("}"), (0, 0))),
563            (true, Token::from(String::from("|"), (0, 0))),
564        ]);
565
566        for (expected, token) in test_data {
567            assert_eq!(expected, token.is_illegal());
568        }
569    }
570
571    #[test]
572    fn is_lparen() {
573        let test_data: HashMap<bool, Token> = HashMap::from([
574            (false, Token::from(String::from("-25"), (0, 1))),
575            (false, Token::from(String::from("-"), (0, 0))),
576            (false, Token::from(String::from(")"), (0, 0))),
577            (true, Token::from(String::from("("), (0, 0))),
578        ]);
579
580        for (expected, token) in test_data {
581            assert_eq!(expected, token.is_lparen());
582        }
583    }
584
585    #[test]
586    fn is_rparen() {
587        let test_data: HashMap<bool, Token> = HashMap::from([
588            (false, Token::from(String::from("-25"), (0, 1))),
589            (false, Token::from(String::from("-"), (0, 0))),
590            (false, Token::from(String::from("("), (0, 0))),
591            (true, Token::from(String::from(")"), (0, 0))),
592        ]);
593
594        for (expected, token) in test_data {
595            assert_eq!(expected, token.is_rparen());
596        }
597    }
598
599    #[test]
600    fn is_pointer() {
601        let test_data: HashMap<bool, Token> = HashMap::from([
602            (false, Token::from(String::from("-25"), (0, 1))),
603            (false, Token::from(String::from("-"), (0, 0))),
604            (false, Token::from(String::from("("), (0, 0))),
605            (true, Token::new_pointer(0, SubMethod::PAREN)),
606        ]);
607
608        for (expected, token) in test_data {
609            assert_eq!(expected, token.is_pointer());
610        }
611    }
612
613    #[test]
614    fn is_sub_exp() {
615        let test_data: HashMap<bool, Token> = HashMap::from([
616            (false, Token::from(String::from("-25"), (0, 1))),
617            (false, Token::from(String::from("-"), (0, 0))),
618            (false, Token::from(String::from("("), (0, 0))),
619            (true, Token::new_sub(vec![], SubMethod::PAREN)),
620        ]);
621
622        for (expected, token) in test_data {
623            assert_eq!(expected, token.is_sub_exp());
624        }
625    }
626
627    #[test]
628    fn is_power() {
629        let test_data: HashMap<bool, Token> = HashMap::from([
630            (false, Token::from(String::from("-25"), (0, 1))),
631            (false, Token::from(String::from("-"), (0, 0))),
632            (false, Token::from(String::from("("), (0, 0))),
633            (true, Token::from(String::from("^"), (0, 0))),
634        ]);
635
636        for (expected, token) in test_data {
637            assert_eq!(expected, token.is_power());
638        }
639    }
640
641    #[test]
642    fn is_labs() {
643        let test_data: HashMap<bool, Token> = HashMap::from([
644            (false, Token::from(String::from("-25"), (0, 1))),
645            (false, Token::from(String::from("-"), (0, 0))),
646            (false, Token::from(String::from("]"), (0, 0))),
647            (true, Token::from(String::from("["), (0, 0))),
648        ]);
649
650        for (expected, token) in test_data {
651            assert_eq!(expected, token.is_labs());
652        }
653    }
654
655    #[test]
656    fn is_rabs() {
657        let test_data: HashMap<bool, Token> = HashMap::from([
658            (false, Token::from(String::from("-25"), (0, 1))),
659            (false, Token::from(String::from("-"), (0, 0))),
660            (false, Token::from(String::from("["), (0, 0))),
661            (true, Token::from(String::from("]"), (0, 0))),
662        ]);
663
664        for (expected, token) in test_data {
665            assert_eq!(expected, token.is_rabs());
666        }
667    }
668
669    #[test]
670    fn matchto() {
671        let test_data: HashMap<bool, (Token, Token)> = HashMap::from([
672            (
673                true,
674                (
675                    Token::from(String::from("("), (0, 0)),
676                    Token::from(String::from(")"), (0, 0)),
677                ),
678            ),
679            (
680                true,
681                (
682                    Token::from(String::from("["), (0, 0)),
683                    Token::from(String::from("]"), (0, 0)),
684                ),
685            ),
686            (
687                false,
688                (
689                    Token::from(String::from("0"), (0, 0)),
690                    Token::from(String::from("1"), (0, 0)),
691                ),
692            ),
693        ]);
694
695        for (expected, tokens) in test_data {
696            assert_eq!(expected, tokens.0.matchto(&tokens.1));
697        }
698    }
699}
mate_rs/token.rs

mate_rs/
token.rs
