chomatter_lexer 0.0.1

use chomatter_commons::{walker::Walker, EGenericToken, EGenericToken::*, GenericToken, Segment};

const CASE: &str = "case";
const FUNCTION: &str = "function";

const IMPORT: &str = "import";
const JUMP: &str = "jump";
const CALL: &str = "call";
const GOTO: &str = "goto";

const NEW: &str = "new";
const RETURN: &str = "return";
const BREAK: &str = "break";
const YIELD: &str = "yield";
const CONTINUE: &str = "continue";
const LET: &str = "let";
const VAR: &str = "var";
const INPUT: &str = "input";

const DEFAULT: &str = "default";
const ARROW: &str = "->";

const IF: &str = "if";
const ELSE_IF: &str = "else if";
const ELSE: &str = "else";
const WHILE: &str = "while";
const THROW: &str = "throw";

const BOOL_TRUES: [&str; 3] = ["true", "yes", "on"];
const BOOL_FALSES: [&str; 3] = ["false", "no", "off"];

const DO: &str = "do";

const CUSTOM_IDENTIFIER_CHARS: &str = "@_";

const SPACE: char = ' ';
const COLON: char = ':';
const COMMA: char = ',';
const END_LINE: char = '\n';
const INDENT: &str = "  ";
const DECIMAL_SEPARATOR: char = '.';

const OPEN_PARENTHESIS: char = '(';
const CLOSE_PARENTHESIS: char = ')';
const OPEN_BRACKET: char = '[';
const CLOSE_BRACKET: char = ']';
const OPEN_BRACE: char = '{';
const CLOSE_BRACE: char = '}';

const INC_ASSIGN: &str = "+=";
const DEC_ASSIGN: &str = "-=";
const MUL_ASSIGN: &str = "*=";
const DIV_ASSIGN: &str = "/=";
const MOD_ASSIGN: &str = "%=";
const ASSIGN: &str = "=";

const PLUS: &str = "+";
const MINUS: &str = "-";
const NEGATIVE: &str = "-";
const MULTIPLY: &str = "*";
const DIVIDE: &str = "/";
const MODULO: &str = "%";

const EQUAL: &str = "==";
const LESS: &str = "<";
const LESS_EQ: &str = "<=";
const GREATER: &str = ">";
const GREATER_EQ: &str = ">=";
const AND: &str = "&&";
const OR: &str = "||";

const NOT: &str = "!";

const RELATIONS: [(&str, EGenericToken); 20] = [
    (INC_ASSIGN, Assign),   // +=
    (DEC_ASSIGN, Assign),   // -=
    (MUL_ASSIGN, Assign),   // *=
    (DIV_ASSIGN, Assign),   // /=
    (MOD_ASSIGN, Assign),   // %=
    (PLUS, Operator),       // +
    (MINUS, Operator),      // -
    (MULTIPLY, Operator),   // *
    (DIVIDE, Operator),     // /
    (MODULO, Operator),     // %
    (EQUAL, Operator),      // ==
    (LESS_EQ, Operator),    // <=
    (LESS, Operator),       // <
    (GREATER_EQ, Operator), // >=
    (GREATER, Operator),    // >
    (AND, Operator),        // &&
    (OR, Operator),         // ||
    (NOT, Unary),           // !
    (NEGATIVE, Unary),      // -
    (ASSIGN, Assign),       // =
];

const DOT: char = '.';

const LITERAL_STRING_CHAR: char = '\'';
const BEGIN_TEMPLATE_STRING: char = '"';
const BEGIN_STRING_EXPR_PATTERN: &str = "#{"; // TODO: Use it
const END_STRING_EXPR_PATTERN: char = '}';
const ESCAPE_CHAR: char = '\\';
const END_TEMPLATE_STRING: char = '"';

const LINE_COMMENT_CHAR: char = '#';
const BEGIN_MULTILINE_COMMENT_PATTERN: &str = "###";
const END_MULTILINE_COMMENT_PATTERN: &str = "###";

pub trait Tokerizer {
    fn resolve(&mut self) -> Vec<GenericToken>;
    fn resolve_string_expr(&mut self) -> Vec<GenericToken>;
}

impl Tokerizer for Walker<char> {
    fn resolve(&mut self) -> Vec<GenericToken> {
        let mut tokens = Vec::new();
        loop {
            if self.position() == self.count() {
                break;
            }

            if self.match_char(END_LINE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::EndOfLine,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(COLON) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Colon,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(SPACE) {
                if let Some(last_token) = tokens.last() {
                    if last_token.kind == EGenericToken::EndOfLine {
                        while self.match_string(INDENT) {
                            tokens.push(GenericToken {
                                kind: EGenericToken::Indent,
                                segment: Segment::new(self.position(), INDENT.len()),
                            });
                            self.advance_by(INDENT.len());

                            if self.is_eof() {
                                break;
                            }
                        }
                        continue;
                    }
                }

                while let Some(c) = self.next() {
                    if *c == SPACE {
                        tokens.push(GenericToken {
                            kind: EGenericToken::Space,
                            segment: Segment::new(self.position() - 1, 1),
                        });
                    } else {
                        self.prev();
                        break;
                    }
                }
            } else if self.match_string(NEW) {
                tokens.push(GenericToken {
                    kind: EGenericToken::New,
                    segment: Segment::new(self.position(), NEW.len()),
                });
                self.advance_by(NEW.len());
            } else if self.match_string(ARROW) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Arrow,
                    segment: Segment::new(self.position(), ARROW.len()),
                });
                self.advance_by(ARROW.len());
            } else if self.match_char(COMMA) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Comma,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(OPEN_PARENTHESIS) {
                tokens.push(GenericToken {
                    kind: EGenericToken::LeftParen,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(CLOSE_PARENTHESIS) {
                tokens.push(GenericToken {
                    kind: EGenericToken::RightParen,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(OPEN_BRACKET) {
                tokens.push(GenericToken {
                    kind: EGenericToken::LeftBracket,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(CLOSE_BRACKET) {
                tokens.push(GenericToken {
                    kind: EGenericToken::RightBracket,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(OPEN_BRACE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::LeftBrace,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(CLOSE_BRACE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::RightBrace,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_mapper(&RELATIONS) {
                let index = self.match_mapper_index(&RELATIONS);
                let item = RELATIONS[index as usize];
                let len = item.0.len();

                tokens.push(GenericToken {
                    kind: item.1,
                    segment: Segment::new(self.position(), len),
                });
                self.advance_by(len);
            } else if self.match_char(DOT) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Dot,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_string(CASE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Case,
                    segment: Segment::new(self.position(), CASE.len()),
                });
                self.advance_by(CASE.len());
            } else if self.match_string(IF) {
                tokens.push(GenericToken {
                    kind: EGenericToken::If,
                    segment: Segment::new(self.position(), IF.len()),
                });
                self.advance_by(IF.len());
            } else if self.match_string(ELSE_IF) {
                tokens.push(GenericToken {
                    kind: EGenericToken::ElseIf,
                    segment: Segment::new(self.position(), ELSE_IF.len()),
                });
                self.advance_by(ELSE_IF.len());
            } else if self.match_string(ELSE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Else,
                    segment: Segment::new(self.position(), ELSE.len()),
                });
                self.advance_by(ELSE.len());
            } else if self.match_string(WHILE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::While,
                    segment: Segment::new(self.position(), WHILE.len()),
                });
                self.advance_by(WHILE.len());
            } else if self.match_string(THROW) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Throw,
                    segment: Segment::new(self.position(), THROW.len()),
                });
                self.advance_by(THROW.len());
            } else if self.match_string(BEGIN_MULTILINE_COMMENT_PATTERN) {
                self.advance_by(BEGIN_MULTILINE_COMMENT_PATTERN.len());

                while !self.match_string(END_MULTILINE_COMMENT_PATTERN) {
                    if self.is_eof() {
                        panic!("Unterminated multiline comment");
                    }

                    self.next();
                }

                self.advance_by(END_MULTILINE_COMMENT_PATTERN.len());
            } else if self.match_char(LINE_COMMENT_CHAR) {
                let mark = self.position();
                while let Some(c) = self.next() {
                    if *c == END_LINE {
                        self.prev();
                        break;
                    }
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::LineComment,
                    segment: Segment::new(mark, self.position() - mark),
                });
            } else if self.match_char(LITERAL_STRING_CHAR) {
                let mark = self.position();
                self.advance_by(1);
                let mut escaped = false;
                while let Some(c) = self.next() {
                    if escaped {
                        escaped = false;
                    } else if *c == LITERAL_STRING_CHAR {
                        break;
                    } else if *c == ESCAPE_CHAR {
                        escaped = true;
                    }
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::StringLiteral,
                    segment: Segment::new(mark, self.position() - mark),
                });
            } else if self.match_char(BEGIN_TEMPLATE_STRING) {
                let mut mark = self.position();
                self.advance_by(1);
                let mut escaped = false;
                tokens.push(GenericToken {
                    kind: EGenericToken::BeginTemplateString,
                    segment: Segment::new(mark, 1),
                });

                loop {
                    if self.is_eof() {
                        let segment = self
                            .segment(mark, self.position())
                            .into_iter()
                            .collect::<String>();
                        panic!("Unterminated template string\n{}", segment);
                    }

                    let current = self.current();
                    if escaped {
                        escaped = false;
                    } else if *current == END_TEMPLATE_STRING {
                        tokens.push(GenericToken {
                            kind: EGenericToken::StringPart,
                            segment: Segment::new(mark, self.position() - mark + 1),
                        });
                        break;
                    } else if *current == ESCAPE_CHAR {
                        escaped = true;
                    } else if self.match_string(BEGIN_STRING_EXPR_PATTERN) {
                        tokens.push(GenericToken {
                            kind: EGenericToken::StringPart,
                            segment: Segment::new(
                                mark,
                                self.position() - mark + BEGIN_STRING_EXPR_PATTERN.len(),
                            ),
                        });
                        self.advance_by(BEGIN_STRING_EXPR_PATTERN.len());

                        let mut expr_tokens = self.resolve_string_expr();
                        tokens.append(&mut expr_tokens);

                        mark = self.position();
                    }

                    self.advance_by(1);
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::EndTemplateString,
                    segment: Segment::new(self.position(), 1),
                });
                self.advance_by(1);
            } else if self.match_string(DO) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Do,
                    segment: Segment::new(self.position(), DO.len()),
                });
                self.advance_by(DO.len());
            } else if self.match_string(FUNCTION) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Function,
                    segment: Segment::new(self.position(), FUNCTION.len()),
                });
                self.advance_by(FUNCTION.len());
            } else if self.current().is_numeric() {
                let mark = self.position();
                while let Some(c) = self.next() {
                    if !c.is_numeric() && *c != DECIMAL_SEPARATOR {
                        self.prev();
                        break;
                    }
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::NumberLiteral,
                    segment: Segment::new(mark, self.position() - mark),
                });
            } else if self.match_strings(&BOOL_TRUES) || self.match_strings(&BOOL_FALSES) {
                for item in BOOL_TRUES.iter().chain(BOOL_FALSES.iter()) {
                    if self.match_string(item) {
                        tokens.push(GenericToken {
                            kind: EGenericToken::BooleanLiteral,
                            segment: Segment::new(self.position(), item.len()),
                        });
                        self.advance_by(item.len());
                        break;
                    }
                }
            } else if self.match_string(INPUT) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Input,
                    segment: Segment::new(self.position(), INPUT.len()),
                });
                self.advance_by(INPUT.len());
            } else if self.match_string(GOTO) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Goto,
                    segment: Segment::new(self.position(), GOTO.len()),
                });
                self.advance_by(GOTO.len());
            } else if self.match_string(JUMP) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Jump,
                    segment: Segment::new(self.position(), JUMP.len()),
                });
                self.advance_by(JUMP.len());
            } else if self.match_string(IMPORT) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Import,
                    segment: Segment::new(self.position(), IMPORT.len()),
                });

                self.advance_by(IMPORT.len());
            } else if self.match_string(CALL) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Call,
                    segment: Segment::new(self.position(), CALL.len()),
                });
                self.advance_by(CALL.len());
            } else if self.match_string(RETURN) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Return,
                    segment: Segment::new(self.position(), RETURN.len()),
                });
                self.advance_by(RETURN.len());
            } else if self.match_string(BREAK) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Break,
                    segment: Segment::new(self.position(), BREAK.len()),
                });
                self.advance_by(BREAK.len());
            } else if self.match_string(CONTINUE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Continue,
                    segment: Segment::new(self.position(), CONTINUE.len()),
                });
                self.advance_by(CONTINUE.len());
            } else if self.match_string(YIELD) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Yield,
                    segment: Segment::new(self.position(), YIELD.len()),
                });
                self.advance_by(YIELD.len());
            } else if self.match_string(DEFAULT) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Default,
                    segment: Segment::new(self.position(), DEFAULT.len()),
                });
                self.advance_by(DEFAULT.len());
            } else if self.match_string(LET) {
                tokens.push(GenericToken {
                    kind: EGenericToken::ScopedVar,
                    segment: Segment::new(self.position(), LET.len()),
                });
                self.advance_by(LET.len());
            } else if self.match_string(VAR) {
                tokens.push(GenericToken {
                    kind: EGenericToken::GlobalVar,
                    segment: Segment::new(self.position(), VAR.len()),
                });
                self.advance_by(VAR.len());
            } else {
                // Identifier
                let mark = self.position();
                let mut cloned = self.clone();
                if let Some(c) = self.next() {
                    if !c.is_ascii_alphabetic() && !CUSTOM_IDENTIFIER_CHARS.contains(*c) {
                        let pos = cloned.position() + 1;
                        cloned.goto(chomatter_commons::walker::EGoto::Begin);
                        panic!(
                            "Unexpected identifier character: {}\nReaded: {}\nTokens:\n {:?}",
                            c,
                            cloned.into_iter().take(pos).collect::<String>().to_string(),
                            tokens
                        );
                    }
                }
                while let Some(c) = self.next() {
                    if !c.is_ascii_alphanumeric() && !CUSTOM_IDENTIFIER_CHARS.contains(*c) {
                        self.prev();
                        break;
                    }
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::Identifier,
                    segment: Segment::new(mark, self.position() - mark),
                });
            }
        }

        return tokens;
    }

    fn resolve_string_expr(&mut self) -> Vec<GenericToken> {
        let mut tokens = Vec::new();
        loop {
            if self.position() == self.count() {
                break;
            }
            if self.match_char(END_STRING_EXPR_PATTERN) {
                break;
            } else if self.match_char(END_LINE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::EndOfLine,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(COLON) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Colon,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(SPACE) {
                if let Some(last_token) = tokens.last() {
                    if last_token.kind == EGenericToken::EndOfLine {
                        while self.match_string(INDENT) {
                            tokens.push(GenericToken {
                                kind: EGenericToken::Indent,
                                segment: Segment::new(self.position(), INDENT.len()),
                            });
                            self.advance_by(INDENT.len());

                            if self.is_eof() {
                                break;
                            }
                        }
                        continue;
                    }
                }

                while let Some(c) = self.next() {
                    if *c == SPACE {
                        tokens.push(GenericToken {
                            kind: EGenericToken::Space,
                            segment: Segment::new(self.position() - 1, 1),
                        });
                    } else {
                        self.prev();
                        break;
                    }
                }
            } else if self.match_string(ARROW) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Arrow,
                    segment: Segment::new(self.position(), ARROW.len()),
                });
                self.advance_by(ARROW.len());
            } else if self.match_char(COMMA) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Comma,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(OPEN_PARENTHESIS) {
                tokens.push(GenericToken {
                    kind: EGenericToken::LeftParen,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(CLOSE_PARENTHESIS) {
                tokens.push(GenericToken {
                    kind: EGenericToken::RightParen,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(OPEN_BRACKET) {
                tokens.push(GenericToken {
                    kind: EGenericToken::LeftBracket,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(CLOSE_BRACKET) {
                tokens.push(GenericToken {
                    kind: EGenericToken::RightBracket,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(OPEN_BRACE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::LeftBrace,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_char(CLOSE_BRACE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::RightBrace,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_mapper(&RELATIONS) {
                let index = self.match_mapper_index(&RELATIONS);
                let item = RELATIONS[index as usize];
                let len = item.0.len();

                tokens.push(GenericToken {
                    kind: item.1,
                    segment: Segment::new(self.position(), len),
                });
                self.advance_by(len);
            } else if self.match_char(DOT) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Dot,
                    segment: Segment::new(self.position(), 1),
                });
                self.next();
            } else if self.match_string(CASE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Case,
                    segment: Segment::new(self.position(), CASE.len()),
                });
                self.advance_by(CASE.len());
            } else if self.match_string(IF) {
                tokens.push(GenericToken {
                    kind: EGenericToken::If,
                    segment: Segment::new(self.position(), IF.len()),
                });
                self.advance_by(IF.len());
            } else if self.match_string(ELSE_IF) {
                tokens.push(GenericToken {
                    kind: EGenericToken::ElseIf,
                    segment: Segment::new(self.position(), ELSE_IF.len()),
                });
                self.advance_by(ELSE_IF.len());
            } else if self.match_string(ELSE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Else,
                    segment: Segment::new(self.position(), ELSE.len()),
                });
                self.advance_by(ELSE.len());
            } else if self.match_string(WHILE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::While,
                    segment: Segment::new(self.position(), WHILE.len()),
                });
                self.advance_by(WHILE.len());
            } else if self.match_string(THROW) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Throw,
                    segment: Segment::new(self.position(), THROW.len()),
                });
                self.advance_by(THROW.len());
            } else if self.match_string(BEGIN_MULTILINE_COMMENT_PATTERN) {
                self.advance_by(BEGIN_MULTILINE_COMMENT_PATTERN.len());

                while !self.match_string(END_MULTILINE_COMMENT_PATTERN) {
                    if self.is_eof() {
                        panic!("Unterminated multiline comment");
                    }

                    self.next();
                }

                self.advance_by(END_MULTILINE_COMMENT_PATTERN.len());
            } else if self.match_char(LINE_COMMENT_CHAR) {
                let mark = self.position();
                while let Some(c) = self.next() {
                    if *c == END_LINE {
                        self.prev();
                        break;
                    }
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::LineComment,
                    segment: Segment::new(mark, self.position() - mark),
                });
            } else if self.match_char(LITERAL_STRING_CHAR) {
                let mark = self.position();
                self.advance_by(1);
                let mut escaped = false;
                while let Some(c) = self.next() {
                    if escaped {
                        escaped = false;
                    } else if *c == LITERAL_STRING_CHAR {
                        break;
                    } else if *c == ESCAPE_CHAR {
                        escaped = true;
                    }
                }
                self.advance_by(1);
                tokens.push(GenericToken {
                    kind: EGenericToken::StringLiteral,
                    segment: Segment::new(mark, self.position() - mark),
                });
            } else if self.match_char(BEGIN_TEMPLATE_STRING) {
                let mut mark = self.position();
                self.advance_by(1);
                let mut escaped = false;
                tokens.push(GenericToken {
                    kind: EGenericToken::BeginTemplateString,
                    segment: Segment::new(mark, 1),
                });

                loop {
                    if self.is_eof() {
                        let segment = self
                            .segment(mark, self.position())
                            .into_iter()
                            .collect::<String>();
                        panic!("Unterminated template string\n{}", segment);
                    }

                    let current = self.current();
                    if escaped {
                        escaped = false;
                    } else if *current == END_TEMPLATE_STRING {
                        tokens.push(GenericToken {
                            kind: EGenericToken::StringPart,
                            segment: Segment::new(mark, self.position() - mark + 1),
                        });
                        break;
                    } else if *current == ESCAPE_CHAR {
                        escaped = true;
                    } else if self.match_string(BEGIN_STRING_EXPR_PATTERN) {
                        tokens.push(GenericToken {
                            kind: EGenericToken::StringPart,
                            segment: Segment::new(
                                mark,
                                self.position() - mark + BEGIN_STRING_EXPR_PATTERN.len(),
                            ),
                        });
                        self.advance_by(BEGIN_STRING_EXPR_PATTERN.len());

                        let mut expr_tokens = self.resolve_string_expr();
                        tokens.append(&mut expr_tokens);

                        mark = self.position();
                    }

                    self.advance_by(1);
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::EndTemplateString,
                    segment: Segment::new(self.position(), 1),
                });
                self.advance_by(1);
            } else if self.match_string(DO) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Do,
                    segment: Segment::new(self.position(), DO.len()),
                });
                self.advance_by(DO.len());
            } else if self.match_string(FUNCTION) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Function,
                    segment: Segment::new(self.position(), FUNCTION.len()),
                });
                self.advance_by(FUNCTION.len());
            } else if self.current().is_numeric() {
                let mark = self.position();
                while let Some(c) = self.next() {
                    if !c.is_numeric() && *c != DECIMAL_SEPARATOR {
                        self.prev();
                        break;
                    }
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::NumberLiteral,
                    segment: Segment::new(mark, self.position() - mark),
                });
            } else if self.match_strings(&BOOL_TRUES) || self.match_strings(&BOOL_FALSES) {
                for item in BOOL_TRUES.iter().chain(BOOL_FALSES.iter()) {
                    if self.match_string(item) {
                        tokens.push(GenericToken {
                            kind: EGenericToken::BooleanLiteral,
                            segment: Segment::new(self.position(), item.len()),
                        });
                        self.advance_by(item.len());
                        break;
                    }
                }
            } else if self.match_string(GOTO) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Goto,
                    segment: Segment::new(self.position(), GOTO.len()),
                });
                self.advance_by(GOTO.len());
            } else if self.match_string(JUMP) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Jump,
                    segment: Segment::new(self.position(), JUMP.len()),
                });
                self.advance_by(JUMP.len());
            } else if self.match_string(IMPORT) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Import,
                    segment: Segment::new(self.position(), IMPORT.len()),
                });

                self.advance_by(IMPORT.len());
            } else if self.match_string(CALL) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Call,
                    segment: Segment::new(self.position(), CALL.len()),
                });
                self.advance_by(CALL.len());
            } else if self.match_string(RETURN) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Return,
                    segment: Segment::new(self.position(), RETURN.len()),
                });
                self.advance_by(RETURN.len());
            } else if self.match_string(BREAK) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Break,
                    segment: Segment::new(self.position(), BREAK.len()),
                });
                self.advance_by(BREAK.len());
            } else if self.match_string(CONTINUE) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Continue,
                    segment: Segment::new(self.position(), CONTINUE.len()),
                });
                self.advance_by(CONTINUE.len());
            } else if self.match_string(YIELD) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Yield,
                    segment: Segment::new(self.position(), YIELD.len()),
                });
                self.advance_by(YIELD.len());
            } else if self.match_string(DEFAULT) {
                tokens.push(GenericToken {
                    kind: EGenericToken::Default,
                    segment: Segment::new(self.position(), DEFAULT.len()),
                });
                self.advance_by(DEFAULT.len());
            } else if self.match_string(LET) {
                tokens.push(GenericToken {
                    kind: EGenericToken::ScopedVar,
                    segment: Segment::new(self.position(), LET.len()),
                });
                self.advance_by(LET.len());
            } else if self.match_string(VAR) {
                tokens.push(GenericToken {
                    kind: EGenericToken::GlobalVar,
                    segment: Segment::new(self.position(), VAR.len()),
                });
                self.advance_by(VAR.len());
            } else {
                // Identifier
                let mark = self.position();
                let mut cloned = self.clone();
                if let Some(c) = self.next() {
                    if !c.is_ascii_alphabetic() && !CUSTOM_IDENTIFIER_CHARS.contains(*c) {
                        let pos = cloned.position() + 1;
                        cloned.goto(chomatter_commons::walker::EGoto::Begin);
                        panic!(
                            "Unexpected identifier character: {}\nReaded: {}\nTokens:\n {:?}",
                            c,
                            cloned.into_iter().take(pos).collect::<String>().to_string(),
                            tokens
                        );
                    }
                }
                while let Some(c) = self.next() {
                    if !c.is_ascii_alphanumeric() && !CUSTOM_IDENTIFIER_CHARS.contains(*c) {
                        self.prev();
                        break;
                    }
                }
                tokens.push(GenericToken {
                    kind: EGenericToken::Identifier,
                    segment: Segment::new(mark, self.position() - mark),
                });
            }
        }

        return tokens;
    }
}