Enum rslua_march1917::tokens::TokenType

source ·

pub enum TokenType {
Show 62 variants    And,
    Break,
    Do,
    Else,
    ElseIf,
    End,
    False,
    For,
    Function,
    Goto,
    IDiv,
    If,
    In,
    Local,
    Nil,
    Not,
    Or,
    Repeat,
    Return,
    Then,
    True,
    Until,
    While,
    Concat,
    Dots,
    Eq,
    Ge,
    Le,
    Ne,
    Shl,
    Shr,
    DbColon,
    Eos,
    Flt,
    Int,
    Name,
    String,
    Lp,
    Rp,
    Ls,
    Rs,
    Lb,
    Rb,
    Add,
    Minus,
    Mul,
    Div,
    Mod,
    Pow,
    Len,
    Assign,
    Lt,
    Gt,
    BAnd,
    BOr,
    BXor,
    Colon,
    Comma,
    Semi,
    Attr,
    SComment,
    MComment,
}

Variants§

§

MComment

Implementations§

source §

impl TokenType

source

pub fn from_keyword(word: &str) -> Option<TokenType>

Examples found in repository ?

src/lexer.rs (line 612)

    fn read_other_tokens(&mut self, ctx: &mut Context) -> LexResult {
        if let Some(c) = ctx.get() {
            let token_type = match c {
                b'+' => Some(TokenType::Add),
                b'-' => Some(TokenType::Minus),
                b'*' => Some(TokenType::Mul),
                b'/' => Some(TokenType::Div),
                b'%' => Some(TokenType::Mod),
                b'^' => Some(TokenType::Pow),
                b'#' => Some(TokenType::Len),
                b'&' => Some(TokenType::BAnd),
                b'|' => Some(TokenType::BOr),
                b'(' => Some(TokenType::Lp),
                b')' => Some(TokenType::Rp),
                b'[' => Some(TokenType::Ls),
                b']' => Some(TokenType::Rs),
                b'{' => Some(TokenType::Lb),
                b'}' => Some(TokenType::Rb),
                b';' => Some(TokenType::Semi),
                b',' => Some(TokenType::Comma),
                _ => None,
            };

            if let Some(t) = token_type {
                ctx.next();
                return success!((t, TokenValue::None));
            } else if self.check_current_if(ctx, |c| Lexer::is_valid_name_start(c)) {
                let mut word: Vec<u8> = Vec::new();
                ctx.write_into(1, &mut word);
                while self.check_current_if(ctx, |c| Lexer::is_valid_name(c)) {
                    ctx.write_into(1, &mut word);
                }
                if let Ok(s) = str::from_utf8(&word) {
                    if let Some(t) = TokenType::from_keyword(s) {
                        return success!((t, TokenValue::None));
                    } else {
                        return success!((TokenType::Name, TokenValue::Str(s.to_string())));
                    }
                }
            } else {
                return lex_error!(self, ctx, &format!("unknown token near {}", c as char));
            }
        }
        unreachable!()
    }