javascript_lexer 0.1.8

Javascript lexer
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

use crate::token::Token;
use internship::IStr;
use std::str;

const IDENTIFIERS: phf::Map<&'static str, Token> = phf_map! {
    "true" => Token::BoolLiteral(true),
    "false" => Token::BoolLiteral(false),
    "null" => Token::LNull,
    "undefined" => Token::LUndefined,
    "function" => Token::KFunction,
    "async" => Token::KAsync,
    "class" => Token::KClass,
    "let" => Token::KLet,
    "if" => Token::KIf,
    "else" => Token::KElse,
    "do" => Token::KDo,
    "while" => Token::KWhile,
    "for" => Token::KFor,
    "var" => Token::KVar,
    "const" => Token::KConst,
    "in" => Token::KIn,
    "of" => Token::KOf,
    "await" => Token::KAwait,
    "switch" => Token::KSwitch,
    "case" => Token::KCase,
    "default" => Token::KDefault,
    "continue" => Token::KContinue,
    "break" => Token::KBreak,
    "return" => Token::KReturn,
    "with" => Token::KWith,
    "throw" => Token::KThrow,
    "try" => Token::KTry,
    "catch" => Token::KCatch,
    "finally" => Token::KFinally,
    "debugger" => Token::KDebugger,
    "extend" => Token::KExtend,
    "static" => Token::KStatic,
    "get" => Token::KGet,
    "set" => Token::KSet,
    "this" => Token::KThis,
    "delete" => Token::KDelete,
    "void" => Token::KVoid,
    "typeof" => Token::KTypeof,
    "new" => Token::KNew,
    "import" => Token::KImport,
    "as" => Token::KAs,
    "from" => Token::KFrom,
};

#[inline]
fn is_identifier_part(cp: u8) -> bool {
    cp == 0x24
        || cp == 0x5F
        || (cp >= 0x41 && cp <= 0x5A)
        || (cp >= 0x61 && cp <= 0x7A)
        || (cp >= 0x30 && cp <= 0x39)
        || cp == 0x5C
        || cp >= 0x80
}

#[inline]
pub fn parse_identifier(input: &[u8], c_src: &mut usize) -> Token {
    let mut it = 0;
    for i in 0..input.len() - *c_src {
        if !unsafe { is_identifier_part(*input.get_unchecked(*c_src + i)) } {
            it = i;
            break;
        }
    }
    let ident = &input[*c_src - 1..*c_src + it];
    *c_src += it;
    let ident = unsafe { str::from_utf8_unchecked(ident) };
    IDENTIFIERS
        .get(ident)
        .cloned()
        .unwrap_or_else(|| Token::IdentifierName(IStr::new(ident)))
}

#[cfg(test)]
mod tests {
    use super::super::*;
    use ::internship::IStr;

    should!(
        parsing_after_token,
        "**=a ",
        vec![
            Token::DoubleStarAssign,
            Token::IdentifierName(IStr::new("a")),
            Token::EOF
        ]
    );

    should!(
        parsing_after_token_2,
        "a( a1 ",
        vec![
            Token::IdentifierName(IStr::new("a")),
            Token::LRound,
            Token::IdentifierName(IStr::new("a1")),
            Token::EOF
        ]
    );

    should!(keyword, "var ", vec![Token::KVar, Token::EOF]);
}