c-lexer-stable 0.1.4

C lexer
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109

use crate::token::Token;
use internship::IStr;
use std::str;

const KEYWORDS: phf::Map<&'static str, Token> = phf_map! {
    "auto" => Token::AUTO,
    "break" => Token::BREAK,
    "case" => Token::CASE,
    "char" => Token::CHAR,
    "const" => Token::CONST,
    "continue"=> Token::CONTINUE,
    "default" => Token::DEFAULT,
    "do" => Token::DO,
    "double" => Token::DOUBLE,
    "else" => Token::ELSE,
    "enum" => Token::ENUM,
    "extern" => Token::EXTERN,
    "float" => Token::FLOAT,
    "for" => Token::FOR,
    "goto" => Token::GOTO,
    "if" => Token::IF,
    "inline" => Token::INLINE,
    "int" => Token::INT,
    "long" => Token::LONG,
    "register" => Token::REGISTER,
    "restrict" => Token::RESTRICT,
    "return" => Token::RETURN,
    "short" => Token::SHORT,
    "signed" => Token::SIGNED,
    "sizeof" => Token::SIZEOF,
    "static" => Token::STATIC,
    "struct" => Token::STRUCT,
    "switch" => Token::SWITCH,
    "typedef" => Token::TYPEDEF,
    "union" => Token::UNION,
    "unsigned" => Token::UNSIGNED,
    "void" => Token::VOID,
    "volatile" => Token::VOLATILE,
    "while" => Token::WHILE,
    "_Alignas" => Token::ALIGNAS,
    "_Alignof" => Token::ALIGNOF,
    "_Atomic" => Token::ATOMIC,
    "_Bool" => Token::BOOL,
    "_Complex" => Token::COMPLEX,
    "_Generic" => Token::GENERIC,
    "_Imaginary" => Token::IMAGINARY,
    "_Noreturn" => Token::NORETURN,
    "_Static_assert" => Token::StaticAssert,
    "_Thread_local" => Token::ThreadLocal,
    "__func__" => Token::FuncName,
};

#[inline]
fn is_identifier_part(cp: u8) -> bool {
    cp == 0x24
        || cp == 0x5F
        || (cp >= 0x41 && cp <= 0x5A)
        || (cp >= 0x61 && cp <= 0x7A)
        || (cp >= 0x30 && cp <= 0x39)
        || cp == 0x5C
        || cp >= 0x80
}

#[inline]
pub fn parse_identifier(input: &[u8], c_src: &mut usize) -> Token {
    let mut it = 0;
    for i in 0..input.len() - *c_src {
        if !unsafe { is_identifier_part(*input.get_unchecked(*c_src + i)) } {
            it = i;
            break;
        }
    }
    let ident = &input[*c_src - 1..*c_src + it];
    *c_src += it;
    let ident = unsafe { str::from_utf8_unchecked(ident) };
    KEYWORDS
        .get(ident)
        .cloned()
        .unwrap_or_else(|| Token::Identifier(IStr::new(ident)))
}

#[cfg(test)]
mod tests {
    use super::super::*;
    use ::internship::IStr;

    should!(
        parsing_after_token,
        "*=a ",
        vec![
            Token::MulAssign,
            Token::Identifier(IStr::new("a")),
            Token::EOF
        ]
    );

    should!(
        parsing_after_token_2,
        "a( a1 ",
        vec![
            Token::Identifier(IStr::new("a")),
            Token::LParen,
            Token::Identifier(IStr::new("a1")),
            Token::EOF
        ]
    );

    should!(keyword, "auto ", vec![Token::AUTO, Token::EOF]);
}