clex 0.1.0

Fast C-lang lexer (library)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145

#![doc = include_str!("../README.md")]

mod char;
mod comment;
mod float;
mod int;
mod keyword;
mod lexer;
mod string;

pub use float::Float;
pub use int::Int;
pub use keyword::Keyword;
pub use lexer::Token;

/// C lexeme
#[derive(Debug, Clone)]
pub struct Lexeme<'l> {
    /// Token kind
    pub token: Token,
    /* TODO:
    /// Position in source code
    pub location: Location,
    */
    pub span: core::ops::Range<usize>,
    /// String slice
    pub slice: &'l str,
}

impl<'l> core::ops::Deref for Lexeme<'l> {
    type Target = Token;

    fn deref(&self) -> &Self::Target {
        &self.token
    }
}

impl<'l> AsRef<str> for Lexeme<'l> {
    fn as_ref(&self) -> &'l str {
        self.slice
    }
}

impl<'l> Lexeme<'l> {
    /// Extract keyword
    pub fn keyword(&self) -> Option<Keyword> {
        if self.token == Token::Identifier {
            self.slice.parse().ok()
        } else {
            None
        }
    }

    /// Extract text from comment
    pub fn comment(&self) -> Option<String> {
        if self.token == Token::Comment {
            comment::extract(self.slice)
        } else {
            None
        }
    }

    /// Extract text from character literal
    pub fn char(&self) -> Option<char> {
        if self.token == Token::Char {
            char::extract(self.slice)
        } else {
            None
        }
    }

    /// Extract text from string literal
    pub fn string(&self) -> Option<String> {
        if self.token == Token::String {
            string::extract(self.slice)
        } else {
            None
        }
    }

    /// Extract number from integer literal
    pub fn int<T: Int>(&self) -> Option<T> {
        if self.token == Token::Int {
            int::extract(self.slice)
        } else {
            None
        }
    }

    /// Extract number from floating-point literal
    pub fn float<T: Float>(&self) -> Option<T> {
        if self.token == Token::Float {
            float::extract(self.slice)
        } else {
            None
        }
    }
}

/* TODO:
/// C token location
#[derive(Debug, Clone)]
pub struct Location {
    /// Source code position
    pub point: u32,
    /// Source code line
    pub line: u32,
    /// Source code column
    pub column: u32,
}
*/

/// C Lexer
pub struct Lexer<'l> {
    inner: logos::Lexer<'l, Token>,
}

impl<'l> From<&'l str> for Lexer<'l> {
    fn from(s: &'l str) -> Self {
        Self {
            inner: logos::Lexer::new(s),
        }
    }
}

impl<'l> Iterator for Lexer<'l> {
    type Item = Lexeme<'l>;

    fn next(&mut self) -> Option<Self::Item> {
        self.inner.next().map(|token| {
            let span = self.inner.span();
            let slice = self.inner.slice();
            Lexeme { token, span, slice }
        })
    }
}

#[cfg(test)]
mod tests {
    #[test]
    fn it_works() {
        let result = 2 + 2;
        assert_eq!(result, 4);
    }
}