skiff/lexer/
lex.rs

1use logos::{Lexer, Logos};
2
3fn string_token(lex: &mut Lexer<Token>) -> Option<String> {
4    let slice = lex.slice();
5    Some(slice[1..slice.len() - 1].into())
6}
7#[derive(Logos, Debug, Clone, PartialEq, Hash)]
8pub enum Token {
9    #[error]
10    #[regex(r"[ \t\n\f]+", logos::skip)]
11    Error,
12
13    #[regex(r"#[^\n]*", logos::skip)]
14    Comment,
15
16    #[token("{")]
17    LBracket,
18    #[token("}")]
19    RBracket,
20    #[token("(")]
21    LParen,
22    #[token(")")]
23    RParen,
24    #[token(".")]
25    Dot,
26    #[token(",")]
27    Comma,
28    #[token(":")]
29    Colon,
30    #[token("|")]
31    Pipe,
32    #[token("=>")]
33    FatArrow,
34    #[token("->")]
35    ThinArrow,
36    #[token("end")]
37    End,
38    #[token("data")]
39    Data,
40    #[token("match")]
41    Match,
42    #[token("let")]
43    Let,
44    #[token("if")]
45    If,
46    #[token("else")]
47    Else,
48    #[token("elif")]
49    Elif,
50    #[token("lambda")]
51    Lambda,
52    #[token("def")]
53    Def,
54    #[token("-")]
55    Minus,
56    #[token("+")]
57    Plus,
58    #[token("*")]
59    Times,
60    #[token("/")]
61    Divide,
62    #[token("**")]
63    Exp,
64    #[token("=")]
65    Eq,
66    #[token("==")]
67    DoubleEq,
68    #[token("%")]
69    Modulo,
70    #[token(">")]
71    Gt,
72    #[token("<")]
73    Lt,
74    #[token(">=")]
75    GtEq,
76    #[token("<=")]
77    LtEq,
78    #[token("and")]
79    LAnd,
80    #[token("or")]
81    LOr,
82    #[token("&")]
83    BitAnd,
84    #[token("^")]
85    BitXor,
86    #[regex("[0-9]+", |lex| lex.slice().parse())]
87    Number(i64),
88    #[regex("[a-zA-Z][a-zA-Z0-9_]*", |lex| lex.slice().parse())]
89    Identifier(String),
90    #[token("true", |_| true)]
91    #[token("false", |_| false)]
92    Bool(bool),
93
94    #[token(r#""[^"]*""#, string_token)]
95    String(String),
96}
97
98#[cfg(test)]
99mod tests {
100    use super::*;
101
102    #[test]
103    fn lexes_number() {
104        let mut lex = Token::lexer("1");
105
106        assert_eq!(lex.next(), Some(Token::Number(1)));
107    }
108
109    #[test]
110    fn lexes_numbers_and_ops() {
111        let mut lex = Token::lexer("1 + 2 * 3");
112
113        assert_eq!(lex.next(), Some(Token::Number(1)));
114        assert_eq!(lex.slice(), "1");
115
116        assert_eq!(lex.next(), Some(Token::Plus));
117        assert_eq!(lex.slice(), "+");
118
119        assert_eq!(lex.next(), Some(Token::Number(2)));
120        assert_eq!(lex.slice(), "2");
121
122        assert_eq!(lex.next(), Some(Token::Times));
123        assert_eq!(lex.slice(), "*");
124
125        assert_eq!(lex.next(), Some(Token::Number(3)));
126        assert_eq!(lex.slice(), "3");
127    }
128
129    #[test]
130    fn lexes_identifiers() {
131        let mut lex = Token::lexer("x");
132
133        assert_eq!(lex.next(), Some(Token::Identifier("x".to_string())));
134        assert_eq!(lex.slice(), "x");
135    }
136}
skiff/lexer/lex.rs

skiff/lexer/
lex.rs
