vdf_reader/
lexer.rs

1use logos::Logos;
2use parse_display::Display;
3use std::str;
4
5/// Parser token.
6#[derive(PartialEq, Debug, Logos, Display, Clone)]
7#[logos(skip r"[ \t\f\r\n]+")] // whitespace
8#[logos(skip r"//[^\n]*")] // comments
9pub enum Token {
10    /// A group is starting.
11    #[token("{")]
12    #[display("start of group")]
13    GroupStart,
14    /// A group is ending.
15    #[token("}")]
16    #[display("end of group")]
17    GroupEnd,
18    /// An enclosed or bare item.
19    #[regex("[^# \t\n{}\"][^ \t\n\r{}]*", priority = 0)]
20    #[display("item")]
21    Item,
22    /// An enclosed or bare item.
23    #[regex("\"([^\"\\\\]|\\\\.)*\"")]
24    #[display("quoted item")]
25    QuotedItem,
26    /// An enclosed or bare statement.
27    #[regex("#[^ \"\t\n\r{}]+")]
28    #[display("statement")]
29    Statement,
30    /// An enclosed or bare statement.
31    #[regex("\"#([^\"\\\\]|\\\\.)*\"")]
32    #[display("quoted statement")]
33    QuotedStatement,
34}
35
36#[cfg(test)]
37mod tests {
38    use super::Token;
39    use logos::Logos;
40
41    fn get_token<'input>(
42        input: &'input str,
43    ) -> Option<Result<Token, <Token as Logos<'input>>::Error>> {
44        let mut lex = Token::lexer(input);
45        lex.next()
46    }
47
48    fn get_tokens<'input>(
49        input: &str,
50    ) -> Result<Vec<(Token, &str)>, <Token as Logos<'input>>::Error> {
51        Token::lexer(input)
52            .spanned()
53            .map(|(res, span)| res.map(|token| (token, &input[span])))
54            .collect()
55    }
56
57    #[test]
58    fn next() {
59        assert_eq!(get_token("test"), Some(Ok(Token::Item)));
60        assert_eq!(get_token("\"test\""), Some(Ok(Token::QuotedItem)));
61        assert_eq!(get_token("\"\""), Some(Ok(Token::QuotedItem)));
62        assert_eq!(get_token("\"\" "), Some(Ok(Token::QuotedItem)));
63        assert_eq!(get_token("#test"), Some(Ok(Token::Statement)));
64        assert_eq!(get_token("\"#test\""), Some(Ok(Token::QuotedStatement)));
65        assert_eq!(get_token("{"), Some(Ok(Token::GroupStart)));
66        assert_eq!(get_token("}"), Some(Ok(Token::GroupEnd)));
67        assert_eq!(get_token("//test more"), None);
68
69        assert_eq!(get_token("test"), Some(Ok(Token::Item)));
70        assert_eq!(get_token("#test"), Some(Ok(Token::Statement)));
71
72        assert_eq!(get_token("lol wut"), Some(Ok(Token::Item)));
73        assert_eq!(get_token("#lol wut"), Some(Ok(Token::Statement)));
74
75        assert_eq!(get_token("lol{"), Some(Ok(Token::Item)));
76        assert_eq!(get_token("#lol{"), Some(Ok(Token::Statement)));
77
78        assert_eq!(get_token("lol}"), Some(Ok(Token::Item)));
79        assert_eq!(get_token("#lol}"), Some(Ok(Token::Statement)));
80
81        assert_eq!(get_token("\"test\""), Some(Ok(Token::QuotedItem)));
82        assert_eq!(get_token("\"#test\""), Some(Ok(Token::QuotedStatement)));
83
84        assert_eq!(get_token("\"te\\\"st\""), Some(Ok(Token::QuotedItem)));
85        assert_eq!(get_token("\"te\\st\""), Some(Ok(Token::QuotedItem)));
86        assert_eq!(get_token("\"#te\\\"st\""), Some(Ok(Token::QuotedStatement)));
87    }
88
89    #[test]
90    fn tokenize() {
91        assert_eq!(
92            get_tokens(
93                r#"foo { // eol comment
94                "asd" "bar"
95                // a comment
96                #include other
97                empty ""
98                \\"broken" comment
99            }"#
100            ),
101            Ok(vec![
102                (Token::Item, "foo"),
103                (Token::GroupStart, "{"),
104                (Token::QuotedItem, r#""asd""#),
105                (Token::QuotedItem, r#""bar""#),
106                (Token::Statement, r#"#include"#),
107                (Token::Item, r#"other"#),
108                (Token::Item, r#"empty"#),
109                (Token::QuotedItem, r#""""#),
110                (Token::Item, r#"\\"broken""#),
111                (Token::Item, r#"comment"#),
112                (Token::GroupEnd, "}")
113            ])
114        )
115    }
116}
vdf_reader/lexer.rs

vdf_reader/
lexer.rs
