vdf_reader/
lexer.rs

1use logos::Logos;
2use parse_display::Display;
3use std::str;
4
5/// Parser token.
6#[derive(PartialEq, Debug, Logos, Display, Clone)]
7#[logos(skip r"[ \t\f\r\n]+")] // whitespace
8#[logos(skip r"//[^\n]*")] // comments
9pub enum Token {
10    /// A group is starting.
11    #[token("{")]
12    #[display("start of group")]
13    GroupStart,
14    /// A group is ending.
15    #[token("}")]
16    #[display("end of group")]
17    GroupEnd,
18    /// An enclosed or bare item.
19    #[regex("[^# \t\n{}\"][^ \t\n\r{}]*", priority = 0)]
20    #[display("item")]
21    Item,
22    /// An enclosed or bare item.
23    #[regex("\"([^\"\\\\]|\\\\.)*\"")]
24    #[display("quoted item")]
25    QuotedItem,
26    /// An enclosed or bare statement.
27    #[regex("#[^ \"\t\n\r{}]+")]
28    #[display("statement")]
29    Statement,
30    /// An enclosed or bare statement.
31    #[regex("\"#([^\"\\\\]|\\\\.)*\"")]
32    #[display("quoted statement")]
33    QuotedStatement,
34}
35
36#[cfg(test)]
37mod tests {
38    use super::Token;
39    use logos::Logos;
40
41    fn get_token(input: &str) -> Option<Result<Token, <Token as Logos>::Error>> {
42        let mut lex = Token::lexer(input);
43        lex.next()
44    }
45
46    fn get_tokens(input: &str) -> Result<Vec<(Token, &str)>, <Token as Logos>::Error> {
47        Token::lexer(input)
48            .spanned()
49            .map(|(res, span)| res.map(|token| (token, &input[span])))
50            .collect()
51    }
52
53    #[test]
54    fn next() {
55        assert_eq!(get_token("test"), Some(Ok(Token::Item)));
56        assert_eq!(get_token("\"test\""), Some(Ok(Token::QuotedItem)));
57        assert_eq!(get_token("\"\""), Some(Ok(Token::QuotedItem)));
58        assert_eq!(get_token("\"\" "), Some(Ok(Token::QuotedItem)));
59        assert_eq!(get_token("#test"), Some(Ok(Token::Statement)));
60        assert_eq!(get_token("\"#test\""), Some(Ok(Token::QuotedStatement)));
61        assert_eq!(get_token("{"), Some(Ok(Token::GroupStart)));
62        assert_eq!(get_token("}"), Some(Ok(Token::GroupEnd)));
63        assert_eq!(get_token("//test more"), None);
64
65        assert_eq!(get_token("test"), Some(Ok(Token::Item)));
66        assert_eq!(get_token("#test"), Some(Ok(Token::Statement)));
67
68        assert_eq!(get_token("lol wut"), Some(Ok(Token::Item)));
69        assert_eq!(get_token("#lol wut"), Some(Ok(Token::Statement)));
70
71        assert_eq!(get_token("lol{"), Some(Ok(Token::Item)));
72        assert_eq!(get_token("#lol{"), Some(Ok(Token::Statement)));
73
74        assert_eq!(get_token("lol}"), Some(Ok(Token::Item)));
75        assert_eq!(get_token("#lol}"), Some(Ok(Token::Statement)));
76
77        assert_eq!(get_token("\"test\""), Some(Ok(Token::QuotedItem)));
78        assert_eq!(get_token("\"#test\""), Some(Ok(Token::QuotedStatement)));
79
80        assert_eq!(get_token("\"te\\\"st\""), Some(Ok(Token::QuotedItem)));
81        assert_eq!(get_token("\"te\\st\""), Some(Ok(Token::QuotedItem)));
82        assert_eq!(get_token("\"#te\\\"st\""), Some(Ok(Token::QuotedStatement)));
83    }
84
85    #[test]
86    fn tokenize() {
87        assert_eq!(
88            get_tokens(
89                r#"foo { // eol comment
90                "asd" "bar"
91                // a comment
92                #include other
93                empty ""
94                \\"broken" comment
95            }"#
96            ),
97            Ok(vec![
98                (Token::Item, "foo"),
99                (Token::GroupStart, "{"),
100                (Token::QuotedItem, r#""asd""#),
101                (Token::QuotedItem, r#""bar""#),
102                (Token::Statement, r#"#include"#),
103                (Token::Item, r#"other"#),
104                (Token::Item, r#"empty"#),
105                (Token::QuotedItem, r#""""#),
106                (Token::Item, r#"\\"broken""#),
107                (Token::Item, r#"comment"#),
108                (Token::GroupEnd, "}")
109            ])
110        )
111    }
112}
vdf_reader/lexer.rs

vdf_reader/
lexer.rs
