lexing/
lexing.rs

1use parse_it::{LexIt, LexerState};
2
3parse_it::parse_it! {
4    #[lexer]
5    mod lex {
6        #[allow(dead_code)]
7        #[derive(Debug)]
8        pub enum Token<'a> {
9            Integer(i64),
10            String(String),
11            Ident(&'a str)
12        }
13
14        pub Initial -> Token<'lex> {
15            r"\s" => continue,
16            "\"" => {
17                let mut buf = String::new();
18                while lex!(StringLiteral(&mut buf)).is_some() {}
19                Token::String(buf)
20            },
21            Integer => Token::Integer(self),
22            r"[\p{XID_Start}_]\p{XID_Continue}*" => Token::Ident(self),
23        }
24
25        Integer -> i64 {
26            r"\d+" => self.parse::<i64>().unwrap(),
27        }
28
29        StringLiteral(buf: &mut String) {
30            r#"""# => break,
31            r"\\n" => buf.push('\n'),
32            r#"\\""# => buf.push('"'),
33            r"\\." => buf.push(self.chars().nth(1).unwrap()),
34            r"." => buf.push_str(self),
35        }
36    }
37}
38
39fn main() {
40    let src = r#"
41        "Hello, World!"
42        42
43        identifier
44    "#;
45    let lexer = lex::Initial::new();
46    let mut lexbuf = LexerState::new(src);
47    while let Some(token) = lexer.next(&mut lexbuf) {
48        println!("{token:?}");
49    }
50}
lexing/lexing.rs

lexing/
lexing.rs
