lex_syntax/
token.rs

1use logos::Logos;
2use std::ops::Range;
3
4#[derive(Logos, Debug, Clone, PartialEq)]
5#[logos(skip r"[ \t\r\f]+")]
6#[logos(skip(r"#[^\n]*", allow_greedy = true))]
7pub enum TokenKind {
8    // keywords
9    #[token("fn")]      Fn,
10    #[token("let")]     Let,
11    #[token("type")]    Type,
12    #[token("match")]   Match,
13    #[token("if")]      If,
14    #[token("else")]    Else,
15    #[token("return")]  Return,
16    #[token("import")]  Import,
17    #[token("as")]      As,
18    #[token("true")]    True,
19    #[token("false")]   False,
20    #[token("and")]     And,
21    #[token("or")]      Or,
22    #[token("not")]     Not,
23
24    // multi-char operators (longer first to win the match race)
25    #[token("|>")] Pipe,
26    #[token("->")] Arrow,
27    #[token("=>")] FatArrow,
28    #[token(":=")] ColonEq,
29    #[token("::")] ColonColon,
30    #[token("==")] EqEq,
31    #[token("!=")] BangEq,
32    #[token("<=")] LtEq,
33    #[token(">=")] GtEq,
34
35    // single-char operators
36    #[token("+")] Plus,
37    #[token("-")] Minus,
38    #[token("*")] Star,
39    #[token("/")] Slash,
40    #[token("%")] Percent,
41    #[token("<")] Lt,
42    #[token(">")] Gt,
43    #[token(".")] Dot,
44    #[token(",")] Comma,
45    #[token(";")] Semi,
46    #[token(":")] Colon,
47    #[token("?")] Question,
48    #[token("(")] LParen,
49    #[token(")")] RParen,
50    #[token("{")] LBrace,
51    #[token("}")] RBrace,
52    #[token("[")] LBracket,
53    #[token("]")] RBracket,
54    #[token("=")] Eq,
55    #[token("|")] Bar,
56    #[token("_")] Underscore,
57    #[token("\n")] Newline,
58
59    // literals
60    #[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*", |lex| lex.slice().replace('_', "").parse::<f64>().ok())]
61    Float(f64),
62
63    #[regex(r"[0-9][0-9_]*", |lex| lex.slice().replace('_', "").parse::<i64>().ok(), priority = 3)]
64    Int(i64),
65
66    #[regex(r#""([^"\\]|\\.)*""#, |lex| unescape(&lex.slice()[1..lex.slice().len()-1]))]
67    Str(String),
68
69    #[regex(r#"b"([^"\\]|\\.)*""#, |lex| unescape(&lex.slice()[2..lex.slice().len()-1]).map(|s| s.into_bytes()))]
70    Bytes(Vec<u8>),
71
72    #[regex(r"[a-zA-Z][a-zA-Z0-9_]*", |lex| lex.slice().to_string())]
73    Ident(String),
74}
75
76fn unescape(s: &str) -> Option<String> {
77    let mut out = String::with_capacity(s.len());
78    let mut chars = s.chars();
79    while let Some(c) = chars.next() {
80        if c == '\\' {
81            match chars.next()? {
82                'n' => out.push('\n'),
83                't' => out.push('\t'),
84                'r' => out.push('\r'),
85                '\\' => out.push('\\'),
86                '"' => out.push('"'),
87                '0' => out.push('\0'),
88                _ => return None,
89            }
90        } else {
91            out.push(c);
92        }
93    }
94    Some(out)
95}
96
97#[derive(Debug, Clone)]
98pub struct Token {
99    pub kind: TokenKind,
100    pub span: Range<usize>,
101}
102
103pub fn lex(src: &str) -> Result<Vec<Token>, LexError> {
104    let mut toks = Vec::new();
105    let mut lx = TokenKind::lexer(src);
106    while let Some(res) = lx.next() {
107        match res {
108            Ok(kind) => toks.push(Token { kind, span: lx.span() }),
109            Err(_) => {
110                return Err(LexError {
111                    span: lx.span(),
112                    snippet: lx.slice().to_string(),
113                });
114            }
115        }
116    }
117    Ok(toks)
118}
119
120#[derive(Debug, thiserror::Error)]
121#[error("unrecognized token `{snippet}` at {span:?}")]
122pub struct LexError {
123    pub span: Range<usize>,
124    pub snippet: String,
125}
lex_syntax/token.rs

lex_syntax/
token.rs
