lex_syntax/
token.rs

1use logos::Logos;
2use std::ops::Range;
3
4#[derive(Logos, Debug, Clone, PartialEq)]
5#[logos(skip r"[ \t\r\f]+")]
6#[logos(skip(r"#[^\n]*", allow_greedy = true))]
7pub enum TokenKind {
8    // keywords
9    #[token("fn")]      Fn,
10    #[token("let")]     Let,
11    #[token("type")]    Type,
12    #[token("match")]   Match,
13    #[token("if")]      If,
14    #[token("else")]    Else,
15    #[token("return")]  Return,
16    #[token("import")]  Import,
17    #[token("as")]      As,
18    #[token("true")]    True,
19    #[token("false")]   False,
20    #[token("and")]     And,
21    #[token("or")]      Or,
22    #[token("not")]     Not,
23
24    // multi-char operators (longer first to win the match race)
25    #[token("|>")] Pipe,
26    #[token("->")] Arrow,
27    #[token("=>")] FatArrow,
28    #[token(":=")] ColonEq,
29    #[token("::")] ColonColon,
30    #[token("==")] EqEq,
31    #[token("!=")] BangEq,
32    #[token("<=")] LtEq,
33    #[token(">=")] GtEq,
34
35    // spread operator (record type spread `{ ...TypeName }`)
36    #[token("...")] DotDotDot,
37
38    // single-char operators
39    #[token("+")] Plus,
40    #[token("-")] Minus,
41    #[token("*")] Star,
42    #[token("/")] Slash,
43    #[token("%")] Percent,
44    #[token("<")] Lt,
45    #[token(">")] Gt,
46    #[token(".")] Dot,
47    #[token(",")] Comma,
48    #[token(";")] Semi,
49    #[token(":")] Colon,
50    #[token("?")] Question,
51    #[token("(")] LParen,
52    #[token(")")] RParen,
53    #[token("{")] LBrace,
54    #[token("}")] RBrace,
55    #[token("[")] LBracket,
56    #[token("]")] RBracket,
57    #[token("=")] Eq,
58    #[token("|")] Bar,
59    #[token("_")] Underscore,
60    #[token("\n")] Newline,
61
62    // literals
63    #[regex(r"[0-9][0-9_]*[eE][+-]?[0-9]+", |lex| lex.slice().replace('_', "").parse::<f64>().ok())]
64    #[regex(r"[0-9][0-9_]*\.[0-9][0-9_]*([eE][+-]?[0-9]+)?", |lex| lex.slice().replace('_', "").parse::<f64>().ok())]
65    Float(f64),
66
67    // Hex (0x1F) and binary (0b1010) integer literals. Underscores allowed
68    // for readability, matching the decimal literal's own `_` grouping.
69    // Without these, `0x80` lexed as `Int(0)` immediately followed by
70    // `Ident("x80")` (`x80` matches the identifier pattern below) --
71    // silently wrong rather than a lex error, so a call like
72    // `bytes.singleton(0x80)` failed downstream in the parser with a
73    // confusing "expected RParen, got Ident" instead of naming the real
74    // problem. Reproduced live: an agent writing RLP encoding (where
75    // `0x80`-style byte constants are the natural idiom) hit exactly this.
76    // Logos resolves the ambiguity with plain decimal Int by longest-match:
77    // "0x80" (len 4) always beats "0" (len 1) for the same input, so this
78    // needs no explicit priority to win.
79    #[regex(r"0[xX][0-9a-fA-F][0-9a-fA-F_]*", |lex| i64::from_str_radix(&lex.slice()[2..].replace('_', ""), 16).ok())]
80    #[regex(r"0[bB][01][01_]*", |lex| i64::from_str_radix(&lex.slice()[2..].replace('_', ""), 2).ok())]
81    #[regex(r"[0-9][0-9_]*", |lex| lex.slice().replace('_', "").parse::<i64>().ok(), priority = 3)]
82    Int(i64),
83
84    #[regex(r#""([^"\\]|\\.)*""#, |lex| unescape(&lex.slice()[1..lex.slice().len()-1]))]
85    Str(String),
86
87    #[regex(r#"b"([^"\\]|\\.)*""#, |lex| unescape(&lex.slice()[2..lex.slice().len()-1]).map(|s| s.into_bytes()))]
88    Bytes(Vec<u8>),
89
90    /// String interpolation literal `f"hello {name}"` (#562). The content
91    /// is unescaped the same way as `Str`; `{...}` segments are desugared
92    /// to `str.concat` chains by the parser.
93    #[regex(r#"f"([^"\\]|\\.)*""#, |lex| unescape(&lex.slice()[2..lex.slice().len()-1]))]
94    FStr(String),
95
96    // Identifier. Two alternatives so a bare `_` keeps lexing as
97    // the discard token (used by `match _ => ...` and the new
98    // `let _ := ...`) while `_name` is recognized as a real
99    // identifier (#200). Logos picks the longer match: for `_`
100    // alone only Underscore matches (Ident requires ≥2 chars on
101    // the underscore branch); for `_x` the Ident branch wins.
102    #[regex(r"[a-zA-Z][a-zA-Z0-9_]*", |lex| lex.slice().to_string())]
103    #[regex(r"_[a-zA-Z0-9_]+", |lex| lex.slice().to_string())]
104    Ident(String),
105}
106
107fn unescape(s: &str) -> Option<String> {
108    let mut out = String::with_capacity(s.len());
109    let mut chars = s.chars();
110    while let Some(c) = chars.next() {
111        if c == '\\' {
112            match chars.next()? {
113                'n' => out.push('\n'),
114                't' => out.push('\t'),
115                'r' => out.push('\r'),
116                '\\' => out.push('\\'),
117                '"' => out.push('"'),
118                '0' => out.push('\0'),
119                _ => return None,
120            }
121        } else {
122            out.push(c);
123        }
124    }
125    Some(out)
126}
127
128#[derive(Debug, Clone)]
129pub struct Token {
130    pub kind: TokenKind,
131    pub span: Range<usize>,
132}
133
134pub fn lex(src: &str) -> Result<Vec<Token>, LexError> {
135    let mut toks = Vec::new();
136    let mut lx = TokenKind::lexer(src);
137    while let Some(res) = lx.next() {
138        match res {
139            Ok(kind) => toks.push(Token { kind, span: lx.span() }),
140            Err(_) => {
141                return Err(LexError {
142                    span: lx.span(),
143                    snippet: lx.slice().to_string(),
144                });
145            }
146        }
147    }
148    Ok(toks)
149}
150
151#[derive(Debug, thiserror::Error)]
152#[error("unrecognized token `{snippet}` at {span:?}")]
153pub struct LexError {
154    pub span: Range<usize>,
155    pub snippet: String,
156}
lex_syntax/token.rs

lex_syntax/
token.rs
