rill_lang/
lexer.rs

1//! Hand-written tokeniser. Produces `Token`s carrying source spans.
2
3use crate::error::{CompileError, Span};
4
5/// A lexical token kind.
6#[derive(Debug, Clone, PartialEq)]
7pub enum Tok {
8    /// Numeric literal that contains a `.` or exponent — a float.
9    Float(f64),
10    /// Numeric literal with no `.` — an integer.
11    Int(i64),
12    /// Identifier / keyword (`sin`, `min`, `process`, user names).
13    Ident(String),
14    /// String literal, e.g. `"cutoff"`.
15    Str(String),
16    /// `_`
17    Wire,
18    /// `!`
19    Cut,
20    /// `:`
21    Colon,
22    /// `<:`
23    Split,
24    /// `:>`
25    Merge,
26    /// `~`
27    Tilde,
28    /// `@`
29    At,
30    /// `,`
31    Comma,
32    /// `+`
33    Plus,
34    /// `-`
35    Minus,
36    /// `*`
37    Star,
38    /// `/`
39    Slash,
40    /// `%`
41    Percent,
42    /// `(`
43    LParen,
44    /// `)`
45    RParen,
46    /// `=`
47    Eq,
48    /// `;`
49    Semi,
50    /// End of input.
51    Eof,
52}
53
54/// A token plus its source span.
55#[derive(Debug, Clone, PartialEq)]
56pub struct Token {
57    /// The token kind.
58    pub tok: Tok,
59    /// Where it came from.
60    pub span: Span,
61}
62
63/// Tokenise `src` into a vector terminated by a single [`Tok::Eof`].
64///
65/// Whitespace is skipped. `//` starts a line comment.
66pub fn tokenize(src: &str) -> Result<Vec<Token>, CompileError> {
67    let bytes = src.as_bytes();
68    let mut i = 0usize;
69    let mut out = Vec::new();
70
71    let is_ident_start = |c: u8| c.is_ascii_alphabetic() || c == b'_';
72    let is_ident_cont = |c: u8| c.is_ascii_alphanumeric() || c == b'_';
73
74    while i < bytes.len() {
75        let c = bytes[i];
76        if c.is_ascii_whitespace() {
77            i += 1;
78            continue;
79        }
80        if c == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'/' {
81            while i < bytes.len() && bytes[i] != b'\n' {
82                i += 1;
83            }
84            continue;
85        }
86        let start = i;
87        if c == b'<' && i + 1 < bytes.len() && bytes[i + 1] == b':' {
88            i += 2;
89            out.push(Token {
90                tok: Tok::Split,
91                span: Span::new(start, i),
92            });
93            continue;
94        }
95        if c == b':' && i + 1 < bytes.len() && bytes[i + 1] == b'>' {
96            i += 2;
97            out.push(Token {
98                tok: Tok::Merge,
99                span: Span::new(start, i),
100            });
101            continue;
102        }
103        if c.is_ascii_digit() {
104            let mut is_float = false;
105            while i < bytes.len()
106                && (bytes[i].is_ascii_digit()
107                    || bytes[i] == b'.'
108                    || bytes[i] == b'e'
109                    || bytes[i] == b'E')
110            {
111                if bytes[i] == b'.' || bytes[i] == b'e' || bytes[i] == b'E' {
112                    is_float = true;
113                }
114                i += 1;
115            }
116            let text = &src[start..i];
117            let span = Span::new(start, i);
118            if is_float {
119                let v: f64 = text.parse().map_err(|_| CompileError::Lex {
120                    msg: format!("invalid float literal `{text}`"),
121                    span,
122                })?;
123                out.push(Token {
124                    tok: Tok::Float(v),
125                    span,
126                });
127            } else {
128                let v: i64 = text.parse().map_err(|_| CompileError::Lex {
129                    msg: format!("invalid int literal `{text}`"),
130                    span,
131                })?;
132                out.push(Token {
133                    tok: Tok::Int(v),
134                    span,
135                });
136            }
137            continue;
138        }
139        if is_ident_start(c) {
140            while i < bytes.len() && is_ident_cont(bytes[i]) {
141                i += 1;
142            }
143            let text = &src[start..i];
144            let span = Span::new(start, i);
145            let tok = if text == "_" {
146                Tok::Wire
147            } else {
148                Tok::Ident(text.to_string())
149            };
150            out.push(Token { tok, span });
151            continue;
152        }
153        if c == b'"' {
154            i += 1;
155            while i < bytes.len() && bytes[i] != b'"' {
156                i += 1;
157            }
158            if i >= bytes.len() {
159                return Err(CompileError::Lex {
160                    msg: "unterminated string literal".into(),
161                    span: Span::new(start, bytes.len()),
162                });
163            }
164            i += 1;
165            let text = src[start + 1..i - 1].to_string();
166            out.push(Token {
167                tok: Tok::Str(text),
168                span: Span::new(start, i),
169            });
170            continue;
171        }
172        let single = match c {
173            b':' => Tok::Colon,
174            b'~' => Tok::Tilde,
175            b'@' => Tok::At,
176            b',' => Tok::Comma,
177            b'+' => Tok::Plus,
178            b'-' => Tok::Minus,
179            b'*' => Tok::Star,
180            b'/' => Tok::Slash,
181            b'%' => Tok::Percent,
182            b'!' => Tok::Cut,
183            b'(' => Tok::LParen,
184            b')' => Tok::RParen,
185            b'=' => Tok::Eq,
186            b';' => Tok::Semi,
187            other => {
188                return Err(CompileError::Lex {
189                    msg: format!("unexpected character `{}`", other as char),
190                    span: Span::new(start, start + 1),
191                })
192            }
193        };
194        i += 1;
195        out.push(Token {
196            tok: single,
197            span: Span::new(start, i),
198        });
199    }
200    out.push(Token {
201        tok: Tok::Eof,
202        span: Span::new(src.len(), src.len()),
203    });
204    Ok(out)
205}
206
207#[cfg(test)]
208mod tests {
209    use super::*;
210
211    fn kinds(src: &str) -> Vec<Tok> {
212        tokenize(src).unwrap().into_iter().map(|t| t.tok).collect()
213    }
214
215    #[test]
216    fn lexes_combinators_and_ops() {
217        assert_eq!(
218            kinds("_ : + <: :> ~ @ , * / % ! ( ) = ;"),
219            vec![
220                Tok::Wire,
221                Tok::Colon,
222                Tok::Plus,
223                Tok::Split,
224                Tok::Merge,
225                Tok::Tilde,
226                Tok::At,
227                Tok::Comma,
228                Tok::Star,
229                Tok::Slash,
230                Tok::Percent,
231                Tok::Cut,
232                Tok::LParen,
233                Tok::RParen,
234                Tok::Eq,
235                Tok::Semi,
236                Tok::Eof,
237            ]
238        );
239    }
240
241    #[test]
242    fn distinguishes_int_and_float() {
243        assert_eq!(
244            kinds("3 3.5 10"),
245            vec![Tok::Int(3), Tok::Float(3.5), Tok::Int(10), Tok::Eof]
246        );
247    }
248
249    #[test]
250    fn lexes_idents_and_skips_comments() {
251        assert_eq!(
252            kinds("process // a comment\n sin"),
253            vec![
254                Tok::Ident("process".into()),
255                Tok::Ident("sin".into()),
256                Tok::Eof
257            ]
258        );
259    }
260
261    #[test]
262    fn split_and_merge_are_multichar() {
263        assert_eq!(kinds(":>"), vec![Tok::Merge, Tok::Eof]);
264        assert_eq!(kinds("<:"), vec![Tok::Split, Tok::Eof]);
265    }
266
267    #[test]
268    fn rejects_unknown_char() {
269        assert!(tokenize("$").is_err());
270    }
271
272    #[test]
273    fn lexes_string_literal() {
274        assert_eq!(
275            kinds(r#""cutoff""#),
276            vec![Tok::Str("cutoff".into()), Tok::Eof]
277        );
278    }
279
280    #[test]
281    fn rejects_unterminated_string() {
282        assert!(tokenize(r#""abc"#).is_err());
283    }
284}
rill_lang/lexer.rs

rill_lang/
lexer.rs
