presk_syntax/
lexer.rs

1//! Hand-written lexer.
2//!
3//! Identifiers follow Unicode XID (so `θ`, `γ`, `ŷ` and the decomposed `x̂`
4//! all lex as identifiers) except that `·` (U+00B7, an XID_Continue
5//! character) is always the contraction operator. Names are NFC-normalized;
6//! spans index the original text. `--` starts a comment. Runs of line breaks
7//! produce one `Newline` token; the parser decides where they matter.
8//!
9//! Every non-ASCII symbol has an ASCII spelling lexed to the same token:
10//! `ℝ`/`Real`, `λ`/`\`, `·`/`@`, `⊗`/`><`, `∘`/`<<`.
11
12use unicode_normalization::UnicodeNormalization;
13
14use crate::diag::Diagnostic;
15use crate::source::{FileId, Span};
16use crate::token::{Kw, Op, TokKind, Token};
17
18pub struct Lexed {
19    pub tokens: Vec<Token>,
20    pub diags: Vec<Diagnostic>,
21}
22
23pub fn lex(file: FileId, text: &str) -> Lexed {
24    Lexer { file, text, pos: 0, tokens: vec![], diags: vec![] }.run()
25}
26
27struct Lexer<'a> {
28    file: FileId,
29    text: &'a str,
30    pos: usize,
31    tokens: Vec<Token>,
32    diags: Vec<Diagnostic>,
33}
34
35fn is_ident_start(c: char) -> bool {
36    c == '_' || (unicode_ident::is_xid_start(c) && c != 'λ' && c != 'ℝ')
37}
38
39fn is_ident_continue(c: char) -> bool {
40    c != '·' && unicode_ident::is_xid_continue(c)
41}
42
43impl<'a> Lexer<'a> {
44    fn peek(&self) -> Option<char> {
45        self.text[self.pos..].chars().next()
46    }
47
48    fn peek2(&self) -> Option<char> {
49        let mut it = self.text[self.pos..].chars();
50        it.next();
51        it.next()
52    }
53
54    fn bump(&mut self) -> Option<char> {
55        let c = self.peek()?;
56        self.pos += c.len_utf8();
57        Some(c)
58    }
59
60    fn push(&mut self, kind: TokKind, start: usize) {
61        self.tokens.push(Token { kind, span: Span::new(self.file, start, self.pos) });
62    }
63
64    fn run(mut self) -> Lexed {
65        while let Some(c) = self.peek() {
66            let start = self.pos;
67            match c {
68                '\n' => {
69                    self.bump();
70                    // A single Newline token per run of blank lines, comments included.
71                    if !matches!(self.tokens.last(), Some(Token { kind: TokKind::Newline, .. })) {
72                        self.push(TokKind::Newline, start);
73                    }
74                }
75                c if c.is_whitespace() => {
76                    self.bump();
77                }
78                '-' if self.peek2() == Some('-') => {
79                    while let Some(c) = self.peek() {
80                        if c == '\n' {
81                            break;
82                        }
83                        self.bump();
84                    }
85                }
86                '"' => self.string(start),
87                c if c.is_ascii_digit() => self.number(start),
88                'λ' => {
89                    self.bump();
90                    self.push(TokKind::Kw(Kw::Lambda), start);
91                }
92                'ℝ' => {
93                    self.bump();
94                    self.push(TokKind::Kw(Kw::Real), start);
95                }
96                '\\' => {
97                    self.bump();
98                    self.push(TokKind::Kw(Kw::Lambda), start);
99                }
100                c if is_ident_start(c) => self.ident(start),
101                _ => self.operator(start),
102            }
103        }
104        // Every file ends with a Newline before Eof, which simplifies the parser.
105        if !matches!(self.tokens.last(), Some(Token { kind: TokKind::Newline, .. }) | None) {
106            let p = self.pos;
107            self.push(TokKind::Newline, p);
108        }
109        let p = self.pos;
110        self.push(TokKind::Eof, p);
111        Lexed { tokens: self.tokens, diags: self.diags }
112    }
113
114    fn ident(&mut self, start: usize) {
115        while let Some(c) = self.peek() {
116            if is_ident_continue(c) {
117                self.bump();
118            } else {
119                break;
120            }
121        }
122        let raw = &self.text[start..self.pos];
123        let name: String = raw.nfc().collect();
124        let kind = match Kw::from_ident(&name) {
125            Some(kw) => TokKind::Kw(kw),
126            None => TokKind::Ident(name),
127        };
128        self.push(kind, start);
129    }
130
131    fn number(&mut self, start: usize) {
132        let mut is_float = false;
133        self.digits();
134        if self.peek() == Some('.') && self.peek2().is_some_and(|c| c.is_ascii_digit()) {
135            is_float = true;
136            self.bump();
137            self.digits();
138        }
139        if matches!(self.peek(), Some('e' | 'E')) {
140            let save = self.pos;
141            self.bump();
142            if matches!(self.peek(), Some('+' | '-')) {
143                self.bump();
144            }
145            if self.peek().is_some_and(|c| c.is_ascii_digit()) {
146                is_float = true;
147                self.digits();
148            } else {
149                self.pos = save; // `e` belongs to a unit, e.g. `3e` is not an exponent without digits
150            }
151        }
152        let num = self.text[start..self.pos].to_string();
153        // A unit glued to the number: `5ms`, `3MiB`, `204GB/s`, `192KiB/sm`.
154        if self.peek().is_some_and(|c| c.is_alphabetic()) {
155            let ustart = self.pos;
156            while self.peek().is_some_and(|c| c.is_alphanumeric()) {
157                self.bump();
158            }
159            if self.peek() == Some('/') && self.peek2().is_some_and(|c| c.is_alphabetic()) {
160                self.bump();
161                while self.peek().is_some_and(|c| c.is_alphanumeric()) {
162                    self.bump();
163                }
164            }
165            let unit = self.text[ustart..self.pos].to_string();
166            self.push(TokKind::Quantity { num, unit }, start);
167        } else if is_float {
168            self.push(TokKind::Float(num), start);
169        } else {
170            self.push(TokKind::Nat(num), start);
171        }
172    }
173
174    fn digits(&mut self) {
175        while self.peek().is_some_and(|c| c.is_ascii_digit()) {
176            self.bump();
177        }
178    }
179
180    fn string(&mut self, start: usize) {
181        self.bump(); // opening quote
182        let mut s = String::new();
183        loop {
184            match self.bump() {
185                Some('"') => break,
186                Some('\\') => match self.bump() {
187                    Some('n') => s.push('\n'),
188                    Some('t') => s.push('\t'),
189                    Some(c) => s.push(c),
190                    None => break,
191                },
192                Some('\n') | None => {
193                    self.diags.push(
194                        Diagnostic::error("E0001", "unterminated string literal")
195                            .label(Span::new(self.file, start, self.pos), "string starts here and never closes"),
196                    );
197                    break;
198                }
199                Some(c) => s.push(c),
200            }
201        }
202        self.push(TokKind::Str(s), start);
203    }
204
205    fn operator(&mut self, start: usize) {
206        let c = self.bump().expect("caller checked a char is available");
207        let two = self.peek();
208        let op = match (c, two) {
209            ('.', Some('.')) if self.text[self.pos..].starts_with("..") => {
210                self.bump();
211                self.bump();
212                Op::Ellipsis
213            }
214            ('.', Some('.')) => {
215                self.bump();
216                Op::DotDot
217            }
218            ('=', Some('=')) => {
219                self.bump();
220                Op::EqEq
221            }
222            ('<', Some('=')) => {
223                self.bump();
224                Op::Le
225            }
226            ('<', Some('<')) => {
227                self.bump();
228                Op::Compose
229            }
230            ('>', Some('<')) => {
231                self.bump();
232                Op::Tensor
233            }
234            ('>', Some('=')) => {
235                self.bump();
236                Op::Ge
237            }
238            ('-', Some('>')) => {
239                self.bump();
240                Op::Arrow
241            }
242            ('|', Some('>')) => {
243                self.bump();
244                Op::Pipe
245            }
246            ('·', _) | ('@', _) => Op::Dot,
247            ('*', _) => Op::Star,
248            ('/', _) => Op::Slash,
249            ('%', _) => Op::Percent,
250            ('+', _) => Op::Plus,
251            ('-', _) => Op::Minus,
252            ('^', _) => Op::Caret,
253            ('<', _) => Op::Lt,
254            ('>', _) => Op::Gt,
255            ('=', _) => Op::Eq,
256            ('|', _) => Op::Bar,
257            ('⊗', _) => Op::Tensor,
258            ('∘', _) => Op::Compose,
259            ('.', _) => Op::Period,
260            (',', _) => Op::Comma,
261            (':', _) => Op::Colon,
262            ('(', _) => Op::LParen,
263            (')', _) => Op::RParen,
264            ('[', _) => Op::LBracket,
265            (']', _) => Op::RBracket,
266            ('{', _) => Op::LBrace,
267            ('}', _) => Op::RBrace,
268            _ => {
269                self.diags.push(
270                    Diagnostic::error("E0002", format!("unexpected character `{c}` (U+{:04X})", c as u32))
271                        .label(Span::new(self.file, start, self.pos), "not part of the Presk syntax"),
272                );
273                return;
274            }
275        };
276        self.push(TokKind::Op(op), start);
277    }
278}
279
280/// One line per token: `kind @ start..end`, for `presk dump --stage tokens`.
281pub fn dump_tokens(tokens: &[Token]) -> String {
282    let mut out = String::new();
283    for t in tokens {
284        out.push_str(&format!("{:?} @ {:?}\n", t.kind, t.span));
285    }
286    out
287}
presk_syntax/lexer.rs

presk_syntax/
lexer.rs
