fml_syntax/
lexer.rs

1//! Converts FML source text into tokens, trivia included.
2
3use rowan::{TextRange, TextSize};
4
5use crate::SyntaxKind;
6use crate::diagnostic::Diagnostic;
7
8/// A token: its kind and where it is in the source.
9#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10pub struct Token {
11    pub kind: SyntaxKind,
12    pub range: TextRange,
13}
14
15/// Splits `source` into tokens. Every byte of the source belongs to exactly one token, so the
16/// concatenated token texts always equal the source. Malformed input produces
17/// [`SyntaxKind::Unknown`] tokens and a diagnostic for each.
18pub fn tokenize(source: &str) -> (Vec<Token>, Vec<Diagnostic>) {
19    let mut lexer = Lexer {
20        source,
21        pos: 0,
22        tokens: Vec::new(),
23        diagnostics: Vec::new(),
24    };
25    while lexer.pos < source.len() {
26        lexer.next_token();
27    }
28    (lexer.tokens, lexer.diagnostics)
29}
30
31struct Lexer<'a> {
32    source: &'a str,
33    pos: usize,
34    tokens: Vec<Token>,
35    diagnostics: Vec<Diagnostic>,
36}
37
38impl Lexer<'_> {
39    fn rest(&self) -> &str {
40        &self.source[self.pos..]
41    }
42
43    fn peek(&self) -> Option<char> {
44        self.rest().chars().next()
45    }
46
47    fn peek_nth(&self, n: usize) -> Option<char> {
48        self.rest().chars().nth(n)
49    }
50
51    fn next_token(&mut self) {
52        let start = self.pos;
53        let c = self.peek().expect("lexer called at end of input");
54        let kind = match c {
55            c if is_whitespace(c) => {
56                self.eat_while(is_whitespace);
57                SyntaxKind::Whitespace
58            }
59            '/' => self.slash(),
60            '\'' => self.quoted('\'', SyntaxKind::SingleQuotedString, "string"),
61            '`' => self.quoted('`', SyntaxKind::DelimitedIdent, "delimited identifier"),
62            '"' if self.rest().starts_with("\"\"\"") => self.triple_quoted(),
63            '"' => self.quoted('"', SyntaxKind::DoubleQuotedString, "string"),
64            '0'..='9' => self.number(),
65            '@' => self.date_time(),
66            '$' if self.peek_nth(1).is_some_and(is_ident_start) => {
67                self.bump();
68                self.eat_while(is_ident_continue);
69                SyntaxKind::DollarIdent
70            }
71            c if is_ident_start(c) => {
72                self.eat_while(is_ident_continue);
73                SyntaxKind::Ident
74            }
75            _ => self.punctuation(c),
76        };
77        self.push(kind, start);
78    }
79
80    fn push(&mut self, kind: SyntaxKind, start: usize) {
81        self.tokens.push(Token {
82            kind,
83            range: range(start, self.pos),
84        });
85    }
86
87    fn error(&mut self, start: usize, message: impl Into<String>) {
88        self.diagnostics
89            .push(Diagnostic::error(range(start, self.pos), message));
90    }
91
92    fn bump(&mut self) {
93        if let Some(c) = self.peek() {
94            self.pos += c.len_utf8();
95        }
96    }
97
98    fn bump_n(&mut self, n: usize) {
99        for _ in 0..n {
100            self.bump();
101        }
102    }
103
104    fn eat_while(&mut self, predicate: impl Fn(char) -> bool) {
105        while self.peek().is_some_and(&predicate) {
106            self.bump();
107        }
108    }
109
110    fn eat_digits(&mut self) -> usize {
111        let start = self.pos;
112        self.eat_while(|c| c.is_ascii_digit());
113        self.pos - start
114    }
115
116    fn slash(&mut self) -> SyntaxKind {
117        let rest = self.rest();
118        if rest.starts_with("///") && !rest.starts_with("////") && is_metadata_line(&rest[3..]) {
119            self.bump_n(3);
120            SyntaxKind::MetadataPrefix
121        } else if rest.starts_with("//") {
122            self.eat_while(|c| c != '\n' && c != '\r');
123            SyntaxKind::LineComment
124        } else if let Some(body) = rest.strip_prefix("/*") {
125            let start = self.pos;
126            match body.find("*/") {
127                Some(end) => {
128                    self.pos += 2 + end + 2;
129                    SyntaxKind::BlockComment
130                }
131                None => {
132                    self.pos = self.source.len();
133                    self.error(start, "unterminated block comment");
134                    SyntaxKind::BlockComment
135                }
136            }
137        } else {
138            self.bump();
139            SyntaxKind::Slash
140        }
141    }
142
143    fn quoted(&mut self, quote: char, kind: SyntaxKind, what: &str) -> SyntaxKind {
144        let start = self.pos;
145        self.bump();
146        loop {
147            match self.peek() {
148                None => {
149                    self.error(start, format!("unterminated {what}"));
150                    return SyntaxKind::Unknown;
151                }
152                Some('\\') => {
153                    self.bump();
154                    self.bump();
155                }
156                Some(c) if c == quote => {
157                    self.bump();
158                    return kind;
159                }
160                Some(_) => self.bump(),
161            }
162        }
163    }
164
165    fn triple_quoted(&mut self) -> SyntaxKind {
166        let start = self.pos;
167        match self.rest()[3..].find("\"\"\"") {
168            Some(end) => {
169                self.pos += 3 + end + 3;
170                SyntaxKind::TripleQuotedString
171            }
172            None => {
173                self.pos = self.source.len();
174                self.error(start, "unterminated triple-quoted string");
175                SyntaxKind::Unknown
176            }
177        }
178    }
179
180    fn number(&mut self) -> SyntaxKind {
181        self.eat_digits();
182        if self.peek() == Some('.') && self.peek_nth(1).is_some_and(|c| c.is_ascii_digit()) {
183            self.bump();
184            self.eat_digits();
185            return SyntaxKind::Decimal;
186        }
187        if self.peek() == Some('L') && !self.peek_nth(1).is_some_and(is_ident_continue) {
188            self.bump();
189            return SyntaxKind::LongNumber;
190        }
191        SyntaxKind::Integer
192    }
193
194    /// `@YYYY[-MM[-DD]][T[hh[:mm[:ss[.fff]]]][tz]]` or `@Thh[:mm[:ss[.fff]]]`.
195    fn date_time(&mut self) -> SyntaxKind {
196        let start = self.pos;
197        self.bump();
198        if self.peek() == Some('T') {
199            self.bump();
200            self.time();
201            return SyntaxKind::Time;
202        }
203        if self.eat_digits() != 4 {
204            self.error(start, "malformed date literal");
205            return SyntaxKind::Unknown;
206        }
207        for _ in 0..2 {
208            if self.peek() == Some('-') && self.peek_nth(1).is_some_and(|c| c.is_ascii_digit()) {
209                self.bump();
210                self.eat_digits();
211            }
212        }
213        if self.peek() != Some('T') {
214            return SyntaxKind::Date;
215        }
216        self.bump();
217        if self.peek().is_some_and(|c| c.is_ascii_digit()) {
218            self.time();
219            match self.peek() {
220                Some('Z') => self.bump(),
221                Some('+' | '-') if self.peek_nth(1).is_some_and(|c| c.is_ascii_digit()) => {
222                    self.bump();
223                    self.eat_digits();
224                    if self.peek() == Some(':') {
225                        self.bump();
226                        self.eat_digits();
227                    }
228                }
229                _ => {}
230            }
231        }
232        SyntaxKind::DateTime
233    }
234
235    fn time(&mut self) {
236        self.eat_digits();
237        for _ in 0..2 {
238            if self.peek() == Some(':') && self.peek_nth(1).is_some_and(|c| c.is_ascii_digit()) {
239                self.bump();
240                self.eat_digits();
241            }
242        }
243        if self.peek() == Some('.') && self.peek_nth(1).is_some_and(|c| c.is_ascii_digit()) {
244            self.bump();
245            self.eat_digits();
246        }
247    }
248
249    fn punctuation(&mut self, c: char) -> SyntaxKind {
250        let next = self.peek_nth(1);
251        let (kind, len) = match (c, next) {
252            ('-', Some('>')) => (SyntaxKind::Arrow, 2),
253            ('.', Some('.')) => (SyntaxKind::DotDot, 2),
254            ('=', Some('=')) => (SyntaxKind::EqEq, 2),
255            ('!', Some('=')) => (SyntaxKind::Neq, 2),
256            ('!', Some('~')) => (SyntaxKind::NotTilde, 2),
257            ('<', Some('<')) => (SyntaxKind::Shl, 2),
258            ('<', Some('=')) => (SyntaxKind::Le, 2),
259            ('>', Some('>')) => (SyntaxKind::Shr, 2),
260            ('>', Some('=')) => (SyntaxKind::Ge, 2),
261            ('(', _) => (SyntaxKind::LParen, 1),
262            (')', _) => (SyntaxKind::RParen, 1),
263            ('{', _) => (SyntaxKind::LBrace, 1),
264            ('}', _) => (SyntaxKind::RBrace, 1),
265            ('[', _) => (SyntaxKind::LBracket, 1),
266            (']', _) => (SyntaxKind::RBracket, 1),
267            (',', _) => (SyntaxKind::Comma, 1),
268            (';', _) => (SyntaxKind::Semicolon, 1),
269            (':', _) => (SyntaxKind::Colon, 1),
270            ('.', _) => (SyntaxKind::Dot, 1),
271            ('=', _) => (SyntaxKind::Eq, 1),
272            ('~', _) => (SyntaxKind::Tilde, 1),
273            ('<', _) => (SyntaxKind::Lt, 1),
274            ('>', _) => (SyntaxKind::Gt, 1),
275            ('+', _) => (SyntaxKind::Plus, 1),
276            ('-', _) => (SyntaxKind::Minus, 1),
277            ('*', _) => (SyntaxKind::Star, 1),
278            ('&', _) => (SyntaxKind::Amp, 1),
279            ('|', _) => (SyntaxKind::Pipe, 1),
280            ('%', _) => (SyntaxKind::Percent, 1),
281            _ => (SyntaxKind::Unknown, 1),
282        };
283        let start = self.pos;
284        self.bump_n(len);
285        if kind == SyntaxKind::Unknown {
286            self.error(start, format!("unexpected character {c:?}"));
287        }
288        kind
289    }
290}
291
292/// Whether the text after `///` reads `key =` on the same line, which makes it metadata rather
293/// than a comment that starts with three slashes.
294fn is_metadata_line(after: &str) -> bool {
295    let line = after.split(['\n', '\r']).next().unwrap_or("");
296    let line = line.trim_start_matches([' ', '\t']);
297    let key_len = line
298        .find(|c: char| !(is_ident_continue(c) || c == '.'))
299        .unwrap_or(line.len());
300    key_len > 0
301        && line.starts_with(is_ident_start)
302        && line[key_len..]
303            .trim_start_matches([' ', '\t'])
304            .starts_with('=')
305}
306
307fn is_whitespace(c: char) -> bool {
308    c.is_whitespace() || c == '\u{feff}'
309}
310
311fn is_ident_start(c: char) -> bool {
312    c.is_alphabetic() || c == '_'
313}
314
315fn is_ident_continue(c: char) -> bool {
316    c.is_alphanumeric() || c == '_'
317}
318
319fn range(start: usize, end: usize) -> TextRange {
320    TextRange::new(TextSize::from(start as u32), TextSize::from(end as u32))
321}
322
323#[cfg(test)]
324mod tests {
325    use super::*;
326
327    fn kinds(source: &str) -> Vec<(SyntaxKind, &str)> {
328        let (tokens, diagnostics) = tokenize(source);
329        assert!(diagnostics.is_empty(), "{diagnostics:?}");
330        tokens
331            .into_iter()
332            .filter(|t| t.kind != SyntaxKind::Whitespace)
333            .map(|t| (t.kind, &source[t.range]))
334            .collect()
335    }
336
337    #[test]
338    fn lossless() {
339        let source =
340            "/// url = 'x'\ngroup G(source s, target t) { s.a as v -> t.b = v \"r\"; } // c\n";
341        let (tokens, _) = tokenize(source);
342        let text: String = tokens.iter().map(|t| &source[t.range]).collect();
343        assert_eq!(text, source);
344    }
345
346    #[test]
347    fn metadata_prefix_needs_key_and_equals() {
348        assert_eq!(
349            kinds("/// url = 'x'")[0],
350            (SyntaxKind::MetadataPrefix, "///")
351        );
352        assert_eq!(
353            kinds("/// just a comment")[0],
354            (SyntaxKind::LineComment, "/// just a comment")
355        );
356        assert_eq!(kinds("//// x = 1")[0].0, SyntaxKind::LineComment);
357    }
358
359    #[test]
360    fn numbers_and_ranges() {
361        assert_eq!(
362            kinds("0..* 1.5 10L"),
363            vec![
364                (SyntaxKind::Integer, "0"),
365                (SyntaxKind::DotDot, ".."),
366                (SyntaxKind::Star, "*"),
367                (SyntaxKind::Decimal, "1.5"),
368                (SyntaxKind::LongNumber, "10L"),
369            ]
370        );
371    }
372
373    #[test]
374    fn dates() {
375        assert_eq!(
376            kinds("@2024-01-02 @2024-01-02T10:11:12.5+02:00 @T10:30"),
377            vec![
378                (SyntaxKind::Date, "@2024-01-02"),
379                (SyntaxKind::DateTime, "@2024-01-02T10:11:12.5+02:00"),
380                (SyntaxKind::Time, "@T10:30"),
381            ]
382        );
383    }
384
385    #[test]
386    fn strings() {
387        assert_eq!(
388            kinds(
389                r#"'a\'b' "c" """
390md
391""" `x y`"#
392            ),
393            vec![
394                (SyntaxKind::SingleQuotedString, r"'a\'b'"),
395                (SyntaxKind::DoubleQuotedString, "\"c\""),
396                (SyntaxKind::TripleQuotedString, "\"\"\"\nmd\n\"\"\""),
397                (SyntaxKind::DelimitedIdent, "`x y`"),
398            ]
399        );
400    }
401
402    #[test]
403    fn operators() {
404        assert_eq!(
405            kinds("-> <<type+>> != <= =="),
406            vec![
407                (SyntaxKind::Arrow, "->"),
408                (SyntaxKind::Shl, "<<"),
409                (SyntaxKind::Ident, "type"),
410                (SyntaxKind::Plus, "+"),
411                (SyntaxKind::Shr, ">>"),
412                (SyntaxKind::Neq, "!="),
413                (SyntaxKind::Le, "<="),
414                (SyntaxKind::EqEq, "=="),
415            ]
416        );
417    }
418
419    #[test]
420    fn unterminated_string_reports() {
421        let (_, diagnostics) = tokenize("'abc");
422        assert_eq!(diagnostics.len(), 1);
423    }
424}
fml_syntax/lexer.rs

fml_syntax/
lexer.rs
