open_cypher/
lexer.rs

1//! Lossless openCypher tokenization built with Logos.
2
3use logos::{Lexer, Logos};
4
5use crate::diagnostic::{Diagnostic, DiagnosticCode};
6use crate::span::Span;
7use crate::token::{Keyword, Token, TokenKind};
8
9/// The result of losslessly tokenizing a source string.
10#[derive(Clone, Debug, Default, PartialEq, Eq)]
11#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
12pub struct LexOutcome {
13    /// Every token in source order, including trivia and invalid regions.
14    pub tokens: Vec<Token>,
15    /// Lexical diagnostics in source order.
16    pub diagnostics: Vec<Diagnostic>,
17}
18
19#[derive(Clone, Debug, Default, PartialEq)]
20enum LexingError {
21    #[default]
22    Invalid,
23    UnterminatedString,
24    UnterminatedIdentifier,
25    UnterminatedComment,
26    InvalidEscape,
27}
28
29#[derive(Logos, Clone, Copy, Debug, PartialEq)]
30#[logos(error = LexingError)]
31enum RawToken {
32    #[regex(r"\p{White_Space}+")]
33    Whitespace,
34
35    #[regex(r"//[^\r\n]*", allow_greedy = true)]
36    LineComment,
37
38    #[token("/*", lex_block_comment)]
39    BlockComment,
40
41    #[token("'", lex_single_string)]
42    #[token("\"", lex_double_string)]
43    String,
44
45    #[token("`", lex_escaped_identifier)]
46    EscapedIdentifier,
47
48    #[regex(r"\$[_\p{XID_Continue}]+")]
49    Parameter,
50
51    #[regex(r"0[xX](?:_?[0-9A-Fa-f])+")]
52    HexInteger,
53
54    #[regex(r"0[oO](?:_?[0-7])+")]
55    OctalInteger,
56
57    #[regex(r"(?:[0-9](?:_?[0-9])*)?\.[0-9](?:_?[0-9])*(?:[eE][+-]?[0-9](?:_?[0-9])*)?[fFdD]?")]
58    #[regex(r"[0-9](?:_?[0-9])*[eE][+-]?[0-9](?:_?[0-9])*[fFdD]?")]
59    Float,
60
61    #[regex(r"[0-9](?:_?[0-9])*")]
62    Integer,
63
64    #[regex(r"[_\p{XID_Start}][_\p{XID_Continue}]*")]
65    Word,
66
67    #[token("||")]
68    DoublePipe,
69    #[token("::")]
70    DoubleColon,
71    #[token("..")]
72    DotDot,
73    #[token("<=")]
74    LessEqual,
75    #[token(">=")]
76    GreaterEqual,
77    #[token("<>")]
78    #[token("!=")]
79    NotEqual,
80    #[token("+=")]
81    PlusEqual,
82    #[token("=>")]
83    FatArrow,
84    #[token("=~")]
85    RegexMatch,
86    #[token("<-")]
87    LeftArrow,
88    #[token("->")]
89    RightArrow,
90
91    #[token("(")]
92    LeftParen,
93    #[token(")")]
94    RightParen,
95    #[token("[")]
96    LeftBracket,
97    #[token("]")]
98    RightBracket,
99    #[token("{")]
100    LeftBrace,
101    #[token("}")]
102    RightBrace,
103    #[token(",")]
104    Comma,
105    #[token(".")]
106    Dot,
107    #[token(":")]
108    Colon,
109    #[token(";")]
110    Semicolon,
111    #[token("|")]
112    Pipe,
113    #[token("&")]
114    Ampersand,
115    #[token("?")]
116    Question,
117    #[token("$")]
118    Dollar,
119    #[token("+")]
120    Plus,
121    #[token("-")]
122    Minus,
123    #[token("*")]
124    Star,
125    #[token("/")]
126    Slash,
127    #[token("%")]
128    Percent,
129    #[token("^")]
130    Caret,
131    #[token("!")]
132    Bang,
133    #[token("=")]
134    Equal,
135    #[token("<")]
136    Less,
137    #[token(">")]
138    Greater,
139}
140
141fn lex_single_string(lexer: &mut Lexer<'_, RawToken>) -> Result<(), LexingError> {
142    consume_quoted(lexer, '\'', LexingError::UnterminatedString)
143}
144
145fn lex_double_string(lexer: &mut Lexer<'_, RawToken>) -> Result<(), LexingError> {
146    consume_quoted(lexer, '"', LexingError::UnterminatedString)
147}
148
149fn lex_escaped_identifier(lexer: &mut Lexer<'_, RawToken>) -> Result<(), LexingError> {
150    consume_quoted(lexer, '`', LexingError::UnterminatedIdentifier)
151}
152
153fn consume_quoted(
154    lexer: &mut Lexer<'_, RawToken>,
155    delimiter: char,
156    unterminated: LexingError,
157) -> Result<(), LexingError> {
158    let remainder = lexer.remainder();
159    let mut characters = remainder.char_indices().peekable();
160    let mut invalid_escape = false;
161
162    while let Some((offset, character)) = characters.next() {
163        if character == '\\' {
164            let Some((_, escaped)) = characters.next() else {
165                continue;
166            };
167            match escaped {
168                '\\' | '\'' | '"' | '`' | 't' | 'b' | 'n' | 'r' | 'f' => {}
169                'u' | 'U' => {
170                    let digits = if escaped == 'u' { 4 } else { 6 };
171                    let mut scalar = 0u32;
172                    let mut complete = true;
173                    for _ in 0..digits {
174                        match characters.peek().copied() {
175                            Some((_, digit)) if digit.is_ascii_hexdigit() => {
176                                let _ = characters.next();
177                                scalar = scalar * 16 + digit.to_digit(16).expect("hex digit");
178                            }
179                            _ => {
180                                complete = false;
181                                break;
182                            }
183                        }
184                    }
185                    if !complete || char::from_u32(scalar).is_none() {
186                        invalid_escape = true;
187                    }
188                }
189                _ => invalid_escape = true,
190            }
191            continue;
192        }
193
194        if character == delimiter {
195            if characters
196                .peek()
197                .is_some_and(|(_, next)| *next == delimiter)
198            {
199                let _ = characters.next();
200                continue;
201            }
202
203            lexer.bump(offset + character.len_utf8());
204            return if invalid_escape {
205                Err(LexingError::InvalidEscape)
206            } else {
207                Ok(())
208            };
209        }
210    }
211
212    lexer.bump(remainder.len());
213    Err(unterminated)
214}
215
216fn lex_block_comment(lexer: &mut Lexer<'_, RawToken>) -> Result<(), LexingError> {
217    let remainder = lexer.remainder().as_bytes();
218    let mut offset = 0;
219    let mut depth = 1usize;
220
221    while offset + 1 < remainder.len() {
222        match (remainder[offset], remainder[offset + 1]) {
223            (b'/', b'*') => {
224                depth += 1;
225                offset += 2;
226            }
227            (b'*', b'/') => {
228                depth -= 1;
229                offset += 2;
230                if depth == 0 {
231                    lexer.bump(offset);
232                    return Ok(());
233                }
234            }
235            _ => offset += 1,
236        }
237    }
238
239    lexer.bump(remainder.len());
240    Err(LexingError::UnterminatedComment)
241}
242
243/// Tokenizes `source` without dropping trivia or invalid input.
244#[must_use]
245pub fn lex(source: &str) -> LexOutcome {
246    let mut lexer = RawToken::lexer(source);
247    let mut tokens = Vec::new();
248    let mut diagnostics = Vec::new();
249
250    while let Some(raw) = lexer.next() {
251        let range = lexer.span();
252        let span = Span::new(range.start, range.end);
253        let text = &source[range];
254
255        let kind = match raw {
256            Ok(raw) => token_kind(raw, text),
257            Err(error) => {
258                let (code, message) = match error {
259                    LexingError::Invalid => {
260                        (DiagnosticCode::InvalidToken, "invalid openCypher token")
261                    }
262                    LexingError::UnterminatedString => (
263                        DiagnosticCode::UnterminatedString,
264                        "unterminated string literal",
265                    ),
266                    LexingError::UnterminatedIdentifier => (
267                        DiagnosticCode::UnterminatedIdentifier,
268                        "unterminated escaped identifier",
269                    ),
270                    LexingError::UnterminatedComment => (
271                        DiagnosticCode::UnterminatedComment,
272                        "unterminated block comment",
273                    ),
274                    LexingError::InvalidEscape => {
275                        (DiagnosticCode::InvalidEscape, "invalid escape sequence")
276                    }
277                };
278                diagnostics.push(Diagnostic::error(code, message, span));
279                TokenKind::Invalid
280            }
281        };
282
283        tokens.push(Token::new(kind, span));
284    }
285
286    LexOutcome {
287        tokens,
288        diagnostics,
289    }
290}
291
292fn token_kind(raw: RawToken, text: &str) -> TokenKind {
293    match raw {
294        RawToken::Whitespace => TokenKind::Whitespace,
295        RawToken::LineComment => TokenKind::LineComment,
296        RawToken::BlockComment => TokenKind::BlockComment,
297        RawToken::String => TokenKind::String,
298        RawToken::EscapedIdentifier => TokenKind::EscapedIdentifier,
299        RawToken::Parameter => TokenKind::Parameter,
300        RawToken::HexInteger => TokenKind::HexInteger,
301        RawToken::OctalInteger => TokenKind::OctalInteger,
302        RawToken::Float => TokenKind::Float,
303        RawToken::Integer => TokenKind::Integer,
304        RawToken::Word => Keyword::from_ascii_case_insensitive(text)
305            .map(TokenKind::Keyword)
306            .unwrap_or(TokenKind::Identifier),
307        RawToken::LeftParen => TokenKind::LeftParen,
308        RawToken::RightParen => TokenKind::RightParen,
309        RawToken::LeftBracket => TokenKind::LeftBracket,
310        RawToken::RightBracket => TokenKind::RightBracket,
311        RawToken::LeftBrace => TokenKind::LeftBrace,
312        RawToken::RightBrace => TokenKind::RightBrace,
313        RawToken::Comma => TokenKind::Comma,
314        RawToken::Dot => TokenKind::Dot,
315        RawToken::DotDot => TokenKind::DotDot,
316        RawToken::Colon => TokenKind::Colon,
317        RawToken::DoubleColon => TokenKind::DoubleColon,
318        RawToken::Semicolon => TokenKind::Semicolon,
319        RawToken::Pipe => TokenKind::Pipe,
320        RawToken::DoublePipe => TokenKind::DoublePipe,
321        RawToken::Ampersand => TokenKind::Ampersand,
322        RawToken::Question => TokenKind::Question,
323        RawToken::Dollar => TokenKind::Dollar,
324        RawToken::Plus => TokenKind::Plus,
325        RawToken::Minus => TokenKind::Minus,
326        RawToken::Star => TokenKind::Star,
327        RawToken::Slash => TokenKind::Slash,
328        RawToken::Percent => TokenKind::Percent,
329        RawToken::Caret => TokenKind::Caret,
330        RawToken::Bang => TokenKind::Bang,
331        RawToken::Equal => TokenKind::Equal,
332        RawToken::NotEqual => TokenKind::NotEqual,
333        RawToken::Less => TokenKind::Less,
334        RawToken::LessEqual => TokenKind::LessEqual,
335        RawToken::Greater => TokenKind::Greater,
336        RawToken::GreaterEqual => TokenKind::GreaterEqual,
337        RawToken::PlusEqual => TokenKind::PlusEqual,
338        RawToken::FatArrow => TokenKind::FatArrow,
339        RawToken::RegexMatch => TokenKind::RegexMatch,
340        RawToken::LeftArrow => TokenKind::LeftArrow,
341        RawToken::RightArrow => TokenKind::RightArrow,
342    }
343}
344
345#[cfg(test)]
346mod tests {
347    use super::*;
348
349    #[test]
350    fn nested_comments_are_one_lossless_token() {
351        let source = "/* outer /* inner */ done */";
352        let outcome = lex(source);
353        assert!(outcome.diagnostics.is_empty());
354        assert_eq!(
355            outcome.tokens,
356            [Token::new(
357                TokenKind::BlockComment,
358                Span::new(0, source.len())
359            )]
360        );
361    }
362
363    #[test]
364    fn range_is_not_swallowed_by_float_lexing() {
365        let tokens = lex("1..2").tokens;
366        assert_eq!(
367            tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
368            [TokenKind::Integer, TokenKind::DotDot, TokenKind::Integer]
369        );
370    }
371}
open_cypher/lexer.rs

open_cypher/
lexer.rs
