Skip to main content

relate_parser/
lexer.rs

1use std::borrow::Cow;
2
3use crate::token::{lookup_keyword, Token, TokenType};
4use crate::unicode_identifier;
5
6mod ascii;
7mod number;
8mod string;
9
10/// Parse-affecting preview options: the ADR-0371 preview gates that widen
11/// what the parser accepts (the lexer's token stream and the parser's own
12/// grammar), resolved by the caller (the front derives them from the
13/// store's `RELATE_PREVIEW` set). The default is every gate closed --
14/// the dialect floor. One struct so a speculative or nested parse inherits
15/// its caller's dialect by plain field copy; the flags are `Copy`, never
16/// re-derived per site.
17#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
18pub struct ParseOptions {
19    /// The `obfuscated-literals` gate: a run of exactly six `*` lexes as
20    /// one [`TokenType::ObfuscatedLiteral`] instead of six `Star` tokens.
21    pub obfuscated_literals: bool,
22    /// The `mixed-label-syntax` gate (Neo4j's
23    /// mixed-label-syntax extra semantic feature):
24    /// legacy colon-conjunction (`:A:B`), GQL label-expression symbols
25    /// (`|`, `&`, `!`, `%`) and the `IS` introducer may mix within one
26    /// clause and within one label expression (`(n:A|B:C)`,
27    /// `(n IS S:R)`), with `:` parsed as an `&`-level conjunction
28    /// operator. Off (the floor), every CIP-40/CIP-14 mixing rejection
29    /// keeps firing.
30    pub mixed_label_syntax: bool,
31}
32
33/// Tokenizes a Cypher query string into a list of tokens.
34///
35/// The tokens borrow `input` (ADR-0505): they stay valid exactly as long as
36/// the query text does.
37pub fn tokenize(input: &str) -> Vec<Token<'_>> {
38    tokenize_with_options(input, ParseOptions::default())
39}
40
41/// Tokenizes with explicit [`ParseOptions`]; [`tokenize`] is the same call
42/// with every gate closed.
43pub fn tokenize_with_options(input: &str, options: ParseOptions) -> Vec<Token<'_>> {
44    let mut lexer = Lexer::new(input, options);
45    lexer.tokenize();
46    lexer.tokens
47}
48
49/// What [`skip_comment`] found at a position in the input.
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum CommentSkip {
52    /// No comment starts here.
53    NotComment,
54    /// A comment ended just before this byte position.
55    Skipped(usize),
56    /// A block comment opened without a closing `*/` — the lexer's error
57    /// case.
58    Unterminated,
59}
60
61/// The Cypher comment starting at `input[pos..]` — the one spelling of the
62/// comment rule, shared by the lexer and by callers that re-read a literal
63/// token's text through the same grammar.
64///
65/// A `//` line comment runs through (and includes) its terminating newline;
66/// a `/* */` block comment does not nest and ends at the first `*/`.
67pub fn skip_comment(input: &[u8], pos: usize) -> CommentSkip {
68    if input.get(pos) == Some(&b'/') {
69        if input.get(pos + 1) == Some(&b'/') {
70            let mut end = pos + 2;
71            while end < input.len() && input[end] != b'\n' {
72                end += 1;
73            }
74            return CommentSkip::Skipped((end + 1).min(input.len()));
75        }
76        if input.get(pos + 1) == Some(&b'*') {
77            let mut end = pos + 2;
78            while end < input.len() {
79                if input[end] == b'*' && input.get(end + 1) == Some(&b'/') {
80                    return CommentSkip::Skipped(end + 2);
81                }
82                end += 1;
83            }
84            return CommentSkip::Unterminated;
85        }
86    }
87    CommentSkip::NotComment
88}
89
90struct Lexer<'a> {
91    /// The input as `&str` — the borrowed-literal slice source (ADR-0505).
92    /// Slicing it costs one O(1) character-boundary check, where
93    /// `from_utf8` over the byte view would re-validate the whole token
94    /// text; an all-ASCII run classified by the fast path is valid by
95    /// construction and never re-validated.
96    text: &'a str,
97    /// `text` as bytes — the scanning view. One input, two immutable
98    /// borrowed views of the same data, created together in [`Lexer::new`].
99    input: &'a [u8],
100    pos: usize,
101    line: usize,
102    col: usize,
103    tokens: Vec<Token<'a>>,
104    options: ParseOptions,
105}
106
107impl<'a> Lexer<'a> {
108    fn new(input: &'a str, options: ParseOptions) -> Self {
109        Lexer {
110            text: input,
111            input: input.as_bytes(),
112            pos: 0,
113            line: 1,
114            col: 1,
115            tokens: Vec::new(),
116            options,
117        }
118    }
119
120    fn tokenize(&mut self) {
121        while self.pos < self.input.len() {
122            self.skip_whitespace();
123            if self.pos >= self.input.len() {
124                break;
125            }
126
127            let ch = self.peek();
128
129            let line = self.line;
130            let col = self.col;
131
132            match ch {
133                b'(' => {
134                    self.advance();
135                    self.emit(TokenType::LeftParen, "(", line, col);
136                }
137                b')' => {
138                    self.advance();
139                    self.emit(TokenType::RightParen, ")", line, col);
140                }
141                b'[' => {
142                    self.advance();
143                    self.emit(TokenType::LeftBracket, "[", line, col);
144                }
145                b']' => {
146                    self.advance();
147                    self.emit(TokenType::RightBracket, "]", line, col);
148                }
149                b'{' => {
150                    self.advance();
151                    self.emit(TokenType::LeftBrace, "{", line, col);
152                }
153                b'}' => {
154                    self.advance();
155                    self.emit(TokenType::RightBrace, "}", line, col);
156                }
157                b':' => {
158                    if self.peek_at(1) == Some(b':') {
159                        self.advance();
160                        self.advance();
161                        self.emit(TokenType::DoubleColon, "::", line, col);
162                    } else {
163                        self.advance();
164                        self.emit(TokenType::Colon, ":", line, col);
165                    }
166                }
167                b',' => {
168                    self.advance();
169                    self.emit(TokenType::Comma, ",", line, col);
170                }
171                b';' => {
172                    self.advance();
173                    self.emit(TokenType::Semicolon, ";", line, col);
174                }
175                b'|' => {
176                    if self.peek_at(1) == Some(b'|') {
177                        self.advance();
178                        self.advance();
179                        self.emit(TokenType::DoublePipe, "||", line, col);
180                    } else {
181                        self.advance();
182                        self.emit(TokenType::Pipe, "|", line, col);
183                    }
184                }
185                b'*' => {
186                    // The obfuscated literal is the one gated spelling:
187                    // exactly six stars under the preview gate lexes as one
188                    // dummy literal; any other run (or the gate closed)
189                    // keeps the ordinary `*` tokens.
190                    if self.options.obfuscated_literals && self.star_run() == 6 {
191                        for _ in 0..6 {
192                            self.advance();
193                        }
194                        self.emit(TokenType::ObfuscatedLiteral, "******", line, col);
195                    } else {
196                        self.advance();
197                        self.emit(TokenType::Star, "*", line, col);
198                    }
199                }
200                b'+' => {
201                    self.advance();
202                    self.emit(TokenType::Plus, "+", line, col);
203                }
204                b'/' => {
205                    // The comment rule lives once in `skip_comment`.
206                    // `Unterminated` must not silently swallow the rest of
207                    // the input (which could hide trailing statements such
208                    // as `RETURN 1 /* DETACH DELETE n`).
209                    match skip_comment(self.input, self.pos) {
210                        CommentSkip::Skipped(end) => self.advance_by(end - self.pos),
211                        CommentSkip::Unterminated => {
212                            self.advance_by(self.input.len() - self.pos);
213                            self.emit(TokenType::Error, "unterminated block comment", line, col);
214                        }
215                        CommentSkip::NotComment => {
216                            self.advance();
217                            self.emit(TokenType::Slash, "/", line, col);
218                        }
219                    }
220                }
221                b'%' => {
222                    self.advance();
223                    self.emit(TokenType::Percent, "%", line, col);
224                }
225                b'^' => {
226                    self.advance();
227                    self.emit(TokenType::Caret, "^", line, col);
228                }
229                b'=' => {
230                    if self.peek_at(1) == Some(b'~') {
231                        self.advance();
232                        self.advance();
233                        self.emit(TokenType::RegexMatch, "=~", line, col);
234                    } else if self.peek_at(1) == Some(b'>') {
235                        self.advance();
236                        self.advance();
237                        self.emit(TokenType::FatArrow, "=>", line, col);
238                    } else {
239                        self.advance();
240                        self.emit(TokenType::Equals, "=", line, col);
241                    }
242                }
243                b'.' => {
244                    if self.peek_at(1) == Some(b'.') {
245                        self.advance();
246                        self.advance();
247                        self.emit(TokenType::DotDot, "..", line, col);
248                    } else if matches!(self.peek_at(1), Some(b'0'..=b'9')) {
249                        self.read_number(line, col);
250                    } else {
251                        self.advance();
252                        self.emit(TokenType::Dot, ".", line, col);
253                    }
254                }
255                b'-' => {
256                    if self.peek_at(1) == Some(b'>') {
257                        self.advance();
258                        self.advance();
259                        self.emit(TokenType::ArrowRight, "->", line, col);
260                    } else {
261                        self.advance();
262                        self.emit(TokenType::Dash, "-", line, col);
263                    }
264                }
265                b'<' => {
266                    if self.peek_at(1) == Some(b'-') {
267                        self.advance();
268                        self.advance();
269                        self.emit(TokenType::ArrowLeft, "<-", line, col);
270                    } else if self.peek_at(1) == Some(b'>') {
271                        self.advance();
272                        self.advance();
273                        self.emit(TokenType::NotEquals, "<>", line, col);
274                    } else if self.peek_at(1) == Some(b'=') {
275                        self.advance();
276                        self.advance();
277                        self.emit(TokenType::LessEqual, "<=", line, col);
278                    } else {
279                        self.advance();
280                        self.emit(TokenType::LessThan, "<", line, col);
281                    }
282                }
283                b'>' => {
284                    if self.peek_at(1) == Some(b'=') {
285                        self.advance();
286                        self.advance();
287                        self.emit(TokenType::GreaterEqual, ">=", line, col);
288                    } else {
289                        self.advance();
290                        self.emit(TokenType::GreaterThan, ">", line, col);
291                    }
292                }
293                b'$' => {
294                    self.advance();
295                    self.read_parameter(line, col);
296                }
297                b'!' => {
298                    self.advance();
299                    self.emit(TokenType::Bang, "!", line, col);
300                }
301                b'&' => {
302                    self.advance();
303                    self.emit(TokenType::Ampersand, "&", line, col);
304                }
305                b'\'' | b'"' => {
306                    self.read_string(line, col);
307                }
308                b'`' => {
309                    self.read_backtick_identifier(line, col);
310                }
311                b'0'..=b'9' => {
312                    self.read_number(line, col);
313                }
314                // The remaining bytes open either an identifier or one
315                // invalid character; the decoded-character decision lives in
316                // `at_ident_start`, so `read_identifier` has exactly this one
317                // call site (ADR-0182).
318                _ if self.at_ident_start() => {
319                    self.read_identifier(line, col);
320                }
321                _ => {
322                    // A decoded non-ASCII character that is not an identifier
323                    // start is ONE invalid Unicode character, not one mojibake
324                    // error per UTF-8 byte (LABS-188).
325                    self.read_invalid_char(line, col);
326                }
327            }
328        }
329    }
330
331    fn peek(&self) -> u8 {
332        if self.pos >= self.input.len() {
333            0
334        } else {
335            self.input[self.pos]
336        }
337    }
338
339    fn peek_at(&self, offset: usize) -> Option<u8> {
340        let p = self.pos + offset;
341        if p >= self.input.len() {
342            None
343        } else {
344            Some(self.input[p])
345        }
346    }
347
348    fn advance(&mut self) -> u8 {
349        if self.pos >= self.input.len() {
350            return 0;
351        }
352        let ch = self.input[self.pos];
353        self.pos += 1;
354        if ch == b'\n' {
355            self.line += 1;
356            self.col = 1;
357        } else {
358            self.col += 1;
359        }
360        ch
361    }
362
363    /// Decode the character at the cursor as `(char, byte_len)` without
364    /// consuming it; `None` only on malformed UTF-8. ASCII is decided on the
365    /// byte; a byte >= 0x80 decodes its full sequence (via the string
366    /// module's [`string::decode_char`]).
367    fn peek_char(&self) -> Option<(char, usize)> {
368        string::decode_char(&self.input[self.pos..])
369    }
370
371    /// Consume `len` bytes — the byte length [`Lexer::peek_char`] reported for
372    /// the character at the cursor — so a multi-byte character is never split.
373    fn advance_by(&mut self, len: usize) {
374        for _ in 0..len {
375            self.advance();
376        }
377    }
378
379    /// The input slice `[start, end)` as `&'a str` — the borrowed-literal
380    /// source of ADR-0505. Both bounds sit on character boundaries by
381    /// construction (token starts, and ends at an ASCII delimiter or a
382    /// decoded-character edge), and the input itself came from a `&str`,
383    /// so slicing it is an O(1) boundary check that panics on a broken
384    /// invariant — never a fallback, and never a re-validation of the
385    /// bytes between the bounds (interior bytes are valid UTF-8 because
386    /// the input is).
387    fn slice(&self, start: usize, end: usize) -> &'a str {
388        &self.text[start..end]
389    }
390
391    /// The eight bytes at the cursor as one little-endian `u64` word for
392    /// the `ascii` fast path, zero-padded past the end of the input. The
393    /// padding bytes classify as none of the ASCII classes, so an
394    /// overhanging word never claims bytes the input does not have.
395    fn load_word(&self) -> u64 {
396        let end = (self.pos + 8).min(self.input.len());
397        let mut buf = [0u8; 8];
398        buf[..end - self.pos].copy_from_slice(&self.input[self.pos..end]);
399        u64::from_le_bytes(buf)
400    }
401
402    /// Emit a token. `literal` follows the ADR-0505 rule: an input slice
403    /// (identifiers, numbers, parameters, escape-free strings), a
404    /// `&'static str` (punctuation, fixed error texts), or an owned `String`
405    /// (escape-decoded / synthesized text) — `Into<Cow>` takes each without
406    /// a hidden copy, so nothing here allocates on the borrowed paths.
407    fn emit(&mut self, ty: TokenType, literal: impl Into<Cow<'a, str>>, line: usize, col: usize) {
408        self.tokens.push(Token::new(ty, literal, line, col));
409    }
410
411    /// Emit a [`TokenType::Error`] token carrying a symbolic openCypher-TCK
412    /// error-*detail* name (see [`Token::detail`]). The parser propagates the
413    /// detail onto the raised `ParseError` so a malformed literal reports its
414    /// specific TCK detail (`InvalidNumberLiteral`, `InvalidUnicodeLiteral`)
415    /// instead of the generic `UnexpectedSyntax`. `literal` takes the same
416    /// borrowed/owned forms as [`Lexer::emit`]; the error path is the one
417    /// place owned text is expected (`format!`-built messages).
418    fn emit_error_detail(
419        &mut self,
420        literal: impl Into<Cow<'a, str>>,
421        line: usize,
422        col: usize,
423        detail: &'static str,
424    ) {
425        self.tokens
426            .push(Token::error_with_detail(literal, line, col, detail));
427    }
428
429    fn skip_whitespace(&mut self) {
430        while self.pos < self.input.len() {
431            // ASCII whitespace (CIP-88 adds vertical tab and form feed to
432            // the previous \t \n \r + space set; verified on 2026.09.0) is
433            // consumed eight bytes per classification step (`ascii`,
434            // ADR-0505), the batch settling line/column exactly as
435            // byte-at-a-time `advance` would.
436            let w = self.load_word();
437            let n = ascii::ws_prefix(w);
438            if n > 0 {
439                self.advance_ws_run(w, n);
440                continue;
441            }
442            // CIP-88: the GQL/Java Unicode whitespace set (Zs + Zl + Zp
443            // categories plus separators, i.e. Rust's char::is_whitespace)
444            // is also accepted between tokens — e.g. NBSP U+00A0. Verified
445            // against Neo4j 2026.09.0. The byte at the cursor is not ASCII
446            // whitespace, so only a non-ASCII character can still qualify;
447            // the decoded path decides.
448            if let Some((c, len)) = self.peek_char() {
449                if c.is_whitespace() {
450                    self.advance_by(len);
451                    continue;
452                }
453            }
454            break;
455        }
456    }
457
458    /// Consume the `n`-byte ASCII whitespace run `[0, n)` of word `w`'s
459    /// lanes, settling line and column in one step — exactly `n` calls to
460    /// [`Lexer::advance`] would leave: the line advances by the run's
461    /// newline count, and the column restarts after the last one. The
462    /// `ascii` module hands back that settled run summary; the per-lane
463    /// bit layout never crosses the module boundary.
464    fn advance_ws_run(&mut self, w: u64, n: usize) {
465        let (newlines, after_last_newline) = ascii::ws_run_summary(w, n);
466        self.line += newlines;
467        if newlines == 0 {
468            self.col += n;
469        } else {
470            // The column after the last newline is the run's bytes past
471            // it, +1.
472            self.col = after_last_newline + 1;
473        }
474        self.pos += n;
475    }
476
477    /// The length of the run of consecutive `*` starting at the cursor.
478    /// The obfuscated-literal gate asks for exactly six.
479    fn star_run(&self) -> usize {
480        let mut n = 0usize;
481        while self.peek_at(n) == Some(b'*') {
482            n += 1;
483        }
484        n
485    }
486
487    /// True when the character at the cursor is an identifier start: an
488    /// ASCII `[A-Za-z_]` byte, decided from the class table with no
489    /// decoding, or a non-ASCII `char` accepted by the hand-written
490    /// Unicode predicate (ADR-0182) — only a byte >= 0x80 is decoded.
491    fn at_ident_start(&self) -> bool {
492        let b = self.peek();
493        if b < 0x80 {
494            return ascii::ascii_ident_start(b);
495        }
496        self.peek_char()
497            .is_some_and(|(c, _)| unicode_identifier::is_identifier_start(c))
498    }
499
500    /// Consume the remaining identifier-part run from `start` and return it
501    /// as the run's literal — the one canonical consume-and-derive for
502    /// identifier runs, consulted by `emit_malformed_number`,
503    /// `read_parameter`'s pass-through, and its part-only-start error block
504    /// (which differ only in message and detail tag). The run is a slice of
505    /// the input (borrowed, ADR-0505) — the decoded text never differs from
506    /// the raw bytes, so [`Lexer::slice`]'s O(1) boundary-checked slice is
507    /// the literal for ASCII and non-ASCII runs alike, and the one place
508    /// the text view is indexed.
509    ///
510    /// Pure-ASCII runs are consumed eight bytes per classification step
511    /// (`ascii`, ADR-0505); the first byte >= 0x80 hands the scan back to
512    /// the decoded-character path, which stays authoritative for the whole
513    /// non-ASCII stretch (ADR-0182).
514    fn take_ident_run(&mut self, start: usize) -> &'a str {
515        loop {
516            let n = ascii::ident_part_prefix(self.load_word());
517            if n > 0 {
518                // No identifier-part byte is a newline, so the batch
519                // advance only moves the column.
520                self.pos += n;
521                self.col += n;
522            }
523            if n == 8 {
524                continue;
525            }
526            // The byte at the cursor is not an ASCII identifier part: the
527            // run's terminator, or a non-ASCII character — the decoded
528            // path decides which, consuming a whole character if it
529            // continues the run.
530            if !self.take_ident_part() {
531                break;
532            }
533        }
534        self.slice(start, self.pos)
535    }
536
537    /// If the character at the current position is an identifier part,
538    /// consume it — all of its UTF-8 bytes — and return true. The character
539    /// is decoded through [`Lexer::peek_char`] (ASCII on the byte; a byte at
540    /// or above 0x80 as its full sequence) and classified with the
541    /// hand-written Unicode predicate, so a multi-byte character is never
542    /// split (ADR-0182).
543    fn take_ident_part(&mut self) -> bool {
544        let Some((c, len)) = self.peek_char() else {
545            return false;
546        };
547        if !unicode_identifier::is_identifier_part(c) {
548            return false;
549        }
550        self.advance_by(len);
551        true
552    }
553
554    /// Consume and report one character that opens neither a token nor an
555    /// identifier: a decoded non-ASCII character is ONE
556    /// `InvalidUnicodeCharacter` error token covering all of its UTF-8 bytes
557    /// (LABS-188); an ASCII byte keeps the historical untagged
558    /// `unexpected character:` error.
559    fn read_invalid_char(&mut self, line: usize, col: usize) {
560        if let Some((c, len)) = self.peek_char() {
561            if !c.is_ascii() {
562                self.advance_by(len);
563                self.emit_error_detail(
564                    format!("invalid unicode character: {c}"),
565                    line,
566                    col,
567                    "InvalidUnicodeCharacter",
568                );
569                return;
570            }
571        }
572        // An ASCII byte and a malformed byte take the same path: one byte
573        // consumed, one untagged `unexpected character:` token. For ASCII the
574        // decoded `char` and `char::from(byte)` are the same thing; malformed
575        // UTF-8 cannot occur in `&str` input — every cursor sits on a complete
576        // sequence — but keep the historical byte-wise report for the
577        // defensive branch.
578        let byte = self.advance();
579        // The one formatted (owned) form of the untagged ASCII error text.
580        self.emit(
581            TokenType::Error,
582            format!("unexpected character: {}", char::from(byte)),
583            line,
584            col,
585        );
586    }
587
588    fn read_identifier(&mut self, line: usize, col: usize) {
589        let start = self.pos;
590        let literal = self.take_ident_run(start);
591        // CIP-261: a lone `s`/`S` immediately followed by a quote starts an
592        // interpolated string literal (`s'...'` / `s"..."`). Anything else
593        // stays a normal identifier/keyword.
594        if literal.len() == 1
595            && literal.as_bytes()[0].eq_ignore_ascii_case(&b's')
596            && matches!(self.peek(), b'\'' | b'"')
597        {
598            self.read_interpolation(line, col);
599            return;
600        }
601        let tok_type = lookup_keyword(literal);
602        self.emit(tok_type, literal, line, col);
603    }
604
605    fn read_parameter(&mut self, line: usize, col: usize) {
606        // A parameter's first character is an identifier start or an ASCII
607        // digit — ADR-0182's Consequences prose, confirmed against the live
608        // Neo4j reference (LABS-209): a part-only character such as `€` is
609        // rejected with the reference's error shape; an arbitrary name (spaces,
610        // part-only characters) is spelled through the backtick form
611        // `$`name``, which this lexer does not offer yet (LABS-209's
612        // acceptance criteria).
613        // `(` after `$` is the dynamic-expression opener (`$(expr)`,
614        // e.g. `n IS LABELED $('A')`): it lexes as an empty parameter token
615        // whose expansion the parser owns, so it stays on the pass-through
616        // path below.
617        let start_ok = self.at_ident_start()
618            || self
619                .peek_char()
620                .is_some_and(|(c, _)| c.is_ascii_digit() || c == '(');
621        // One consume-and-derive for both outcomes: the accepted name and
622        // the rejected run are the same identifier-part run from the same
623        // position; the guard above only classifies its first character.
624        let start = self.pos;
625        let run = self.take_ident_run(start);
626        if !start_ok {
627            // The offending run becomes one error token, not a cascade (the
628            // `emit_malformed_number` shape).
629            let literal = if run.is_empty() {
630                Cow::Borrowed("Invalid input after '$': expected an identifier or an integer value")
631            } else {
632                Cow::Owned(format!(
633                    "Invalid input '{run}': expected an identifier or an integer value"
634                ))
635            };
636            self.emit_error_detail(literal, line, col, "InvalidParameterName");
637            return;
638        }
639        self.emit(TokenType::Parameter, run, line, col);
640    }
641}
642
643#[cfg(test)]
644#[path = "lexer_tests.rs"]
645mod tests;