relate_parser/lexer.rs
1use std::borrow::Cow;
2
3use crate::token::{lookup_keyword, Token, TokenType};
4use crate::unicode_identifier;
5
6mod ascii;
7mod number;
8mod string;
9
10/// Parse-affecting preview options: the ADR-0371 preview gates that widen
11/// what the parser accepts (the lexer's token stream and the parser's own
12/// grammar), resolved by the caller (the front derives them from the
13/// store's `RELATE_PREVIEW` set). The default is every gate closed --
14/// the dialect floor. One struct so a speculative or nested parse inherits
15/// its caller's dialect by plain field copy; the flags are `Copy`, never
16/// re-derived per site.
17#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
18pub struct ParseOptions {
19 /// The `obfuscated-literals` gate: a run of exactly six `*` lexes as
20 /// one [`TokenType::ObfuscatedLiteral`] instead of six `Star` tokens.
21 pub obfuscated_literals: bool,
22 /// The `mixed-label-syntax` gate (Neo4j's
23 /// mixed-label-syntax extra semantic feature):
24 /// legacy colon-conjunction (`:A:B`), GQL label-expression symbols
25 /// (`|`, `&`, `!`, `%`) and the `IS` introducer may mix within one
26 /// clause and within one label expression (`(n:A|B:C)`,
27 /// `(n IS S:R)`), with `:` parsed as an `&`-level conjunction
28 /// operator. Off (the floor), every CIP-40/CIP-14 mixing rejection
29 /// keeps firing.
30 pub mixed_label_syntax: bool,
31}
32
33/// Tokenizes a Cypher query string into a list of tokens.
34///
35/// The tokens borrow `input` (ADR-0505): they stay valid exactly as long as
36/// the query text does.
37pub fn tokenize(input: &str) -> Vec<Token<'_>> {
38 tokenize_with_options(input, ParseOptions::default())
39}
40
41/// Tokenizes with explicit [`ParseOptions`]; [`tokenize`] is the same call
42/// with every gate closed.
43pub fn tokenize_with_options(input: &str, options: ParseOptions) -> Vec<Token<'_>> {
44 let mut lexer = Lexer::new(input, options);
45 lexer.tokenize();
46 lexer.tokens
47}
48
49/// What [`skip_comment`] found at a position in the input.
50#[derive(Debug, Clone, Copy, PartialEq, Eq)]
51pub enum CommentSkip {
52 /// No comment starts here.
53 NotComment,
54 /// A comment ended just before this byte position.
55 Skipped(usize),
56 /// A block comment opened without a closing `*/` — the lexer's error
57 /// case.
58 Unterminated,
59}
60
61/// The Cypher comment starting at `input[pos..]` — the one spelling of the
62/// comment rule, shared by the lexer and by callers that re-read a literal
63/// token's text through the same grammar.
64///
65/// A `//` line comment runs through (and includes) its terminating newline;
66/// a `/* */` block comment does not nest and ends at the first `*/`.
67pub fn skip_comment(input: &[u8], pos: usize) -> CommentSkip {
68 if input.get(pos) == Some(&b'/') {
69 if input.get(pos + 1) == Some(&b'/') {
70 let mut end = pos + 2;
71 while end < input.len() && input[end] != b'\n' {
72 end += 1;
73 }
74 return CommentSkip::Skipped((end + 1).min(input.len()));
75 }
76 if input.get(pos + 1) == Some(&b'*') {
77 let mut end = pos + 2;
78 while end < input.len() {
79 if input[end] == b'*' && input.get(end + 1) == Some(&b'/') {
80 return CommentSkip::Skipped(end + 2);
81 }
82 end += 1;
83 }
84 return CommentSkip::Unterminated;
85 }
86 }
87 CommentSkip::NotComment
88}
89
90struct Lexer<'a> {
91 /// The input as `&str` — the borrowed-literal slice source (ADR-0505).
92 /// Slicing it costs one O(1) character-boundary check, where
93 /// `from_utf8` over the byte view would re-validate the whole token
94 /// text; an all-ASCII run classified by the fast path is valid by
95 /// construction and never re-validated.
96 text: &'a str,
97 /// `text` as bytes — the scanning view. One input, two immutable
98 /// borrowed views of the same data, created together in [`Lexer::new`].
99 input: &'a [u8],
100 pos: usize,
101 line: usize,
102 col: usize,
103 tokens: Vec<Token<'a>>,
104 options: ParseOptions,
105}
106
107impl<'a> Lexer<'a> {
108 fn new(input: &'a str, options: ParseOptions) -> Self {
109 Lexer {
110 text: input,
111 input: input.as_bytes(),
112 pos: 0,
113 line: 1,
114 col: 1,
115 tokens: Vec::new(),
116 options,
117 }
118 }
119
120 fn tokenize(&mut self) {
121 while self.pos < self.input.len() {
122 self.skip_whitespace();
123 if self.pos >= self.input.len() {
124 break;
125 }
126
127 let ch = self.peek();
128
129 let line = self.line;
130 let col = self.col;
131
132 match ch {
133 b'(' => {
134 self.advance();
135 self.emit(TokenType::LeftParen, "(", line, col);
136 }
137 b')' => {
138 self.advance();
139 self.emit(TokenType::RightParen, ")", line, col);
140 }
141 b'[' => {
142 self.advance();
143 self.emit(TokenType::LeftBracket, "[", line, col);
144 }
145 b']' => {
146 self.advance();
147 self.emit(TokenType::RightBracket, "]", line, col);
148 }
149 b'{' => {
150 self.advance();
151 self.emit(TokenType::LeftBrace, "{", line, col);
152 }
153 b'}' => {
154 self.advance();
155 self.emit(TokenType::RightBrace, "}", line, col);
156 }
157 b':' => {
158 if self.peek_at(1) == Some(b':') {
159 self.advance();
160 self.advance();
161 self.emit(TokenType::DoubleColon, "::", line, col);
162 } else {
163 self.advance();
164 self.emit(TokenType::Colon, ":", line, col);
165 }
166 }
167 b',' => {
168 self.advance();
169 self.emit(TokenType::Comma, ",", line, col);
170 }
171 b';' => {
172 self.advance();
173 self.emit(TokenType::Semicolon, ";", line, col);
174 }
175 b'|' => {
176 if self.peek_at(1) == Some(b'|') {
177 self.advance();
178 self.advance();
179 self.emit(TokenType::DoublePipe, "||", line, col);
180 } else {
181 self.advance();
182 self.emit(TokenType::Pipe, "|", line, col);
183 }
184 }
185 b'*' => {
186 // The obfuscated literal is the one gated spelling:
187 // exactly six stars under the preview gate lexes as one
188 // dummy literal; any other run (or the gate closed)
189 // keeps the ordinary `*` tokens.
190 if self.options.obfuscated_literals && self.star_run() == 6 {
191 for _ in 0..6 {
192 self.advance();
193 }
194 self.emit(TokenType::ObfuscatedLiteral, "******", line, col);
195 } else {
196 self.advance();
197 self.emit(TokenType::Star, "*", line, col);
198 }
199 }
200 b'+' => {
201 self.advance();
202 self.emit(TokenType::Plus, "+", line, col);
203 }
204 b'/' => {
205 // The comment rule lives once in `skip_comment`.
206 // `Unterminated` must not silently swallow the rest of
207 // the input (which could hide trailing statements such
208 // as `RETURN 1 /* DETACH DELETE n`).
209 match skip_comment(self.input, self.pos) {
210 CommentSkip::Skipped(end) => self.advance_by(end - self.pos),
211 CommentSkip::Unterminated => {
212 self.advance_by(self.input.len() - self.pos);
213 self.emit(TokenType::Error, "unterminated block comment", line, col);
214 }
215 CommentSkip::NotComment => {
216 self.advance();
217 self.emit(TokenType::Slash, "/", line, col);
218 }
219 }
220 }
221 b'%' => {
222 self.advance();
223 self.emit(TokenType::Percent, "%", line, col);
224 }
225 b'^' => {
226 self.advance();
227 self.emit(TokenType::Caret, "^", line, col);
228 }
229 b'=' => {
230 if self.peek_at(1) == Some(b'~') {
231 self.advance();
232 self.advance();
233 self.emit(TokenType::RegexMatch, "=~", line, col);
234 } else if self.peek_at(1) == Some(b'>') {
235 self.advance();
236 self.advance();
237 self.emit(TokenType::FatArrow, "=>", line, col);
238 } else {
239 self.advance();
240 self.emit(TokenType::Equals, "=", line, col);
241 }
242 }
243 b'.' => {
244 if self.peek_at(1) == Some(b'.') {
245 self.advance();
246 self.advance();
247 self.emit(TokenType::DotDot, "..", line, col);
248 } else if matches!(self.peek_at(1), Some(b'0'..=b'9')) {
249 self.read_number(line, col);
250 } else {
251 self.advance();
252 self.emit(TokenType::Dot, ".", line, col);
253 }
254 }
255 b'-' => {
256 if self.peek_at(1) == Some(b'>') {
257 self.advance();
258 self.advance();
259 self.emit(TokenType::ArrowRight, "->", line, col);
260 } else {
261 self.advance();
262 self.emit(TokenType::Dash, "-", line, col);
263 }
264 }
265 b'<' => {
266 if self.peek_at(1) == Some(b'-') {
267 self.advance();
268 self.advance();
269 self.emit(TokenType::ArrowLeft, "<-", line, col);
270 } else if self.peek_at(1) == Some(b'>') {
271 self.advance();
272 self.advance();
273 self.emit(TokenType::NotEquals, "<>", line, col);
274 } else if self.peek_at(1) == Some(b'=') {
275 self.advance();
276 self.advance();
277 self.emit(TokenType::LessEqual, "<=", line, col);
278 } else {
279 self.advance();
280 self.emit(TokenType::LessThan, "<", line, col);
281 }
282 }
283 b'>' => {
284 if self.peek_at(1) == Some(b'=') {
285 self.advance();
286 self.advance();
287 self.emit(TokenType::GreaterEqual, ">=", line, col);
288 } else {
289 self.advance();
290 self.emit(TokenType::GreaterThan, ">", line, col);
291 }
292 }
293 b'$' => {
294 self.advance();
295 self.read_parameter(line, col);
296 }
297 b'!' => {
298 self.advance();
299 self.emit(TokenType::Bang, "!", line, col);
300 }
301 b'&' => {
302 self.advance();
303 self.emit(TokenType::Ampersand, "&", line, col);
304 }
305 b'\'' | b'"' => {
306 self.read_string(line, col);
307 }
308 b'`' => {
309 self.read_backtick_identifier(line, col);
310 }
311 b'0'..=b'9' => {
312 self.read_number(line, col);
313 }
314 // The remaining bytes open either an identifier or one
315 // invalid character; the decoded-character decision lives in
316 // `at_ident_start`, so `read_identifier` has exactly this one
317 // call site (ADR-0182).
318 _ if self.at_ident_start() => {
319 self.read_identifier(line, col);
320 }
321 _ => {
322 // A decoded non-ASCII character that is not an identifier
323 // start is ONE invalid Unicode character, not one mojibake
324 // error per UTF-8 byte (LABS-188).
325 self.read_invalid_char(line, col);
326 }
327 }
328 }
329 }
330
331 fn peek(&self) -> u8 {
332 if self.pos >= self.input.len() {
333 0
334 } else {
335 self.input[self.pos]
336 }
337 }
338
339 fn peek_at(&self, offset: usize) -> Option<u8> {
340 let p = self.pos + offset;
341 if p >= self.input.len() {
342 None
343 } else {
344 Some(self.input[p])
345 }
346 }
347
348 fn advance(&mut self) -> u8 {
349 if self.pos >= self.input.len() {
350 return 0;
351 }
352 let ch = self.input[self.pos];
353 self.pos += 1;
354 if ch == b'\n' {
355 self.line += 1;
356 self.col = 1;
357 } else {
358 self.col += 1;
359 }
360 ch
361 }
362
363 /// Decode the character at the cursor as `(char, byte_len)` without
364 /// consuming it; `None` only on malformed UTF-8. ASCII is decided on the
365 /// byte; a byte >= 0x80 decodes its full sequence (via the string
366 /// module's [`string::decode_char`]).
367 fn peek_char(&self) -> Option<(char, usize)> {
368 string::decode_char(&self.input[self.pos..])
369 }
370
371 /// Consume `len` bytes — the byte length [`Lexer::peek_char`] reported for
372 /// the character at the cursor — so a multi-byte character is never split.
373 fn advance_by(&mut self, len: usize) {
374 for _ in 0..len {
375 self.advance();
376 }
377 }
378
379 /// The input slice `[start, end)` as `&'a str` — the borrowed-literal
380 /// source of ADR-0505. Both bounds sit on character boundaries by
381 /// construction (token starts, and ends at an ASCII delimiter or a
382 /// decoded-character edge), and the input itself came from a `&str`,
383 /// so slicing it is an O(1) boundary check that panics on a broken
384 /// invariant — never a fallback, and never a re-validation of the
385 /// bytes between the bounds (interior bytes are valid UTF-8 because
386 /// the input is).
387 fn slice(&self, start: usize, end: usize) -> &'a str {
388 &self.text[start..end]
389 }
390
391 /// The eight bytes at the cursor as one little-endian `u64` word for
392 /// the `ascii` fast path, zero-padded past the end of the input. The
393 /// padding bytes classify as none of the ASCII classes, so an
394 /// overhanging word never claims bytes the input does not have.
395 fn load_word(&self) -> u64 {
396 let end = (self.pos + 8).min(self.input.len());
397 let mut buf = [0u8; 8];
398 buf[..end - self.pos].copy_from_slice(&self.input[self.pos..end]);
399 u64::from_le_bytes(buf)
400 }
401
402 /// Emit a token. `literal` follows the ADR-0505 rule: an input slice
403 /// (identifiers, numbers, parameters, escape-free strings), a
404 /// `&'static str` (punctuation, fixed error texts), or an owned `String`
405 /// (escape-decoded / synthesized text) — `Into<Cow>` takes each without
406 /// a hidden copy, so nothing here allocates on the borrowed paths.
407 fn emit(&mut self, ty: TokenType, literal: impl Into<Cow<'a, str>>, line: usize, col: usize) {
408 self.tokens.push(Token::new(ty, literal, line, col));
409 }
410
411 /// Emit a [`TokenType::Error`] token carrying a symbolic openCypher-TCK
412 /// error-*detail* name (see [`Token::detail`]). The parser propagates the
413 /// detail onto the raised `ParseError` so a malformed literal reports its
414 /// specific TCK detail (`InvalidNumberLiteral`, `InvalidUnicodeLiteral`)
415 /// instead of the generic `UnexpectedSyntax`. `literal` takes the same
416 /// borrowed/owned forms as [`Lexer::emit`]; the error path is the one
417 /// place owned text is expected (`format!`-built messages).
418 fn emit_error_detail(
419 &mut self,
420 literal: impl Into<Cow<'a, str>>,
421 line: usize,
422 col: usize,
423 detail: &'static str,
424 ) {
425 self.tokens
426 .push(Token::error_with_detail(literal, line, col, detail));
427 }
428
429 fn skip_whitespace(&mut self) {
430 while self.pos < self.input.len() {
431 // ASCII whitespace (CIP-88 adds vertical tab and form feed to
432 // the previous \t \n \r + space set; verified on 2026.09.0) is
433 // consumed eight bytes per classification step (`ascii`,
434 // ADR-0505), the batch settling line/column exactly as
435 // byte-at-a-time `advance` would.
436 let w = self.load_word();
437 let n = ascii::ws_prefix(w);
438 if n > 0 {
439 self.advance_ws_run(w, n);
440 continue;
441 }
442 // CIP-88: the GQL/Java Unicode whitespace set (Zs + Zl + Zp
443 // categories plus separators, i.e. Rust's char::is_whitespace)
444 // is also accepted between tokens — e.g. NBSP U+00A0. Verified
445 // against Neo4j 2026.09.0. The byte at the cursor is not ASCII
446 // whitespace, so only a non-ASCII character can still qualify;
447 // the decoded path decides.
448 if let Some((c, len)) = self.peek_char() {
449 if c.is_whitespace() {
450 self.advance_by(len);
451 continue;
452 }
453 }
454 break;
455 }
456 }
457
458 /// Consume the `n`-byte ASCII whitespace run `[0, n)` of word `w`'s
459 /// lanes, settling line and column in one step — exactly `n` calls to
460 /// [`Lexer::advance`] would leave: the line advances by the run's
461 /// newline count, and the column restarts after the last one. The
462 /// `ascii` module hands back that settled run summary; the per-lane
463 /// bit layout never crosses the module boundary.
464 fn advance_ws_run(&mut self, w: u64, n: usize) {
465 let (newlines, after_last_newline) = ascii::ws_run_summary(w, n);
466 self.line += newlines;
467 if newlines == 0 {
468 self.col += n;
469 } else {
470 // The column after the last newline is the run's bytes past
471 // it, +1.
472 self.col = after_last_newline + 1;
473 }
474 self.pos += n;
475 }
476
477 /// The length of the run of consecutive `*` starting at the cursor.
478 /// The obfuscated-literal gate asks for exactly six.
479 fn star_run(&self) -> usize {
480 let mut n = 0usize;
481 while self.peek_at(n) == Some(b'*') {
482 n += 1;
483 }
484 n
485 }
486
487 /// True when the character at the cursor is an identifier start: an
488 /// ASCII `[A-Za-z_]` byte, decided from the class table with no
489 /// decoding, or a non-ASCII `char` accepted by the hand-written
490 /// Unicode predicate (ADR-0182) — only a byte >= 0x80 is decoded.
491 fn at_ident_start(&self) -> bool {
492 let b = self.peek();
493 if b < 0x80 {
494 return ascii::ascii_ident_start(b);
495 }
496 self.peek_char()
497 .is_some_and(|(c, _)| unicode_identifier::is_identifier_start(c))
498 }
499
500 /// Consume the remaining identifier-part run from `start` and return it
501 /// as the run's literal — the one canonical consume-and-derive for
502 /// identifier runs, consulted by `emit_malformed_number`,
503 /// `read_parameter`'s pass-through, and its part-only-start error block
504 /// (which differ only in message and detail tag). The run is a slice of
505 /// the input (borrowed, ADR-0505) — the decoded text never differs from
506 /// the raw bytes, so [`Lexer::slice`]'s O(1) boundary-checked slice is
507 /// the literal for ASCII and non-ASCII runs alike, and the one place
508 /// the text view is indexed.
509 ///
510 /// Pure-ASCII runs are consumed eight bytes per classification step
511 /// (`ascii`, ADR-0505); the first byte >= 0x80 hands the scan back to
512 /// the decoded-character path, which stays authoritative for the whole
513 /// non-ASCII stretch (ADR-0182).
514 fn take_ident_run(&mut self, start: usize) -> &'a str {
515 loop {
516 let n = ascii::ident_part_prefix(self.load_word());
517 if n > 0 {
518 // No identifier-part byte is a newline, so the batch
519 // advance only moves the column.
520 self.pos += n;
521 self.col += n;
522 }
523 if n == 8 {
524 continue;
525 }
526 // The byte at the cursor is not an ASCII identifier part: the
527 // run's terminator, or a non-ASCII character — the decoded
528 // path decides which, consuming a whole character if it
529 // continues the run.
530 if !self.take_ident_part() {
531 break;
532 }
533 }
534 self.slice(start, self.pos)
535 }
536
537 /// If the character at the current position is an identifier part,
538 /// consume it — all of its UTF-8 bytes — and return true. The character
539 /// is decoded through [`Lexer::peek_char`] (ASCII on the byte; a byte at
540 /// or above 0x80 as its full sequence) and classified with the
541 /// hand-written Unicode predicate, so a multi-byte character is never
542 /// split (ADR-0182).
543 fn take_ident_part(&mut self) -> bool {
544 let Some((c, len)) = self.peek_char() else {
545 return false;
546 };
547 if !unicode_identifier::is_identifier_part(c) {
548 return false;
549 }
550 self.advance_by(len);
551 true
552 }
553
554 /// Consume and report one character that opens neither a token nor an
555 /// identifier: a decoded non-ASCII character is ONE
556 /// `InvalidUnicodeCharacter` error token covering all of its UTF-8 bytes
557 /// (LABS-188); an ASCII byte keeps the historical untagged
558 /// `unexpected character:` error.
559 fn read_invalid_char(&mut self, line: usize, col: usize) {
560 if let Some((c, len)) = self.peek_char() {
561 if !c.is_ascii() {
562 self.advance_by(len);
563 self.emit_error_detail(
564 format!("invalid unicode character: {c}"),
565 line,
566 col,
567 "InvalidUnicodeCharacter",
568 );
569 return;
570 }
571 }
572 // An ASCII byte and a malformed byte take the same path: one byte
573 // consumed, one untagged `unexpected character:` token. For ASCII the
574 // decoded `char` and `char::from(byte)` are the same thing; malformed
575 // UTF-8 cannot occur in `&str` input — every cursor sits on a complete
576 // sequence — but keep the historical byte-wise report for the
577 // defensive branch.
578 let byte = self.advance();
579 // The one formatted (owned) form of the untagged ASCII error text.
580 self.emit(
581 TokenType::Error,
582 format!("unexpected character: {}", char::from(byte)),
583 line,
584 col,
585 );
586 }
587
588 fn read_identifier(&mut self, line: usize, col: usize) {
589 let start = self.pos;
590 let literal = self.take_ident_run(start);
591 // CIP-261: a lone `s`/`S` immediately followed by a quote starts an
592 // interpolated string literal (`s'...'` / `s"..."`). Anything else
593 // stays a normal identifier/keyword.
594 if literal.len() == 1
595 && literal.as_bytes()[0].eq_ignore_ascii_case(&b's')
596 && matches!(self.peek(), b'\'' | b'"')
597 {
598 self.read_interpolation(line, col);
599 return;
600 }
601 let tok_type = lookup_keyword(literal);
602 self.emit(tok_type, literal, line, col);
603 }
604
605 fn read_parameter(&mut self, line: usize, col: usize) {
606 // A parameter's first character is an identifier start or an ASCII
607 // digit — ADR-0182's Consequences prose, confirmed against the live
608 // Neo4j reference (LABS-209): a part-only character such as `€` is
609 // rejected with the reference's error shape; an arbitrary name (spaces,
610 // part-only characters) is spelled through the backtick form
611 // `$`name``, which this lexer does not offer yet (LABS-209's
612 // acceptance criteria).
613 // `(` after `$` is the dynamic-expression opener (`$(expr)`,
614 // e.g. `n IS LABELED $('A')`): it lexes as an empty parameter token
615 // whose expansion the parser owns, so it stays on the pass-through
616 // path below.
617 let start_ok = self.at_ident_start()
618 || self
619 .peek_char()
620 .is_some_and(|(c, _)| c.is_ascii_digit() || c == '(');
621 // One consume-and-derive for both outcomes: the accepted name and
622 // the rejected run are the same identifier-part run from the same
623 // position; the guard above only classifies its first character.
624 let start = self.pos;
625 let run = self.take_ident_run(start);
626 if !start_ok {
627 // The offending run becomes one error token, not a cascade (the
628 // `emit_malformed_number` shape).
629 let literal = if run.is_empty() {
630 Cow::Borrowed("Invalid input after '$': expected an identifier or an integer value")
631 } else {
632 Cow::Owned(format!(
633 "Invalid input '{run}': expected an identifier or an integer value"
634 ))
635 };
636 self.emit_error_detail(literal, line, col, "InvalidParameterName");
637 return;
638 }
639 self.emit(TokenType::Parameter, run, line, col);
640 }
641}
642
643#[cfg(test)]
644#[path = "lexer_tests.rs"]
645mod tests;