relate_parser/token.rs
1use std::borrow::Cow;
2
3/// Token types for the Cypher lexer.
4#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
5pub enum TokenType {
6 // Special tokens
7 EOF,
8 Error,
9 Identifier,
10 Integer,
11 Float,
12 String,
13 /// CIP-261 interpolated string literal `s'...'` / `s"..."`. Payload is
14 /// the body between the quotes with standard escapes resolved; `\{` and
15 /// `\}` stay as two characters so the parser can tell an escaped literal
16 /// brace from an expression delimiter, and raw `{`/`}` are kept for the
17 /// parser to split into text/expression parts.
18 InterpString,
19 /// The Neo4j internal obfuscated literal `******` (exactly six stars),
20 /// lexed only under the `obfuscated-literals` preview gate: a dummy
21 /// `false` for semantic analysis, never meant to execute. Without the
22 /// gate the run lexes as ordinary `*` tokens and the query fails with
23 /// the parser's own syntax error, Neo4j's disabled-form behavior.
24 ObfuscatedLiteral,
25 Parameter, // $param
26
27 // Punctuation
28 LeftParen, // (
29 RightParen, // )
30 LeftBracket, // [
31 RightBracket, // ]
32 LeftBrace, // {
33 RightBrace, // }
34 Colon, // :
35 DoubleColon, // ::
36 Comma, // ,
37 Dot, // .
38 Semicolon, // ;
39 Pipe, // |
40 DoublePipe, // ||
41 Dash, // -
42 ArrowRight, // ->
43 ArrowLeft, // <-
44 FatArrow, // => (CIP-176/CIP-187 graph type declaration syntax)
45 Star, // *
46 Plus, // +
47 Slash, // /
48 Percent, // %
49 Caret, // ^
50 DotDot, // ..
51 Bang, // ! (CIP-102 NOT NULL type shorthand; also GQL label negation, CIP-14)
52 Ampersand, // & (GQL label expression conjunction, CIP-14)
53
54 // Comparison operators
55 Equals, // =
56 NotEquals, // <>
57 LessThan, // <
58 GreaterThan, // >
59 LessEqual, // <=
60 GreaterEqual, // >=
61 RegexMatch, // =~
62
63 // Keywords
64 MATCH,
65 OPTIONAL,
66 CREATE,
67 INSERT,
68 MERGE,
69 DELETE,
70 DETACH,
71 SET,
72 REMOVE,
73 RETURN,
74 WITH,
75 UNWIND,
76 WHERE,
77 ORDER,
78 BY,
79 ASC,
80 DESC,
81 LIMIT,
82 SKIP,
83 AS,
84 DISTINCT,
85 AND,
86 OR,
87 NOT,
88 XOR,
89 IN,
90 STARTS,
91 ENDS,
92 CONTAINS,
93 IS,
94 NULL,
95 TRUE,
96 FALSE,
97 CASE,
98 WHEN,
99 THEN,
100 ELSE,
101 END,
102 FOREACH,
103 CALL,
104 YIELD,
105 UNION,
106 ALL,
107 ON,
108 INDEX,
109 CONSTRAINT,
110 FOR,
111 REQUIRE,
112 UNIQUE,
113 EXISTS,
114 IF,
115 DROP,
116 VECTOR,
117 POINT,
118 OPTIONS,
119 NODE,
120 REL,
121 COUNT,
122 COLLECT,
123 LOAD,
124 FROM,
125 CSV,
126 REDUCE,
127 SPATIAL,
128 EXPLAIN,
129 PROFILE,
130 INFINITY, // Infinity/Inf literal (CIP-55)
131 NAN, // NaN literal (CIP-55)
132}
133
134impl TokenType {
135 /// Returns true if this token type is a keyword (MATCH..PROFILE range).
136 pub fn is_keyword(self) -> bool {
137 matches!(
138 self,
139 TokenType::MATCH
140 | TokenType::OPTIONAL
141 | TokenType::CREATE
142 | TokenType::INSERT
143 | TokenType::MERGE
144 | TokenType::DELETE
145 | TokenType::DETACH
146 | TokenType::SET
147 | TokenType::REMOVE
148 | TokenType::RETURN
149 | TokenType::WITH
150 | TokenType::UNWIND
151 | TokenType::WHERE
152 | TokenType::ORDER
153 | TokenType::BY
154 | TokenType::ASC
155 | TokenType::DESC
156 | TokenType::LIMIT
157 | TokenType::SKIP
158 | TokenType::AS
159 | TokenType::DISTINCT
160 | TokenType::AND
161 | TokenType::OR
162 | TokenType::NOT
163 | TokenType::XOR
164 | TokenType::IN
165 | TokenType::STARTS
166 | TokenType::ENDS
167 | TokenType::CONTAINS
168 | TokenType::IS
169 | TokenType::NULL
170 | TokenType::TRUE
171 | TokenType::FALSE
172 | TokenType::CASE
173 | TokenType::WHEN
174 | TokenType::THEN
175 | TokenType::ELSE
176 | TokenType::END
177 | TokenType::FOREACH
178 | TokenType::CALL
179 | TokenType::YIELD
180 | TokenType::UNION
181 | TokenType::ALL
182 | TokenType::ON
183 | TokenType::INDEX
184 | TokenType::CONSTRAINT
185 | TokenType::FOR
186 | TokenType::REQUIRE
187 | TokenType::UNIQUE
188 | TokenType::EXISTS
189 | TokenType::IF
190 | TokenType::DROP
191 | TokenType::VECTOR
192 | TokenType::POINT
193 | TokenType::OPTIONS
194 | TokenType::NODE
195 | TokenType::REL
196 | TokenType::COUNT
197 | TokenType::COLLECT
198 | TokenType::LOAD
199 | TokenType::FROM
200 | TokenType::CSV
201 | TokenType::REDUCE
202 | TokenType::SPATIAL
203 | TokenType::EXPLAIN
204 | TokenType::PROFILE
205 | TokenType::INFINITY
206 | TokenType::NAN
207 )
208 }
209
210 /// Returns true if the token can be used as an identifier (variable,
211 /// alias, map key, ...): `Identifier` or any keyword except the
212 /// [`CLAUSE_KEYWORDS`].
213 pub fn is_ident_like(self) -> bool {
214 self.is_schema_name() && !CLAUSE_KEYWORDS.contains(&self)
215 }
216
217 /// Returns true if the token is `Identifier` or any keyword: openCypher's
218 /// `SchemaName ::= SymbolicName | ReservedWord`, for positions where the
219 /// grammar already requires a name, so no keyword there starts a clause
220 /// -- property keys after `.`, the LET binding variable, and label and
221 /// relationship-type names. ADR-0169: a strict superset of
222 /// `is_ident_like()`, so `(o:Order)` and `-[:ORDER]->` parse unquoted
223 /// with the name kept as written.
224 ///
225 /// This token class is unrelated to, and deliberately wider than, the
226 /// narrower thing this crate also calls a schema name: the
227 /// index/constraint name `ast::SchemaName` that
228 /// `parse_optional_schema_name` / `parse_required_schema_name` read
229 /// (`Identifier`, `INDEX`, `CONSTRAINT` or a `$param` only).
230 pub fn is_schema_name(self) -> bool {
231 self == TokenType::Identifier || self.is_keyword()
232 }
233}
234
235/// The clause keywords: keyword tokens that start or continue a clause
236/// (`MATCH`, `ORDER BY`, `AS`, `DISTINCT`, ...). `is_ident_like()` rejects
237/// them as variables, aliases and map keys, so `(WHERE ...)` does not read
238/// `WHERE` as the element variable. ADR-0169: they are still label and
239/// relationship-type names (`is_schema_name()`).
240///
241/// `RETURN` is not one of them. CIP-150 NOTES.md Q2 (`@neo4j-verified`
242/// against live Neo4j 2026.05, both CYPHER 5 and CYPHER 25): `RETURN` is a
243/// non-reserved word usable as an ordinary variable/alias name (e.g.
244/// `WITH 1 AS RETURN RETURN RETURN AS r`) -- `is_ident_like()` is only
245/// checked in positions the grammar already knows unambiguously expect an
246/// identifier (after AS, as a pattern/FOREACH variable, ...), so it never
247/// shadows RETURN's role starting a new clause.
248pub const CLAUSE_KEYWORDS: [TokenType; 22] = [
249 TokenType::MATCH,
250 TokenType::OPTIONAL,
251 TokenType::CREATE,
252 TokenType::INSERT,
253 TokenType::MERGE,
254 TokenType::DELETE,
255 TokenType::DETACH,
256 TokenType::SET,
257 TokenType::REMOVE,
258 TokenType::WITH,
259 TokenType::UNWIND,
260 TokenType::WHERE,
261 TokenType::ORDER,
262 TokenType::BY,
263 TokenType::LIMIT,
264 TokenType::SKIP,
265 TokenType::AS,
266 TokenType::DISTINCT,
267 TokenType::FOREACH,
268 TokenType::UNION,
269 TokenType::REQUIRE,
270 TokenType::DROP,
271];
272
273/// The contextual `FIELDTERMINATOR` word of LOAD CSV: an identifier-spelled
274/// keyword the clause parser discriminates on by spelling. The one
275/// constant, shared with the statement normalizer's grammar guard (a
276/// `String` after this word is a literal-only position there) — the
277/// spelling is a centralized fact, never re-derived at a second site.
278pub(crate) const FIELDTERMINATOR_WORD: &str = "FIELDTERMINATOR";
279
280/// Whether an identifier's literal spells the LOAD CSV
281/// [`FIELDTERMINATOR_WORD`] (case-insensitively, as the clause grammar
282/// reads it).
283pub(crate) fn is_field_terminator_word(literal: &str) -> bool {
284 literal.eq_ignore_ascii_case(FIELDTERMINATOR_WORD)
285}
286
287/// The reserved-keyword clause-starter tokens: the keyword tokens that
288/// start a new clause at clause position (`MATCH`, `CREATE`, `UNWIND`,
289/// `FOR` (CIP-250), ...). The grammar authority is the clause dispatch
290/// (`Parser::parse_clause_inner`); this table is the derived vocabulary
291/// every *lookahead* over the same fact consults instead of re-spelling a
292/// second hand-maintained list -- through [`is_clause_boundary`], by
293/// `Parser::group_by_ends_clause` (whose round-5 comment records the
294/// LOAD/DROP drift that motivated it) and the simple-CASE lookahead -- and
295/// the dispatch-coverage pin test. A new clause-start dispatch arm
296/// extends this one table. `OPTIONAL` is a member: it starts a clause
297/// only immediately before `MATCH`/`CALL` (the arm that checks what
298/// follows). The contextual (identifier-spelled) starters are the
299/// separate [`CONTEXTUAL_CLAUSE_WORDS`] below.
300pub const CLAUSE_STARTER_TOKENS: &[TokenType] = &[
301 TokenType::MATCH,
302 TokenType::OPTIONAL,
303 TokenType::CREATE,
304 TokenType::INSERT,
305 TokenType::MERGE,
306 TokenType::DELETE,
307 TokenType::DETACH,
308 TokenType::SET,
309 TokenType::REMOVE,
310 TokenType::UNWIND,
311 // CIP-250 (GQ10): `FOR <variable> IN <expression>`.
312 TokenType::FOR,
313 TokenType::FOREACH,
314 TokenType::CALL,
315 TokenType::UNION,
316 TokenType::LOAD,
317 TokenType::DROP,
318];
319
320/// The one derived "this token ends the enclosing clause" predicate: a
321/// reserved clause keyword ([`CLAUSE_KEYWORDS`]) or a reserved clause
322/// starter ([`CLAUSE_STARTER_TOKENS`]). Every lookahead that must stop at a
323/// clause boundary consults this instead of re-spelling the vocabulary —
324/// `Parser::group_by_ends_clause` and the simple-CASE lookahead
325/// (`ends_case_test_scan`), each adding only its own statement/item
326/// terminals.
327pub(crate) fn is_clause_boundary(ty: TokenType) -> bool {
328 CLAUSE_KEYWORDS.contains(&ty) || CLAUSE_STARTER_TOKENS.contains(&ty)
329}
330
331/// The contextual (non-reserved, identifier-spelled) clause keywords:
332/// words that start or continue a clause when they appear as a plain
333/// `Identifier` at clause position, recognized by spelling -- the same
334/// role as [`CLAUSE_KEYWORDS`], but these stay usable as ordinary
335/// variable/label names everywhere else (`MATCH (show) RETURN show`).
336/// Single-sourced here so the clause dispatcher
337/// (`Parser::parse_clause_inner`) and the `SHOW SETTINGS`-style
338/// filter-expression starter (`Parser::peek_starts_show_filter_expr`,
339/// which excludes exactly this set so a following composed clause is not
340/// swallowed as a filter expression) can never drift apart.
341///
342/// `NEXT` (CIP-213) and `CYPHER` (preamble) are members even though they
343/// never start a clause inside `parse_clause_inner` -- they are
344/// recognized at statement-block boundaries and in the pre-parser
345/// preamble respectively. `START`/`STOP`/`NODETACH`/`OFFSET` are
346/// deliberately *not* members: they only act clause-like before a
347/// specific following word (`DATABASE`/`DELETE`/paging), so a bare
348/// `SHOW SETTINGS start` still reads `start` as a name filter.
349pub const CONTEXTUAL_CLAUSE_WORDS: [&str; 9] = [
350 "alter",
351 "cypher",
352 "filter",
353 "finish",
354 "let",
355 "next",
356 "show",
357 "terminate",
358 "use",
359];
360
361/// True if `literal` (any case) spells one of the
362/// [`CONTEXTUAL_CLAUSE_WORDS`].
363pub fn is_contextual_clause_keyword(literal: &str) -> bool {
364 CONTEXTUAL_CLAUSE_WORDS
365 .iter()
366 .any(|w| literal.eq_ignore_ascii_case(w))
367}
368
369/// A lexical token with its type, literal value, and source position.
370///
371/// The literal text borrows from the lexer's input (`'a`): identifiers,
372/// keywords, numbers, parameters and escape-free string literals are slices
373/// of the query text, and punctuation/operators and fixed error texts are
374/// `&'static str` (which coerces to `'a`). Only synthesized text is owned —
375/// escape-decoded string literals and `format!`-built error messages — so
376/// the token stream costs no heap allocation per token beyond the
377/// `Vec<Token>` itself (ADR-0505). Tokens never outlive the input: the only
378/// consumer is the [`crate::parser::Parser`], built and dropped inside the
379/// parse entry points.
380#[derive(Debug, Clone, PartialEq)]
381pub struct Token<'a> {
382 pub ty: TokenType,
383 pub literal: Cow<'a, str>,
384 pub line: usize,
385 pub col: usize,
386 /// Optional symbolic openCypher-TCK error-*detail* name carried by a
387 /// [`TokenType::Error`] token (e.g. `"InvalidNumberLiteral"`,
388 /// `"InvalidUnicodeLiteral"`). The parser propagates it onto the
389 /// [`crate::ParseError`] it raises when it surfaces the error token, so a
390 /// malformed number/unicode literal reports its specific TCK detail rather
391 /// than the generic `UnexpectedSyntax`. `None` for every non-error token.
392 pub detail: Option<&'static str>,
393}
394
395impl<'a> Token<'a> {
396 /// Build a token whose text is borrowed (`&'a str` / `&'static str`) or
397 /// owned (`String`), per the ADR-0505 literal rule — `Into<Cow>` accepts
398 /// all three without a hidden copy.
399 pub fn new(ty: TokenType, literal: impl Into<Cow<'a, str>>, line: usize, col: usize) -> Self {
400 Token {
401 ty,
402 literal: literal.into(),
403 line,
404 col,
405 detail: None,
406 }
407 }
408
409 /// Build a [`TokenType::Error`] token tagged with a TCK error-detail name
410 /// (see [`Token::detail`]).
411 pub fn error_with_detail(
412 literal: impl Into<Cow<'a, str>>,
413 line: usize,
414 col: usize,
415 detail: &'static str,
416 ) -> Self {
417 Token {
418 ty: TokenType::Error,
419 literal: literal.into(),
420 line,
421 col,
422 detail: Some(detail),
423 }
424 }
425}
426
427/// The one declaration of the keyword vocabulary [`lookup_keyword`]
428/// accepts: each `spelling => variant` pair (aliases included) generates
429/// the stack-buffer match cascade, the derived [`LONGEST_KEYWORD_BYTES`]
430/// bound and the test's spelling table, so the cascade, the bound and the
431/// test can never drift apart — a spelling added here is classified,
432/// bounded and exercised in the same change, and a spelling forgotten
433/// cannot exist (there is no second list to forget it from).
434macro_rules! declare_keywords {
435 ($( $spelling:literal => $variant:expr ),+ $(,)?) => {
436 /// The longest keyword spelling accepted by [`lookup_keyword`] in
437 /// bytes (`CONSTRAINT`, `DESCENDING`) — derived from the one
438 /// vocabulary declaration below, never re-spelled at the
439 /// short-circuit or the stack buffer size, so no arm can fall
440 /// outside the bound.
441 const LONGEST_KEYWORD_BYTES: usize = {
442 let lengths = [$($spelling.len()),+];
443 let mut longest = 0;
444 let mut i = 0;
445 while i < lengths.len() {
446 if lengths[i] > longest {
447 longest = lengths[i];
448 }
449 i += 1;
450 }
451 longest
452 };
453
454 /// Look up a keyword from an identifier string (case-insensitive
455 /// uppercase).
456 ///
457 /// No heap copy (ADR-0505): identifiers longer than
458 /// [`LONGEST_KEYWORD_BYTES`] short-circuit to `Identifier`; shorter
459 /// ones are ASCII-uppercased into a fixed stack buffer and matched
460 /// against byte literals. Non-ASCII bytes pass through untouched
461 /// (only `make_ascii_uppercase` is applied), so a Unicode
462 /// identifier (ADR-0182) never matches a keyword — the same
463 /// behavior `to_ascii_uppercase` gave.
464 pub fn lookup_keyword(ident: &str) -> TokenType {
465 if ident.len() > LONGEST_KEYWORD_BYTES {
466 return TokenType::Identifier;
467 }
468 let mut buf = [0u8; LONGEST_KEYWORD_BYTES];
469 let bytes = ident.as_bytes();
470 buf[..bytes.len()].copy_from_slice(bytes);
471 buf[..bytes.len()].make_ascii_uppercase();
472 match &buf[..bytes.len()] {
473 $( $spelling => $variant, )+
474 _ => TokenType::Identifier,
475 }
476 }
477
478 /// Every `(spelling, variant)` pair of the vocabulary, aliases
479 /// included — generated from the same declaration as the cascade,
480 /// so the test below pins the cascade itself, not a hand-copied
481 /// twin of it.
482 #[cfg(test)]
483 const KEYWORD_SPELLINGS: &[(&[u8], TokenType)] = &[$(($spelling, $variant)),+];
484 };
485}
486
487declare_keywords! {
488 b"MATCH" => TokenType::MATCH,
489 b"OPTIONAL" => TokenType::OPTIONAL,
490 b"CREATE" => TokenType::CREATE,
491 b"INSERT" => TokenType::INSERT,
492 b"MERGE" => TokenType::MERGE,
493 b"DELETE" => TokenType::DELETE,
494 b"DETACH" => TokenType::DETACH,
495 b"SET" => TokenType::SET,
496 b"REMOVE" => TokenType::REMOVE,
497 b"RETURN" => TokenType::RETURN,
498 b"WITH" => TokenType::WITH,
499 b"UNWIND" => TokenType::UNWIND,
500 b"WHERE" => TokenType::WHERE,
501 b"ORDER" => TokenType::ORDER,
502 b"BY" => TokenType::BY,
503 b"ASC" => TokenType::ASC,
504 b"ASCENDING" => TokenType::ASC,
505 b"DESC" => TokenType::DESC,
506 b"DESCENDING" => TokenType::DESC,
507 b"LIMIT" => TokenType::LIMIT,
508 b"SKIP" => TokenType::SKIP,
509 b"AS" => TokenType::AS,
510 b"DISTINCT" => TokenType::DISTINCT,
511 b"AND" => TokenType::AND,
512 b"OR" => TokenType::OR,
513 b"NOT" => TokenType::NOT,
514 b"XOR" => TokenType::XOR,
515 b"IN" => TokenType::IN,
516 b"STARTS" => TokenType::STARTS,
517 b"ENDS" => TokenType::ENDS,
518 b"CONTAINS" => TokenType::CONTAINS,
519 b"IS" => TokenType::IS,
520 b"NULL" => TokenType::NULL,
521 b"TRUE" => TokenType::TRUE,
522 b"FALSE" => TokenType::FALSE,
523 b"CASE" => TokenType::CASE,
524 b"WHEN" => TokenType::WHEN,
525 b"THEN" => TokenType::THEN,
526 b"ELSE" => TokenType::ELSE,
527 b"END" => TokenType::END,
528 b"FOREACH" => TokenType::FOREACH,
529 b"CALL" => TokenType::CALL,
530 b"YIELD" => TokenType::YIELD,
531 b"UNION" => TokenType::UNION,
532 b"ALL" => TokenType::ALL,
533 b"ON" => TokenType::ON,
534 b"INDEX" => TokenType::INDEX,
535 b"CONSTRAINT" => TokenType::CONSTRAINT,
536 b"FOR" => TokenType::FOR,
537 b"REQUIRE" => TokenType::REQUIRE,
538 b"UNIQUE" => TokenType::UNIQUE,
539 b"EXISTS" => TokenType::EXISTS,
540 b"IF" => TokenType::IF,
541 b"DROP" => TokenType::DROP,
542 b"VECTOR" => TokenType::VECTOR,
543 b"POINT" => TokenType::POINT,
544 b"OPTIONS" => TokenType::OPTIONS,
545 b"NODE" => TokenType::NODE,
546 b"REL" => TokenType::REL,
547 b"COUNT" => TokenType::COUNT,
548 b"COLLECT" => TokenType::COLLECT,
549 b"LOAD" => TokenType::LOAD,
550 b"FROM" => TokenType::FROM,
551 b"CSV" => TokenType::CSV,
552 b"REDUCE" => TokenType::REDUCE,
553 b"SPATIAL" => TokenType::SPATIAL,
554 b"EXPLAIN" => TokenType::EXPLAIN,
555 b"PROFILE" => TokenType::PROFILE,
556 b"INFINITY" => TokenType::INFINITY,
557 b"INF" => TokenType::INFINITY,
558 b"NAN" => TokenType::NAN,
559}
560
561/// True when `name` must be backtick-quoted to round-trip as a Cypher name.
562///
563/// The DDL / canonical identifier-quoting policy: a name is bare only when it
564/// has the plain identifier shape
565/// ([`relate_model::identifier::has_plain_identifier_shape`]) **and**
566/// [`lookup_keyword`] does not classify it as a keyword.
567///
568/// Keyword names need quotes even in positions that accept them bare (a
569/// property key after `.`, a label or relationship type per ADR-0169) because
570/// the same spelling is a keyword in the positions that call
571/// [`TokenType::is_ident_like`] — a variable, alias, map key, or the
572/// `ON (n.prop)` property name of `CREATE INDEX` — so the unquoted form
573/// would not parse everywhere. Quoting every keyword keeps one rendered
574/// statement runnable in every position instead of deciding per site.
575pub fn identifier_needs_quotes(name: &str) -> bool {
576 !relate_model::identifier::has_plain_identifier_shape(name) || lookup_keyword(name).is_keyword()
577}
578
579/// Renders `name` as a Cypher identifier for generated Cypher text: bare
580/// when [`identifier_needs_quotes`] is false, otherwise through
581/// [`relate_model::identifier::quote_identifier_always`] (the one
582/// spelling of the quoting escape).
583pub fn quote_identifier(name: &str) -> String {
584 if identifier_needs_quotes(name) {
585 relate_model::identifier::quote_identifier_always(name)
586 } else {
587 name.to_string()
588 }
589}
590
591/// Byte offset of the first byte of each 1-based line; O(source) once.
592/// The one spelling of the line table, shared by the parser's
593/// `token_byte_offset` and the statement normalizer (ADR-0503).
594pub(crate) fn line_starts_of(source: &str) -> Vec<usize> {
595 let mut starts = vec![0];
596 for (i, b) in source.bytes().enumerate() {
597 if b == b'\n' {
598 starts.push(i + 1);
599 }
600 }
601 starts
602}
603
604/// The byte offset of `tok`'s first byte in its source, given that
605/// source's [`line_starts_of`] table and its byte length: the lexer
606/// counts columns in bytes (`Lexer::advance` does `col += 1` per byte),
607/// so `line_start + col - 1` is exactly the token's byte offset — no
608/// per-token source rescan, and no char/byte-column mismatch after
609/// non-ASCII text. `None` when the position does not land inside the
610/// source — the one spelling of that half of the contract, so no caller
611/// re-checks it.
612pub(crate) fn token_start_byte(
613 tok: &Token<'_>,
614 line_starts: &[usize],
615 source_len: usize,
616) -> Option<usize> {
617 let line_idx = tok.line.checked_sub(1)?;
618 let col_bytes = tok.col.checked_sub(1)?;
619 let offset = line_starts.get(line_idx)? + col_bytes;
620 (offset <= source_len).then_some(offset)
621}
622
623#[cfg(test)]
624mod tests {
625 use relate_model::identifier::{quote_identifier_always, quote_identifier_shaped};
626
627 use super::*;
628
629 #[test]
630 fn test_lookup_keyword() {
631 assert_eq!(lookup_keyword("MATCH"), TokenType::MATCH);
632 assert_eq!(lookup_keyword("match"), TokenType::MATCH);
633 assert_eq!(lookup_keyword("Match"), TokenType::MATCH);
634 assert_eq!(lookup_keyword("RETURN"), TokenType::RETURN);
635 assert_eq!(lookup_keyword("foo"), TokenType::Identifier);
636 assert_eq!(lookup_keyword("ASCENDING"), TokenType::ASC);
637 assert_eq!(lookup_keyword("DESCENDING"), TokenType::DESC);
638 }
639
640 /// ADR-0505's stack-buffer lookup: the length short-circuit and the
641 /// untouched non-ASCII bytes keep every classification identical to the
642 /// old `to_ascii_uppercase` cascade.
643 #[test]
644 fn test_lookup_keyword_stack_buffer_paths() {
645 // The longest keywords still classify at exactly the bound.
646 assert_eq!(lookup_keyword("constraint"), TokenType::CONSTRAINT);
647 assert_eq!(lookup_keyword("descending"), TokenType::DESC);
648 // One byte past the bound short-circuits, keyword or not.
649 assert_eq!(lookup_keyword("constraints"), TokenType::Identifier);
650 assert_eq!(lookup_keyword("descendingx"), TokenType::Identifier);
651 // A full-width buffer that is not a keyword.
652 assert_eq!(lookup_keyword("abcdefghij"), TokenType::Identifier);
653 // Non-ASCII bytes are uppercased never, so a Unicode identifier
654 // (ADR-0182) cannot collide with an ASCII keyword spelling.
655 assert_eq!(lookup_keyword("mätch"), TokenType::Identifier);
656 assert_eq!(lookup_keyword("ÖR"), TokenType::Identifier);
657 }
658
659 /// Every keyword spelling of the one vocabulary declaration, aliases
660 /// included: each must classify to its variant and stay
661 /// case-insensitive. The table is generated from the same declaration
662 /// as `lookup_keyword`'s cascade (and `LONGEST_KEYWORD_BYTES` is
663 /// derived from it), so this pins the cascade itself — a spelling added
664 /// to the declaration is exercised here in the same change, and a
665 /// longer keyword raises the derived bound instead of silently
666 /// short-circuiting to `Identifier` in the lexer.
667 #[test]
668 fn test_every_keyword_spelling_classifies_case_insensitively() {
669 for &(spelling, ty) in KEYWORD_SPELLINGS {
670 // The declaration is byte-literal — the cascade's own arm form;
671 // every spelling is ASCII by construction.
672 let spelling = std::str::from_utf8(spelling).expect("keyword spellings are ASCII");
673 assert!(ty.is_keyword(), "{spelling}");
674 assert_eq!(lookup_keyword(spelling), ty, "{spelling}");
675 // The cascade is case-insensitive; the lowercase spelling
676 // classifies identically.
677 assert_eq!(
678 lookup_keyword(&spelling.to_ascii_lowercase()),
679 ty,
680 "{spelling}"
681 );
682 }
683 }
684
685 #[test]
686 fn test_is_ident_like() {
687 assert!(TokenType::Identifier.is_ident_like());
688 // CIP-150 NOTES.md Q2 (`@neo4j-verified`): `RETURN` is a
689 // non-reserved word usable as an ordinary variable/alias name.
690 for ty in [
691 TokenType::RETURN,
692 TokenType::NODE,
693 TokenType::COUNT,
694 TokenType::ALL,
695 TokenType::NOT,
696 TokenType::IN,
697 TokenType::IS,
698 TokenType::NULL,
699 TokenType::TRUE,
700 TokenType::INFINITY,
701 ] {
702 assert!(ty.is_ident_like(), "{ty:?}");
703 }
704 for ty in CLAUSE_KEYWORDS {
705 assert!(ty.is_keyword() && !ty.is_ident_like(), "{ty:?}");
706 }
707 assert!(!TokenType::LeftParen.is_ident_like());
708 }
709
710 #[test]
711 fn test_is_schema_name() {
712 // ADR-0169: every clause keyword is a label name.
713 for ty in CLAUSE_KEYWORDS {
714 assert!(ty.is_schema_name(), "{ty:?}");
715 }
716 assert!(TokenType::Identifier.is_schema_name());
717 assert!(TokenType::RETURN.is_schema_name());
718 for ty in [
719 TokenType::LeftParen,
720 TokenType::Colon,
721 TokenType::Pipe,
722 TokenType::Parameter,
723 TokenType::String,
724 TokenType::Integer,
725 ] {
726 assert!(!ty.is_schema_name(), "{ty:?}");
727 }
728 }
729
730 #[test]
731 fn test_is_keyword() {
732 assert!(TokenType::MATCH.is_keyword());
733 assert!(TokenType::RETURN.is_keyword());
734 assert!(TokenType::PROFILE.is_keyword());
735 assert!(!TokenType::Identifier.is_keyword());
736 assert!(!TokenType::LeftParen.is_keyword());
737 }
738
739 #[test]
740 fn test_identifier_needs_quotes_shape() {
741 for bare in ["a", "_", "_a1", "Person", "ship_date", "x9"] {
742 assert!(
743 !identifier_needs_quotes(bare),
744 "{bare} is a bare identifier"
745 );
746 assert_eq!(quote_identifier(bare), bare);
747 }
748 for quoted in ["", "1abc", "a-b", "a b", "n.prop", "Ünïcode", "a`b"] {
749 assert!(
750 identifier_needs_quotes(quoted),
751 "{quoted:?} must be backtick-quoted"
752 );
753 }
754 assert_eq!(
755 quote_identifier("we`ird"),
756 "`we``ird`",
757 "an embedded backtick is doubled"
758 );
759 }
760
761 /// The display policy is shape-only: a keyword spelling renders bare (the
762 /// CIP-257 display corpus writes `when :: STRING`), while a name that
763 /// fails the plain identifier shape keeps its backticks. The DDL/canonical
764 /// policy stays keyword-aware: `quote_identifier("when")` is `` `when` ``,
765 /// which is what a generated statement needs in variable/alias positions.
766 #[test]
767 fn test_quote_identifier_shaped_is_shape_only() {
768 assert_eq!(quote_identifier_shaped("when"), "when");
769 assert_eq!(quote_identifier_shaped("order"), "order");
770 assert_eq!(quote_identifier_shaped("prop-3"), "`prop-3`");
771 assert_eq!(quote_identifier_shaped("a b"), "`a b`");
772 assert_eq!(quote_identifier_shaped(""), "``");
773 assert_eq!(
774 quote_identifier_shaped("we`ird"),
775 "`we``ird`",
776 "the display quoter reuses the one escape"
777 );
778 assert_eq!(
779 quote_identifier_shaped("prop-3"),
780 quote_identifier_always("prop-3")
781 );
782 assert_eq!(
783 quote_identifier("when"),
784 "`when`",
785 "the DDL/canonical policy stays keyword-aware"
786 );
787 }
788
789 /// The forced-quoted constructor always quotes, and shares the one
790 /// escape rule with [`quote_identifier`]'s quoting arm: an embedded
791 /// backtick is doubled in both spellings.
792 #[test]
793 fn test_quote_identifier_always_forces_the_quoting_arm() {
794 assert_eq!(quote_identifier_always("Person"), "`Person`");
795 assert_eq!(quote_identifier_always("ship_date"), "`ship_date`");
796 assert_eq!(quote_identifier_always("we`ird"), "`we``ird`");
797 assert_eq!(
798 quote_identifier_always("we`ird"),
799 quote_identifier("we`ird")
800 );
801 }
802
803 /// The rendering gate is ASCII-only even though the lexer accepts Unicode
804 /// identifiers (ADR-0182): a non-ASCII name that parses bare still renders
805 /// backticked, keeping generated Cypher ASCII and independent of the
806 /// pinned Unicode table version. The quoted form round-trips exactly.
807 #[test]
808 fn test_identifier_needs_quotes_is_ascii_only() {
809 for name in ["café", "Ünïcode", "名前"] {
810 assert_eq!(one_identifier(name), name, "{name} lexes bare");
811 assert!(identifier_needs_quotes(name), "{name} must render quoted");
812 let rendered = quote_identifier(name);
813 assert_eq!(rendered, format!("`{name}`"));
814 assert_eq!(one_identifier(&rendered), name);
815 }
816 }
817
818 /// The policy is keyword-aware: a keyword spelling is quoted even where
819 /// the grammar would take it bare (a property key after `.`, a label per
820 /// ADR-0169), because `CREATE INDEX ... ON (n.<prop>)` reads the property
821 /// through `is_ident_like()` and rejects a clause keyword. Every quoted
822 /// form lexes back to one `Identifier` token with the original spelling.
823 #[test]
824 fn test_identifier_quoting_is_keyword_aware_and_round_trips() {
825 for name in ["order", "Order", "match", "WHERE", "return", "count"] {
826 let rendered = quote_identifier(name);
827 assert_eq!(rendered, format!("`{name}`"), "keyword names are quoted");
828 assert_eq!(one_identifier(&rendered), name);
829 }
830 for ty in CLAUSE_KEYWORDS {
831 let spelling = format!("{ty:?}");
832 assert!(
833 identifier_needs_quotes(&spelling),
834 "{spelling} is a clause keyword"
835 );
836 assert_eq!(one_identifier("e_identifier(&spelling)), spelling);
837 }
838 }
839
840 /// The single `Identifier` token a rendered name lexes to, with its
841 /// literal as written — the round-trip half of the quoting policy.
842 fn one_identifier(rendered: &str) -> String {
843 let tokens = crate::lexer::tokenize(rendered);
844 let [token] = tokens.as_slice() else {
845 panic!("{rendered} must lex to one identifier token, got {tokens:?}");
846 };
847 assert_eq!(token.ty, TokenType::Identifier, "{rendered}");
848 token.literal.to_string()
849 }
850}