Skip to main content

relate_parser/
token.rs

1use std::borrow::Cow;
2
3/// Token types for the Cypher lexer.
4#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
5pub enum TokenType {
6    // Special tokens
7    EOF,
8    Error,
9    Identifier,
10    Integer,
11    Float,
12    String,
13    /// CIP-261 interpolated string literal `s'...'` / `s"..."`. Payload is
14    /// the body between the quotes with standard escapes resolved; `\{` and
15    /// `\}` stay as two characters so the parser can tell an escaped literal
16    /// brace from an expression delimiter, and raw `{`/`}` are kept for the
17    /// parser to split into text/expression parts.
18    InterpString,
19    /// The Neo4j internal obfuscated literal `******` (exactly six stars),
20    /// lexed only under the `obfuscated-literals` preview gate: a dummy
21    /// `false` for semantic analysis, never meant to execute. Without the
22    /// gate the run lexes as ordinary `*` tokens and the query fails with
23    /// the parser's own syntax error, Neo4j's disabled-form behavior.
24    ObfuscatedLiteral,
25    Parameter, // $param
26
27    // Punctuation
28    LeftParen,    // (
29    RightParen,   // )
30    LeftBracket,  // [
31    RightBracket, // ]
32    LeftBrace,    // {
33    RightBrace,   // }
34    Colon,        // :
35    DoubleColon,  // ::
36    Comma,        // ,
37    Dot,          // .
38    Semicolon,    // ;
39    Pipe,         // |
40    DoublePipe,   // ||
41    Dash,         // -
42    ArrowRight,   // ->
43    ArrowLeft,    // <-
44    FatArrow,     // => (CIP-176/CIP-187 graph type declaration syntax)
45    Star,         // *
46    Plus,         // +
47    Slash,        // /
48    Percent,      // %
49    Caret,        // ^
50    DotDot,       // ..
51    Bang,         // ! (CIP-102 NOT NULL type shorthand; also GQL label negation, CIP-14)
52    Ampersand,    // & (GQL label expression conjunction, CIP-14)
53
54    // Comparison operators
55    Equals,       // =
56    NotEquals,    // <>
57    LessThan,     // <
58    GreaterThan,  // >
59    LessEqual,    // <=
60    GreaterEqual, // >=
61    RegexMatch,   // =~
62
63    // Keywords
64    MATCH,
65    OPTIONAL,
66    CREATE,
67    INSERT,
68    MERGE,
69    DELETE,
70    DETACH,
71    SET,
72    REMOVE,
73    RETURN,
74    WITH,
75    UNWIND,
76    WHERE,
77    ORDER,
78    BY,
79    ASC,
80    DESC,
81    LIMIT,
82    SKIP,
83    AS,
84    DISTINCT,
85    AND,
86    OR,
87    NOT,
88    XOR,
89    IN,
90    STARTS,
91    ENDS,
92    CONTAINS,
93    IS,
94    NULL,
95    TRUE,
96    FALSE,
97    CASE,
98    WHEN,
99    THEN,
100    ELSE,
101    END,
102    FOREACH,
103    CALL,
104    YIELD,
105    UNION,
106    ALL,
107    ON,
108    INDEX,
109    CONSTRAINT,
110    FOR,
111    REQUIRE,
112    UNIQUE,
113    EXISTS,
114    IF,
115    DROP,
116    VECTOR,
117    POINT,
118    OPTIONS,
119    NODE,
120    REL,
121    COUNT,
122    COLLECT,
123    LOAD,
124    FROM,
125    CSV,
126    REDUCE,
127    SPATIAL,
128    EXPLAIN,
129    PROFILE,
130    INFINITY, // Infinity/Inf literal (CIP-55)
131    NAN,      // NaN literal (CIP-55)
132}
133
134impl TokenType {
135    /// Returns true if this token type is a keyword (MATCH..PROFILE range).
136    pub fn is_keyword(self) -> bool {
137        matches!(
138            self,
139            TokenType::MATCH
140                | TokenType::OPTIONAL
141                | TokenType::CREATE
142                | TokenType::INSERT
143                | TokenType::MERGE
144                | TokenType::DELETE
145                | TokenType::DETACH
146                | TokenType::SET
147                | TokenType::REMOVE
148                | TokenType::RETURN
149                | TokenType::WITH
150                | TokenType::UNWIND
151                | TokenType::WHERE
152                | TokenType::ORDER
153                | TokenType::BY
154                | TokenType::ASC
155                | TokenType::DESC
156                | TokenType::LIMIT
157                | TokenType::SKIP
158                | TokenType::AS
159                | TokenType::DISTINCT
160                | TokenType::AND
161                | TokenType::OR
162                | TokenType::NOT
163                | TokenType::XOR
164                | TokenType::IN
165                | TokenType::STARTS
166                | TokenType::ENDS
167                | TokenType::CONTAINS
168                | TokenType::IS
169                | TokenType::NULL
170                | TokenType::TRUE
171                | TokenType::FALSE
172                | TokenType::CASE
173                | TokenType::WHEN
174                | TokenType::THEN
175                | TokenType::ELSE
176                | TokenType::END
177                | TokenType::FOREACH
178                | TokenType::CALL
179                | TokenType::YIELD
180                | TokenType::UNION
181                | TokenType::ALL
182                | TokenType::ON
183                | TokenType::INDEX
184                | TokenType::CONSTRAINT
185                | TokenType::FOR
186                | TokenType::REQUIRE
187                | TokenType::UNIQUE
188                | TokenType::EXISTS
189                | TokenType::IF
190                | TokenType::DROP
191                | TokenType::VECTOR
192                | TokenType::POINT
193                | TokenType::OPTIONS
194                | TokenType::NODE
195                | TokenType::REL
196                | TokenType::COUNT
197                | TokenType::COLLECT
198                | TokenType::LOAD
199                | TokenType::FROM
200                | TokenType::CSV
201                | TokenType::REDUCE
202                | TokenType::SPATIAL
203                | TokenType::EXPLAIN
204                | TokenType::PROFILE
205                | TokenType::INFINITY
206                | TokenType::NAN
207        )
208    }
209
210    /// Returns true if the token can be used as an identifier (variable,
211    /// alias, map key, ...): `Identifier` or any keyword except the
212    /// [`CLAUSE_KEYWORDS`].
213    pub fn is_ident_like(self) -> bool {
214        self.is_schema_name() && !CLAUSE_KEYWORDS.contains(&self)
215    }
216
217    /// Returns true if the token is `Identifier` or any keyword: openCypher's
218    /// `SchemaName ::= SymbolicName | ReservedWord`, for positions where the
219    /// grammar already requires a name, so no keyword there starts a clause
220    /// -- property keys after `.`, the LET binding variable, and label and
221    /// relationship-type names. ADR-0169: a strict superset of
222    /// `is_ident_like()`, so `(o:Order)` and `-[:ORDER]->` parse unquoted
223    /// with the name kept as written.
224    ///
225    /// This token class is unrelated to, and deliberately wider than, the
226    /// narrower thing this crate also calls a schema name: the
227    /// index/constraint name `ast::SchemaName` that
228    /// `parse_optional_schema_name` / `parse_required_schema_name` read
229    /// (`Identifier`, `INDEX`, `CONSTRAINT` or a `$param` only).
230    pub fn is_schema_name(self) -> bool {
231        self == TokenType::Identifier || self.is_keyword()
232    }
233}
234
235/// The clause keywords: keyword tokens that start or continue a clause
236/// (`MATCH`, `ORDER BY`, `AS`, `DISTINCT`, ...). `is_ident_like()` rejects
237/// them as variables, aliases and map keys, so `(WHERE ...)` does not read
238/// `WHERE` as the element variable. ADR-0169: they are still label and
239/// relationship-type names (`is_schema_name()`).
240///
241/// `RETURN` is not one of them. Verified
242/// against live Neo4j 2026.05 (both CYPHER 5 and CYPHER 25): `RETURN` is a
243/// non-reserved word usable as an ordinary variable/alias name (e.g.
244/// `WITH 1 AS RETURN RETURN RETURN AS r`) -- `is_ident_like()` is only
245/// checked in positions the grammar already knows unambiguously expect an
246/// identifier (after AS, as a pattern/FOREACH variable, ...), so it never
247/// shadows RETURN's role starting a new clause.
248pub const CLAUSE_KEYWORDS: [TokenType; 22] = [
249    TokenType::MATCH,
250    TokenType::OPTIONAL,
251    TokenType::CREATE,
252    TokenType::INSERT,
253    TokenType::MERGE,
254    TokenType::DELETE,
255    TokenType::DETACH,
256    TokenType::SET,
257    TokenType::REMOVE,
258    TokenType::WITH,
259    TokenType::UNWIND,
260    TokenType::WHERE,
261    TokenType::ORDER,
262    TokenType::BY,
263    TokenType::LIMIT,
264    TokenType::SKIP,
265    TokenType::AS,
266    TokenType::DISTINCT,
267    TokenType::FOREACH,
268    TokenType::UNION,
269    TokenType::REQUIRE,
270    TokenType::DROP,
271];
272
273/// The contextual `FIELDTERMINATOR` word of LOAD CSV: an identifier-spelled
274/// keyword the clause parser discriminates on by spelling. The one
275/// constant, shared with the statement normalizer's grammar guard (a
276/// `String` after this word is a literal-only position there) — the
277/// spelling is a centralized fact, never re-derived at a second site.
278pub(crate) const FIELDTERMINATOR_WORD: &str = "FIELDTERMINATOR";
279
280/// Whether an identifier's literal spells the LOAD CSV
281/// [`FIELDTERMINATOR_WORD`] (case-insensitively, as the clause grammar
282/// reads it).
283pub(crate) fn is_field_terminator_word(literal: &str) -> bool {
284    literal.eq_ignore_ascii_case(FIELDTERMINATOR_WORD)
285}
286
287/// The reserved-keyword clause-starter tokens: the keyword tokens that
288/// start a new clause at clause position (`MATCH`, `CREATE`, `UNWIND`,
289/// `FOR` (CIP-250), ...). The grammar authority is the clause dispatch
290/// (`Parser::parse_clause_inner`); this table is the derived vocabulary
291/// every *lookahead* over the same fact consults instead of re-spelling a
292/// second hand-maintained list -- through [`is_clause_boundary`], by
293/// `Parser::group_by_ends_clause` (whose round-5 comment records the
294/// LOAD/DROP drift that motivated it) and the simple-CASE lookahead -- and
295/// the dispatch-coverage pin test. A new clause-start dispatch arm
296/// extends this one table. `OPTIONAL` is a member: it starts a clause
297/// only immediately before `MATCH`/`CALL` (the arm that checks what
298/// follows). The contextual (identifier-spelled) starters are the
299/// separate [`CONTEXTUAL_CLAUSE_WORDS`] below.
300pub const CLAUSE_STARTER_TOKENS: &[TokenType] = &[
301    TokenType::MATCH,
302    TokenType::OPTIONAL,
303    TokenType::CREATE,
304    TokenType::INSERT,
305    TokenType::MERGE,
306    TokenType::DELETE,
307    TokenType::DETACH,
308    TokenType::SET,
309    TokenType::REMOVE,
310    TokenType::UNWIND,
311    // `FOR <variable> IN <expression>`.
312    TokenType::FOR,
313    TokenType::FOREACH,
314    TokenType::CALL,
315    TokenType::UNION,
316    TokenType::LOAD,
317    TokenType::DROP,
318];
319
320/// The one derived "this token ends the enclosing clause" predicate: a
321/// reserved clause keyword ([`CLAUSE_KEYWORDS`]) or a reserved clause
322/// starter ([`CLAUSE_STARTER_TOKENS`]). Every lookahead that must stop at a
323/// clause boundary consults this instead of re-spelling the vocabulary —
324/// `Parser::group_by_ends_clause` and the simple-CASE lookahead
325/// (`ends_case_test_scan`), each adding only its own statement/item
326/// terminals.
327pub(crate) fn is_clause_boundary(ty: TokenType) -> bool {
328    CLAUSE_KEYWORDS.contains(&ty) || CLAUSE_STARTER_TOKENS.contains(&ty)
329}
330
331/// The contextual (non-reserved, identifier-spelled) clause keywords:
332/// words that start or continue a clause when they appear as a plain
333/// `Identifier` at clause position, recognized by spelling -- the same
334/// role as [`CLAUSE_KEYWORDS`], but these stay usable as ordinary
335/// variable/label names everywhere else (`MATCH (show) RETURN show`).
336/// Single-sourced here so the clause dispatcher
337/// (`Parser::parse_clause_inner`) and the `SHOW SETTINGS`-style
338/// filter-expression starter (`Parser::peek_starts_show_filter_expr`,
339/// which excludes exactly this set so a following composed clause is not
340/// swallowed as a filter expression) can never drift apart.
341///
342/// `NEXT` (CIP-213) and `CYPHER` (preamble) are members even though they
343/// never start a clause inside `parse_clause_inner` -- they are
344/// recognized at statement-block boundaries and in the pre-parser
345/// preamble respectively. `START`/`STOP`/`NODETACH`/`OFFSET` are
346/// deliberately *not* members: they only act clause-like before a
347/// specific following word (`DATABASE`/`DELETE`/paging), so a bare
348/// `SHOW SETTINGS start` still reads `start` as a name filter.
349pub const CONTEXTUAL_CLAUSE_WORDS: [&str; 9] = [
350    "alter",
351    "cypher",
352    "filter",
353    "finish",
354    "let",
355    "next",
356    "show",
357    "terminate",
358    "use",
359];
360
361/// True if `literal` (any case) spells one of the
362/// [`CONTEXTUAL_CLAUSE_WORDS`].
363pub fn is_contextual_clause_keyword(literal: &str) -> bool {
364    CONTEXTUAL_CLAUSE_WORDS
365        .iter()
366        .any(|w| literal.eq_ignore_ascii_case(w))
367}
368
369/// A lexical token with its type, literal value, and source position.
370///
371/// The literal text borrows from the lexer's input (`'a`): identifiers,
372/// keywords, numbers, parameters and escape-free string literals are slices
373/// of the query text, and punctuation/operators and fixed error texts are
374/// `&'static str` (which coerces to `'a`). Only synthesized text is owned —
375/// escape-decoded string literals and `format!`-built error messages — so
376/// the token stream costs no heap allocation per token beyond the
377/// `Vec<Token>` itself (ADR-0505). Tokens never outlive the input: the only
378/// consumer is the [`crate::parser::Parser`], built and dropped inside the
379/// parse entry points.
380#[derive(Debug, Clone, PartialEq)]
381pub struct Token<'a> {
382    pub ty: TokenType,
383    pub literal: Cow<'a, str>,
384    pub line: usize,
385    pub col: usize,
386    /// Optional symbolic openCypher-TCK error-*detail* name carried by a
387    /// [`TokenType::Error`] token (e.g. `"InvalidNumberLiteral"`,
388    /// `"InvalidUnicodeLiteral"`). The parser propagates it onto the
389    /// [`crate::ParseError`] it raises when it surfaces the error token, so a
390    /// malformed number/unicode literal reports its specific TCK detail rather
391    /// than the generic `UnexpectedSyntax`. `None` for every non-error token.
392    pub detail: Option<&'static str>,
393}
394
395impl<'a> Token<'a> {
396    /// Build a token whose text is borrowed (`&'a str` / `&'static str`) or
397    /// owned (`String`), per the ADR-0505 literal rule — `Into<Cow>` accepts
398    /// all three without a hidden copy.
399    pub fn new(ty: TokenType, literal: impl Into<Cow<'a, str>>, line: usize, col: usize) -> Self {
400        Token {
401            ty,
402            literal: literal.into(),
403            line,
404            col,
405            detail: None,
406        }
407    }
408
409    /// Build a [`TokenType::Error`] token tagged with a TCK error-detail name
410    /// (see [`Token::detail`]).
411    pub fn error_with_detail(
412        literal: impl Into<Cow<'a, str>>,
413        line: usize,
414        col: usize,
415        detail: &'static str,
416    ) -> Self {
417        Token {
418            ty: TokenType::Error,
419            literal: literal.into(),
420            line,
421            col,
422            detail: Some(detail),
423        }
424    }
425}
426
427/// The one declaration of the keyword vocabulary [`lookup_keyword`]
428/// accepts: each `spelling => variant` pair (aliases included) generates
429/// the stack-buffer match cascade, the derived [`LONGEST_KEYWORD_BYTES`]
430/// bound and the test's spelling table, so the cascade, the bound and the
431/// test can never drift apart — a spelling added here is classified,
432/// bounded and exercised in the same change, and a spelling forgotten
433/// cannot exist (there is no second list to forget it from).
434macro_rules! declare_keywords {
435    ($( $spelling:literal => $variant:expr ),+ $(,)?) => {
436        /// The longest keyword spelling accepted by [`lookup_keyword`] in
437        /// bytes (`CONSTRAINT`, `DESCENDING`) — derived from the one
438        /// vocabulary declaration below, never re-spelled at the
439        /// short-circuit or the stack buffer size, so no arm can fall
440        /// outside the bound.
441        const LONGEST_KEYWORD_BYTES: usize = {
442            let lengths = [$($spelling.len()),+];
443            let mut longest = 0;
444            let mut i = 0;
445            while i < lengths.len() {
446                if lengths[i] > longest {
447                    longest = lengths[i];
448                }
449                i += 1;
450            }
451            longest
452        };
453
454        /// Look up a keyword from an identifier string (case-insensitive
455        /// uppercase).
456        ///
457        /// No heap copy (ADR-0505): identifiers longer than
458        /// [`LONGEST_KEYWORD_BYTES`] short-circuit to `Identifier`; shorter
459        /// ones are ASCII-uppercased into a fixed stack buffer and matched
460        /// against byte literals. Non-ASCII bytes pass through untouched
461        /// (only `make_ascii_uppercase` is applied), so a Unicode
462        /// identifier (ADR-0182) never matches a keyword — the same
463        /// behavior `to_ascii_uppercase` gave.
464        pub fn lookup_keyword(ident: &str) -> TokenType {
465            if ident.len() > LONGEST_KEYWORD_BYTES {
466                return TokenType::Identifier;
467            }
468            let mut buf = [0u8; LONGEST_KEYWORD_BYTES];
469            let bytes = ident.as_bytes();
470            buf[..bytes.len()].copy_from_slice(bytes);
471            buf[..bytes.len()].make_ascii_uppercase();
472            match &buf[..bytes.len()] {
473                $( $spelling => $variant, )+
474                _ => TokenType::Identifier,
475            }
476        }
477
478        /// Every `(spelling, variant)` pair of the vocabulary, aliases
479        /// included — generated from the same declaration as the cascade,
480        /// so the test below pins the cascade itself, not a hand-copied
481        /// twin of it.
482        #[cfg(test)]
483        const KEYWORD_SPELLINGS: &[(&[u8], TokenType)] = &[$(($spelling, $variant)),+];
484    };
485}
486
487declare_keywords! {
488    b"MATCH" => TokenType::MATCH,
489    b"OPTIONAL" => TokenType::OPTIONAL,
490    b"CREATE" => TokenType::CREATE,
491    b"INSERT" => TokenType::INSERT,
492    b"MERGE" => TokenType::MERGE,
493    b"DELETE" => TokenType::DELETE,
494    b"DETACH" => TokenType::DETACH,
495    b"SET" => TokenType::SET,
496    b"REMOVE" => TokenType::REMOVE,
497    b"RETURN" => TokenType::RETURN,
498    b"WITH" => TokenType::WITH,
499    b"UNWIND" => TokenType::UNWIND,
500    b"WHERE" => TokenType::WHERE,
501    b"ORDER" => TokenType::ORDER,
502    b"BY" => TokenType::BY,
503    b"ASC" => TokenType::ASC,
504    b"ASCENDING" => TokenType::ASC,
505    b"DESC" => TokenType::DESC,
506    b"DESCENDING" => TokenType::DESC,
507    b"LIMIT" => TokenType::LIMIT,
508    b"SKIP" => TokenType::SKIP,
509    b"AS" => TokenType::AS,
510    b"DISTINCT" => TokenType::DISTINCT,
511    b"AND" => TokenType::AND,
512    b"OR" => TokenType::OR,
513    b"NOT" => TokenType::NOT,
514    b"XOR" => TokenType::XOR,
515    b"IN" => TokenType::IN,
516    b"STARTS" => TokenType::STARTS,
517    b"ENDS" => TokenType::ENDS,
518    b"CONTAINS" => TokenType::CONTAINS,
519    b"IS" => TokenType::IS,
520    b"NULL" => TokenType::NULL,
521    b"TRUE" => TokenType::TRUE,
522    b"FALSE" => TokenType::FALSE,
523    b"CASE" => TokenType::CASE,
524    b"WHEN" => TokenType::WHEN,
525    b"THEN" => TokenType::THEN,
526    b"ELSE" => TokenType::ELSE,
527    b"END" => TokenType::END,
528    b"FOREACH" => TokenType::FOREACH,
529    b"CALL" => TokenType::CALL,
530    b"YIELD" => TokenType::YIELD,
531    b"UNION" => TokenType::UNION,
532    b"ALL" => TokenType::ALL,
533    b"ON" => TokenType::ON,
534    b"INDEX" => TokenType::INDEX,
535    b"CONSTRAINT" => TokenType::CONSTRAINT,
536    b"FOR" => TokenType::FOR,
537    b"REQUIRE" => TokenType::REQUIRE,
538    b"UNIQUE" => TokenType::UNIQUE,
539    b"EXISTS" => TokenType::EXISTS,
540    b"IF" => TokenType::IF,
541    b"DROP" => TokenType::DROP,
542    b"VECTOR" => TokenType::VECTOR,
543    b"POINT" => TokenType::POINT,
544    b"OPTIONS" => TokenType::OPTIONS,
545    b"NODE" => TokenType::NODE,
546    b"REL" => TokenType::REL,
547    b"COUNT" => TokenType::COUNT,
548    b"COLLECT" => TokenType::COLLECT,
549    b"LOAD" => TokenType::LOAD,
550    b"FROM" => TokenType::FROM,
551    b"CSV" => TokenType::CSV,
552    b"REDUCE" => TokenType::REDUCE,
553    b"SPATIAL" => TokenType::SPATIAL,
554    b"EXPLAIN" => TokenType::EXPLAIN,
555    b"PROFILE" => TokenType::PROFILE,
556    b"INFINITY" => TokenType::INFINITY,
557    b"INF" => TokenType::INFINITY,
558    b"NAN" => TokenType::NAN,
559}
560
561/// True when `name` must be backtick-quoted to round-trip as a Cypher name.
562///
563/// The DDL / canonical identifier-quoting policy: a name is bare only when it
564/// has the plain identifier shape
565/// ([`relate_model::identifier::has_plain_identifier_shape`]) **and**
566/// [`lookup_keyword`] does not classify it as a keyword.
567///
568/// Keyword names need quotes even in positions that accept them bare (a
569/// property key after `.`, a label or relationship type per ADR-0169) because
570/// the same spelling is a keyword in the positions that call
571/// [`TokenType::is_ident_like`] — a variable, alias, map key, or the
572/// `ON (n.prop)` property name of `CREATE INDEX` — so the unquoted form
573/// would not parse everywhere. Quoting every keyword keeps one rendered
574/// statement runnable in every position instead of deciding per site.
575pub fn identifier_needs_quotes(name: &str) -> bool {
576    !relate_model::identifier::has_plain_identifier_shape(name) || lookup_keyword(name).is_keyword()
577}
578
579/// Renders `name` as a Cypher identifier for generated Cypher text: bare
580/// when [`identifier_needs_quotes`] is false, otherwise through
581/// [`relate_model::identifier::quote_identifier_always`] (the one
582/// spelling of the quoting escape).
583pub fn quote_identifier(name: &str) -> String {
584    if identifier_needs_quotes(name) {
585        relate_model::identifier::quote_identifier_always(name)
586    } else {
587        name.to_string()
588    }
589}
590
591/// Byte offset of the first byte of each 1-based line; O(source) once.
592/// The one spelling of the line table, shared by the parser's
593/// `token_byte_offset` and the statement normalizer (ADR-0503).
594pub(crate) fn line_starts_of(source: &str) -> Vec<usize> {
595    let mut starts = vec![0];
596    for (i, b) in source.bytes().enumerate() {
597        if b == b'\n' {
598            starts.push(i + 1);
599        }
600    }
601    starts
602}
603
604/// The byte offset of `tok`'s first byte in its source, given that
605/// source's [`line_starts_of`] table and its byte length: the lexer
606/// counts columns in bytes (`Lexer::advance` does `col += 1` per byte),
607/// so `line_start + col - 1` is exactly the token's byte offset — no
608/// per-token source rescan, and no char/byte-column mismatch after
609/// non-ASCII text. `None` when the position does not land inside the
610/// source — the one spelling of that half of the contract, so no caller
611/// re-checks it.
612pub(crate) fn token_start_byte(
613    tok: &Token<'_>,
614    line_starts: &[usize],
615    source_len: usize,
616) -> Option<usize> {
617    let line_idx = tok.line.checked_sub(1)?;
618    let col_bytes = tok.col.checked_sub(1)?;
619    let offset = line_starts.get(line_idx)? + col_bytes;
620    (offset <= source_len).then_some(offset)
621}
622
623#[cfg(test)]
624mod tests {
625    use relate_model::identifier::{quote_identifier_always, quote_identifier_shaped};
626
627    use super::*;
628
629    #[test]
630    fn test_lookup_keyword() {
631        assert_eq!(lookup_keyword("MATCH"), TokenType::MATCH);
632        assert_eq!(lookup_keyword("match"), TokenType::MATCH);
633        assert_eq!(lookup_keyword("Match"), TokenType::MATCH);
634        assert_eq!(lookup_keyword("RETURN"), TokenType::RETURN);
635        assert_eq!(lookup_keyword("foo"), TokenType::Identifier);
636        assert_eq!(lookup_keyword("ASCENDING"), TokenType::ASC);
637        assert_eq!(lookup_keyword("DESCENDING"), TokenType::DESC);
638    }
639
640    /// ADR-0505's stack-buffer lookup: the length short-circuit and the
641    /// untouched non-ASCII bytes keep every classification identical to the
642    /// old `to_ascii_uppercase` cascade.
643    #[test]
644    fn test_lookup_keyword_stack_buffer_paths() {
645        // The longest keywords still classify at exactly the bound.
646        assert_eq!(lookup_keyword("constraint"), TokenType::CONSTRAINT);
647        assert_eq!(lookup_keyword("descending"), TokenType::DESC);
648        // One byte past the bound short-circuits, keyword or not.
649        assert_eq!(lookup_keyword("constraints"), TokenType::Identifier);
650        assert_eq!(lookup_keyword("descendingx"), TokenType::Identifier);
651        // A full-width buffer that is not a keyword.
652        assert_eq!(lookup_keyword("abcdefghij"), TokenType::Identifier);
653        // Non-ASCII bytes are uppercased never, so a Unicode identifier
654        // (ADR-0182) cannot collide with an ASCII keyword spelling.
655        assert_eq!(lookup_keyword("mätch"), TokenType::Identifier);
656        assert_eq!(lookup_keyword("ÖR"), TokenType::Identifier);
657    }
658
659    /// Every keyword spelling of the one vocabulary declaration, aliases
660    /// included: each must classify to its variant and stay
661    /// case-insensitive. The table is generated from the same declaration
662    /// as `lookup_keyword`'s cascade (and `LONGEST_KEYWORD_BYTES` is
663    /// derived from it), so this pins the cascade itself — a spelling added
664    /// to the declaration is exercised here in the same change, and a
665    /// longer keyword raises the derived bound instead of silently
666    /// short-circuiting to `Identifier` in the lexer.
667    #[test]
668    fn test_every_keyword_spelling_classifies_case_insensitively() {
669        for &(spelling, ty) in KEYWORD_SPELLINGS {
670            // The declaration is byte-literal — the cascade's own arm form;
671            // every spelling is ASCII by construction.
672            let spelling = std::str::from_utf8(spelling).expect("keyword spellings are ASCII");
673            assert!(ty.is_keyword(), "{spelling}");
674            assert_eq!(lookup_keyword(spelling), ty, "{spelling}");
675            // The cascade is case-insensitive; the lowercase spelling
676            // classifies identically.
677            assert_eq!(
678                lookup_keyword(&spelling.to_ascii_lowercase()),
679                ty,
680                "{spelling}"
681            );
682        }
683    }
684
685    #[test]
686    fn test_is_ident_like() {
687        assert!(TokenType::Identifier.is_ident_like());
688        // `RETURN` is a
689        // non-reserved word usable as an ordinary variable/alias name.
690        for ty in [
691            TokenType::RETURN,
692            TokenType::NODE,
693            TokenType::COUNT,
694            TokenType::ALL,
695            TokenType::NOT,
696            TokenType::IN,
697            TokenType::IS,
698            TokenType::NULL,
699            TokenType::TRUE,
700            TokenType::INFINITY,
701        ] {
702            assert!(ty.is_ident_like(), "{ty:?}");
703        }
704        for ty in CLAUSE_KEYWORDS {
705            assert!(ty.is_keyword() && !ty.is_ident_like(), "{ty:?}");
706        }
707        assert!(!TokenType::LeftParen.is_ident_like());
708    }
709
710    #[test]
711    fn test_is_schema_name() {
712        // ADR-0169: every clause keyword is a label name.
713        for ty in CLAUSE_KEYWORDS {
714            assert!(ty.is_schema_name(), "{ty:?}");
715        }
716        assert!(TokenType::Identifier.is_schema_name());
717        assert!(TokenType::RETURN.is_schema_name());
718        for ty in [
719            TokenType::LeftParen,
720            TokenType::Colon,
721            TokenType::Pipe,
722            TokenType::Parameter,
723            TokenType::String,
724            TokenType::Integer,
725        ] {
726            assert!(!ty.is_schema_name(), "{ty:?}");
727        }
728    }
729
730    #[test]
731    fn test_is_keyword() {
732        assert!(TokenType::MATCH.is_keyword());
733        assert!(TokenType::RETURN.is_keyword());
734        assert!(TokenType::PROFILE.is_keyword());
735        assert!(!TokenType::Identifier.is_keyword());
736        assert!(!TokenType::LeftParen.is_keyword());
737    }
738
739    #[test]
740    fn test_identifier_needs_quotes_shape() {
741        for bare in ["a", "_", "_a1", "Person", "ship_date", "x9"] {
742            assert!(
743                !identifier_needs_quotes(bare),
744                "{bare} is a bare identifier"
745            );
746            assert_eq!(quote_identifier(bare), bare);
747        }
748        for quoted in ["", "1abc", "a-b", "a b", "n.prop", "Ünïcode", "a`b"] {
749            assert!(
750                identifier_needs_quotes(quoted),
751                "{quoted:?} must be backtick-quoted"
752            );
753        }
754        assert_eq!(
755            quote_identifier("we`ird"),
756            "`we``ird`",
757            "an embedded backtick is doubled"
758        );
759    }
760
761    /// The display policy is shape-only: a keyword spelling renders bare (the
762    /// CIP-257 display corpus writes `when :: STRING`), while a name that
763    /// fails the plain identifier shape keeps its backticks. The DDL/canonical
764    /// policy stays keyword-aware: `quote_identifier("when")` is `` `when` ``,
765    /// which is what a generated statement needs in variable/alias positions.
766    #[test]
767    fn test_quote_identifier_shaped_is_shape_only() {
768        assert_eq!(quote_identifier_shaped("when"), "when");
769        assert_eq!(quote_identifier_shaped("order"), "order");
770        assert_eq!(quote_identifier_shaped("prop-3"), "`prop-3`");
771        assert_eq!(quote_identifier_shaped("a b"), "`a b`");
772        assert_eq!(quote_identifier_shaped(""), "``");
773        assert_eq!(
774            quote_identifier_shaped("we`ird"),
775            "`we``ird`",
776            "the display quoter reuses the one escape"
777        );
778        assert_eq!(
779            quote_identifier_shaped("prop-3"),
780            quote_identifier_always("prop-3")
781        );
782        assert_eq!(
783            quote_identifier("when"),
784            "`when`",
785            "the DDL/canonical policy stays keyword-aware"
786        );
787    }
788
789    /// The forced-quoted constructor always quotes, and shares the one
790    /// escape rule with [`quote_identifier`]'s quoting arm: an embedded
791    /// backtick is doubled in both spellings.
792    #[test]
793    fn test_quote_identifier_always_forces_the_quoting_arm() {
794        assert_eq!(quote_identifier_always("Person"), "`Person`");
795        assert_eq!(quote_identifier_always("ship_date"), "`ship_date`");
796        assert_eq!(quote_identifier_always("we`ird"), "`we``ird`");
797        assert_eq!(
798            quote_identifier_always("we`ird"),
799            quote_identifier("we`ird")
800        );
801    }
802
803    /// The rendering gate is ASCII-only even though the lexer accepts Unicode
804    /// identifiers (ADR-0182): a non-ASCII name that parses bare still renders
805    /// backticked, keeping generated Cypher ASCII and independent of the
806    /// pinned Unicode table version. The quoted form round-trips exactly.
807    #[test]
808    fn test_identifier_needs_quotes_is_ascii_only() {
809        for name in ["café", "Ünïcode", "名前"] {
810            assert_eq!(one_identifier(name), name, "{name} lexes bare");
811            assert!(identifier_needs_quotes(name), "{name} must render quoted");
812            let rendered = quote_identifier(name);
813            assert_eq!(rendered, format!("`{name}`"));
814            assert_eq!(one_identifier(&rendered), name);
815        }
816    }
817
818    /// The policy is keyword-aware: a keyword spelling is quoted even where
819    /// the grammar would take it bare (a property key after `.`, a label per
820    /// ADR-0169), because `CREATE INDEX ... ON (n.<prop>)` reads the property
821    /// through `is_ident_like()` and rejects a clause keyword. Every quoted
822    /// form lexes back to one `Identifier` token with the original spelling.
823    #[test]
824    fn test_identifier_quoting_is_keyword_aware_and_round_trips() {
825        for name in ["order", "Order", "match", "WHERE", "return", "count"] {
826            let rendered = quote_identifier(name);
827            assert_eq!(rendered, format!("`{name}`"), "keyword names are quoted");
828            assert_eq!(one_identifier(&rendered), name);
829        }
830        for ty in CLAUSE_KEYWORDS {
831            let spelling = format!("{ty:?}");
832            assert!(
833                identifier_needs_quotes(&spelling),
834                "{spelling} is a clause keyword"
835            );
836            assert_eq!(one_identifier(&quote_identifier(&spelling)), spelling);
837        }
838    }
839
840    /// The single `Identifier` token a rendered name lexes to, with its
841    /// literal as written — the round-trip half of the quoting policy.
842    fn one_identifier(rendered: &str) -> String {
843        let tokens = crate::lexer::tokenize(rendered);
844        let [token] = tokens.as_slice() else {
845            panic!("{rendered} must lex to one identifier token, got {tokens:?}");
846        };
847        assert_eq!(token.ty, TokenType::Identifier, "{rendered}");
848        token.literal.to_string()
849    }
850}