open_cypher/
token.rs

1//! Backend-neutral lexical tokens.
2//!
3//! Tokens contain only their kind and source span. The original spelling is
4//! intentionally not copied: use [`Token::text`] with the source query when a
5//! lossless representation is needed.
6
7use std::fmt;
8
9use crate::span::Span;
10
11/// A token produced by the lexer.
12#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
13#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
14pub struct Token {
15    /// The lexical class of the token.
16    pub kind: TokenKind,
17    /// The token's half-open UTF-8 byte range.
18    pub span: Span,
19}
20
21impl Token {
22    /// Creates a token.
23    #[must_use]
24    pub const fn new(kind: TokenKind, span: Span) -> Self {
25        Self { kind, span }
26    }
27
28    /// Returns the original token spelling, when `span` is valid for `source`.
29    #[must_use]
30    pub fn text(self, source: &str) -> Option<&str> {
31        self.span.text(source)
32    }
33
34    /// Returns `true` for whitespace and comments.
35    #[must_use]
36    pub const fn is_trivia(self) -> bool {
37        self.kind.is_trivia()
38    }
39}
40
41/// A lexical token class understood by the openCypher parser.
42///
43/// Keyword recognition is case-insensitive. The parser reclassifies
44/// non-reserved [`Keyword`] tokens contextually when a name is required.
45#[non_exhaustive]
46#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
47#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
48pub enum TokenKind {
49    /// A case-insensitive language keyword.
50    Keyword(Keyword),
51    /// An unescaped symbolic name.
52    Identifier,
53    /// A backtick-delimited symbolic name.
54    EscapedIdentifier,
55    /// A parameter whose separated name is unescaped, such as `$name`,
56    /// `$123`, or `$1_name`.
57    Parameter,
58    /// A base-ten integer literal.
59    Integer,
60    /// A hexadecimal integer literal.
61    HexInteger,
62    /// An octal integer literal.
63    OctalInteger,
64    /// A decimal floating-point literal, optionally with an exponent.
65    Float,
66    /// A single- or double-quoted string literal.
67    String,
68
69    LeftParen,
70    RightParen,
71    LeftBracket,
72    RightBracket,
73    LeftBrace,
74    RightBrace,
75    Comma,
76    Dot,
77    DotDot,
78    Colon,
79    DoubleColon,
80    Semicolon,
81    Pipe,
82    DoublePipe,
83    Ampersand,
84    Question,
85    /// A bare `$`; the parser combines it with a following backtick-delimited
86    /// identifier to form an escaped parameter name.
87    Dollar,
88
89    Plus,
90    Minus,
91    Star,
92    Slash,
93    Percent,
94    Caret,
95    Bang,
96    Equal,
97    NotEqual,
98    Less,
99    LessEqual,
100    Greater,
101    GreaterEqual,
102    PlusEqual,
103    FatArrow,
104    RegexMatch,
105    /// A contiguous `<-`; spaced or comment-separated forms remain separate
106    /// [`Less`](Self::Less) and [`Minus`](Self::Minus) tokens.
107    LeftArrow,
108    /// A contiguous `->`; spaced or comment-separated forms remain separate
109    /// [`Minus`](Self::Minus) and [`Greater`](Self::Greater) tokens.
110    RightArrow,
111
112    /// One or more Unicode whitespace characters.
113    Whitespace,
114    /// A `//` comment, including its marker but excluding a line terminator.
115    LineComment,
116    /// A `/* ... */` comment, including its delimiters.
117    BlockComment,
118    /// A source region which cannot begin any valid token.
119    Invalid,
120}
121
122impl TokenKind {
123    /// Returns `true` for whitespace and comments.
124    #[must_use]
125    pub const fn is_trivia(self) -> bool {
126        matches!(
127            self,
128            Self::Whitespace | Self::LineComment | Self::BlockComment
129        )
130    }
131
132    /// Returns `true` for literal tokens.
133    #[must_use]
134    pub const fn is_literal(self) -> bool {
135        matches!(
136            self,
137            Self::Integer
138                | Self::HexInteger
139                | Self::OctalInteger
140                | Self::Float
141                | Self::String
142                | Self::Keyword(Keyword::True | Keyword::False | Keyword::Null)
143        )
144    }
145
146    /// Returns a concise, user-facing name suitable for diagnostics.
147    #[must_use]
148    pub const fn display_name(self) -> &'static str {
149        match self {
150            Self::Keyword(keyword) => keyword.as_str(),
151            Self::Identifier => "an identifier",
152            Self::EscapedIdentifier => "an escaped identifier",
153            Self::Parameter => "a parameter",
154            Self::Integer => "an integer",
155            Self::HexInteger => "a hexadecimal integer",
156            Self::OctalInteger => "an octal integer",
157            Self::Float => "a floating-point number",
158            Self::String => "a string",
159            Self::LeftParen => "`(`",
160            Self::RightParen => "`)`",
161            Self::LeftBracket => "`[`",
162            Self::RightBracket => "`]`",
163            Self::LeftBrace => "`{`",
164            Self::RightBrace => "`}`",
165            Self::Comma => "`,`",
166            Self::Dot => "`.`",
167            Self::DotDot => "`..`",
168            Self::Colon => "`:`",
169            Self::DoubleColon => "`::`",
170            Self::Semicolon => "`;`",
171            Self::Pipe => "`|`",
172            Self::DoublePipe => "`||`",
173            Self::Ampersand => "`&`",
174            Self::Question => "`?`",
175            Self::Dollar => "`$`",
176            Self::Plus => "`+`",
177            Self::Minus => "`-`",
178            Self::Star => "`*`",
179            Self::Slash => "`/`",
180            Self::Percent => "`%`",
181            Self::Caret => "`^`",
182            Self::Bang => "`!`",
183            Self::Equal => "`=`",
184            Self::NotEqual => "`<>` or `!=`",
185            Self::Less => "`<`",
186            Self::LessEqual => "`<=`",
187            Self::Greater => "`>`",
188            Self::GreaterEqual => "`>=`",
189            Self::PlusEqual => "`+=`",
190            Self::FatArrow => "`=>`",
191            Self::RegexMatch => "`=~`",
192            Self::LeftArrow => "`<-`",
193            Self::RightArrow => "`->`",
194            Self::Whitespace => "whitespace",
195            Self::LineComment => "a line comment",
196            Self::BlockComment => "a block comment",
197            Self::Invalid => "an invalid token",
198        }
199    }
200}
201
202impl fmt::Display for TokenKind {
203    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
204        formatter.write_str(self.display_name())
205    }
206}
207
208/// A word recognized specially by this crate's lexer.
209///
210/// This contains the openCypher 2024.3 keyword spellings plus the documented
211/// path-mode and subquery-expression extensions consumed by the grammar. The
212/// parser decides whether a keyword can act as a symbolic name at a particular
213/// location.
214#[non_exhaustive]
215#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
216#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
217pub enum Keyword {
218    Acyclic,
219    All,
220    AllShortestPaths,
221    And,
222    Any,
223    As,
224    Asc,
225    Ascending,
226    By,
227    Call,
228    Case,
229    Collect,
230    Contains,
231    Count,
232    Create,
233    Delete,
234    Desc,
235    Descending,
236    Detach,
237    Distinct,
238    Else,
239    End,
240    Ends,
241    Exists,
242    False,
243    Group,
244    Groups,
245    In,
246    Inf,
247    Infinity,
248    Is,
249    Limit,
250    Match,
251    Merge,
252    Nan,
253    None,
254    Not,
255    Null,
256    Offset,
257    On,
258    Optional,
259    Or,
260    Order,
261    Path,
262    Paths,
263    Reduce,
264    Remove,
265    Return,
266    Set,
267    Shortest,
268    ShortestPath,
269    Simple,
270    Single,
271    Skip,
272    Starts,
273    Then,
274    Trail,
275    Trim,
276    True,
277    Union,
278    Unwind,
279    Walk,
280    When,
281    Where,
282    With,
283    Xor,
284    Yield,
285}
286
287impl Keyword {
288    /// Returns the canonical uppercase spelling.
289    #[must_use]
290    pub const fn as_str(self) -> &'static str {
291        match self {
292            Self::Acyclic => "ACYCLIC",
293            Self::All => "ALL",
294            Self::AllShortestPaths => "ALLSHORTESTPATHS",
295            Self::And => "AND",
296            Self::Any => "ANY",
297            Self::As => "AS",
298            Self::Asc => "ASC",
299            Self::Ascending => "ASCENDING",
300            Self::By => "BY",
301            Self::Call => "CALL",
302            Self::Case => "CASE",
303            Self::Collect => "COLLECT",
304            Self::Contains => "CONTAINS",
305            Self::Count => "COUNT",
306            Self::Create => "CREATE",
307            Self::Delete => "DELETE",
308            Self::Desc => "DESC",
309            Self::Descending => "DESCENDING",
310            Self::Detach => "DETACH",
311            Self::Distinct => "DISTINCT",
312            Self::Else => "ELSE",
313            Self::End => "END",
314            Self::Ends => "ENDS",
315            Self::Exists => "EXISTS",
316            Self::False => "FALSE",
317            Self::Group => "GROUP",
318            Self::Groups => "GROUPS",
319            Self::In => "IN",
320            Self::Inf => "INF",
321            Self::Infinity => "INFINITY",
322            Self::Is => "IS",
323            Self::Limit => "LIMIT",
324            Self::Match => "MATCH",
325            Self::Merge => "MERGE",
326            Self::Nan => "NAN",
327            Self::None => "NONE",
328            Self::Not => "NOT",
329            Self::Null => "NULL",
330            Self::Offset => "OFFSET",
331            Self::On => "ON",
332            Self::Optional => "OPTIONAL",
333            Self::Or => "OR",
334            Self::Order => "ORDER",
335            Self::Path => "PATH",
336            Self::Paths => "PATHS",
337            Self::Reduce => "REDUCE",
338            Self::Remove => "REMOVE",
339            Self::Return => "RETURN",
340            Self::Set => "SET",
341            Self::Shortest => "SHORTEST",
342            Self::ShortestPath => "SHORTESTPATH",
343            Self::Simple => "SIMPLE",
344            Self::Single => "SINGLE",
345            Self::Skip => "SKIP",
346            Self::Starts => "STARTS",
347            Self::Then => "THEN",
348            Self::Trail => "TRAIL",
349            Self::Trim => "TRIM",
350            Self::True => "TRUE",
351            Self::Union => "UNION",
352            Self::Unwind => "UNWIND",
353            Self::Walk => "WALK",
354            Self::When => "WHEN",
355            Self::Where => "WHERE",
356            Self::With => "WITH",
357            Self::Xor => "XOR",
358            Self::Yield => "YIELD",
359        }
360    }
361
362    /// Recognizes an ASCII keyword without regard to case.
363    ///
364    /// Non-ASCII input is never a keyword.
365    #[must_use]
366    pub fn from_ascii_case_insensitive(text: &str) -> Option<Self> {
367        if !text.is_ascii() {
368            return None;
369        }
370
371        let candidates: &[Keyword] = match text.len() {
372            2 => &[Self::As, Self::By, Self::In, Self::Is, Self::On, Self::Or],
373            3 => &[
374                Self::All,
375                Self::And,
376                Self::Any,
377                Self::Asc,
378                Self::End,
379                Self::Inf,
380                Self::Nan,
381                Self::Not,
382                Self::Set,
383                Self::Xor,
384            ],
385            4 => &[
386                Self::Call,
387                Self::Case,
388                Self::Desc,
389                Self::Else,
390                Self::Ends,
391                Self::None,
392                Self::Null,
393                Self::Path,
394                Self::Skip,
395                Self::Then,
396                Self::Trim,
397                Self::True,
398                Self::Walk,
399                Self::When,
400                Self::With,
401            ],
402            5 => &[
403                Self::Count,
404                Self::False,
405                Self::Group,
406                Self::Limit,
407                Self::Match,
408                Self::Merge,
409                Self::Order,
410                Self::Paths,
411                Self::Trail,
412                Self::Union,
413                Self::Where,
414                Self::Yield,
415            ],
416            6 => &[
417                Self::Create,
418                Self::Delete,
419                Self::Detach,
420                Self::Exists,
421                Self::Groups,
422                Self::Offset,
423                Self::Reduce,
424                Self::Remove,
425                Self::Return,
426                Self::Simple,
427                Self::Single,
428                Self::Starts,
429                Self::Unwind,
430            ],
431            7 => &[Self::Acyclic, Self::Collect],
432            8 => &[
433                Self::Contains,
434                Self::Distinct,
435                Self::Optional,
436                Self::Shortest,
437                Self::Infinity,
438            ],
439            9 => &[Self::Ascending],
440            10 => &[Self::Descending],
441            12 => &[Self::ShortestPath],
442            16 => &[Self::AllShortestPaths],
443            _ => return None,
444        };
445
446        candidates
447            .iter()
448            .copied()
449            .find(|keyword| text.eq_ignore_ascii_case(keyword.as_str()))
450    }
451}
452
453impl fmt::Display for Keyword {
454    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
455        formatter.write_str(self.as_str())
456    }
457}
open_cypher/token.rs

open_cypher/
token.rs
