rigsql_core/
token.rs

1use smol_str::SmolStr;
2use strum::{Display, EnumString};
3
4use crate::Span;
5
6/// A single token produced by the lexer.
7#[derive(Debug, Clone, PartialEq, Eq)]
8pub struct Token {
9    pub kind: TokenKind,
10    pub span: Span,
11    pub text: SmolStr,
12}
13
14impl Token {
15    pub fn new(kind: TokenKind, span: Span, text: impl Into<SmolStr>) -> Self {
16        Self {
17            kind,
18            span,
19            text: text.into(),
20        }
21    }
22}
23
24/// Classification of tokens.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, EnumString)]
26pub enum TokenKind {
27    // Identifiers & Keywords
28    /// An unquoted word (keyword or identifier — distinguished later by dialect).
29    Word,
30
31    // Literals
32    NumberLiteral,
33    /// Single-quoted string: 'hello'
34    StringLiteral,
35    /// Quoted identifier: "col", [col], `col`
36    QuotedIdentifier,
37
38    // Operators
39    Dot,
40    Comma,
41    Semicolon,
42    LParen,
43    RParen,
44    Star,
45    Plus,
46    Minus,
47    Slash,
48    Percent,
49    Eq,
50    Neq, // <> or !=
51    Lt,
52    Gt,
53    LtEq,
54    GtEq,
55    Concat,     // ||
56    ColonColon, // :: (PostgreSQL cast)
57    AtSign,     // @ (SQL Server variable prefix)
58    Colon,      // : (named parameter)
59    LBracket,   // [ (array subscript, PostgreSQL)
60    RBracket,   // ] (array subscript, PostgreSQL)
61
62    // Whitespace & Comments
63    Whitespace,
64    Newline,
65    LineComment,  // -- ...
66    BlockComment, // /* ... */
67
68    // Special
69    /// Parameter placeholder: :name, $1, ?
70    Placeholder,
71    /// End of file
72    Eof,
73}
74
75impl TokenKind {
76    /// Returns true if this token is trivia (whitespace or comment).
77    pub fn is_trivia(self) -> bool {
78        matches!(
79            self,
80            TokenKind::Whitespace
81                | TokenKind::Newline
82                | TokenKind::LineComment
83                | TokenKind::BlockComment
84        )
85    }
86}
rigsql_core/token.rs

rigsql_core/
token.rs
