rigsql_core/
token.rs

1use smol_str::SmolStr;
2use strum::{Display, EnumString};
3
4use crate::Span;
5
6/// A single token produced by the lexer.
7#[derive(Debug, Clone, PartialEq, Eq)]
8pub struct Token {
9    pub kind: TokenKind,
10    pub span: Span,
11    pub text: SmolStr,
12}
13
14impl Token {
15    pub fn new(kind: TokenKind, span: Span, text: impl Into<SmolStr>) -> Self {
16        Self {
17            kind,
18            span,
19            text: text.into(),
20        }
21    }
22}
23
24/// Classification of tokens.
25#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display, EnumString)]
26pub enum TokenKind {
27    // Identifiers & Keywords
28    /// An unquoted word (keyword or identifier — distinguished later by dialect).
29    Word,
30
31    // Literals
32    NumberLiteral,
33    /// Single-quoted string: 'hello'
34    StringLiteral,
35    /// Quoted identifier: "col", [col], `col`
36    QuotedIdentifier,
37
38    // Operators
39    Dot,
40    Comma,
41    Semicolon,
42    LParen,
43    RParen,
44    Star,
45    Plus,
46    Minus,
47    Slash,
48    Percent,
49    Eq,
50    Neq, // <> or !=
51    Lt,
52    Gt,
53    LtEq,
54    GtEq,
55    Concat,     // ||
56    ColonColon, // :: (PostgreSQL cast)
57    AtSign,     // @ (SQL Server variable prefix)
58    Colon,      // : (named parameter)
59
60    // Whitespace & Comments
61    Whitespace,
62    Newline,
63    LineComment,  // -- ...
64    BlockComment, // /* ... */
65
66    // Special
67    /// Parameter placeholder: :name, $1, ?
68    Placeholder,
69    /// End of file
70    Eof,
71}
72
73impl TokenKind {
74    /// Returns true if this token is trivia (whitespace or comment).
75    pub fn is_trivia(self) -> bool {
76        matches!(
77            self,
78            TokenKind::Whitespace
79                | TokenKind::Newline
80                | TokenKind::LineComment
81                | TokenKind::BlockComment
82        )
83    }
84}
rigsql_core/token.rs

rigsql_core/
token.rs
