vibesql_parser/
token.rs

1use std::fmt;
2
3use crate::keywords::Keyword;
4
5/// Multi-character operators that require heap allocation if stored as String.
6/// Using an enum eliminates allocation and enables fast matching.
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum MultiCharOperator {
9    /// <= (less than or equal)
10    LessEqual,
11    /// >= (greater than or equal)
12    GreaterEqual,
13    /// != (not equal)
14    NotEqual,
15    /// <> (not equal, SQL standard)
16    NotEqualAlt,
17    /// || (string concatenation)
18    Concat,
19}
20
21impl fmt::Display for MultiCharOperator {
22    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
23        match self {
24            MultiCharOperator::LessEqual => write!(f, "<="),
25            MultiCharOperator::GreaterEqual => write!(f, ">="),
26            MultiCharOperator::NotEqual => write!(f, "!="),
27            MultiCharOperator::NotEqualAlt => write!(f, "<>"),
28            MultiCharOperator::Concat => write!(f, "||"),
29        }
30    }
31}
32
33/// SQL Token produced by the lexer.
34#[derive(Debug, Clone, PartialEq)]
35pub enum Token {
36    /// SQL keyword (SELECT, FROM, etc.)
37    Keyword(Keyword),
38    /// Identifier (table name, column name, etc.)
39    Identifier(String),
40    /// Delimited identifier ("columnName" - case-sensitive, can use reserved words)
41    DelimitedIdentifier(String),
42    /// Numeric literal (42, 3.14, etc.)
43    Number(String),
44    /// String literal ('hello')
45    String(String),
46    /// Single character symbols (+, -, *, /, =, <, >, etc.)
47    Symbol(char),
48    /// Multi-character operators (<=, >=, !=, <>, ||)
49    Operator(MultiCharOperator),
50    /// Session variable (@@variable, @@session.variable, @@global.variable)
51    SessionVariable(String),
52    /// User variable (@variable)
53    UserVariable(String),
54    /// Parameter placeholder (?) for prepared statements
55    /// The index is assigned during parsing (0-indexed, in order of appearance)
56    Placeholder,
57    /// Numbered parameter placeholder ($1, $2, etc.) for prepared statements
58    /// PostgreSQL-style: 1-indexed as written in SQL ($1 = first parameter)
59    NumberedPlaceholder(usize),
60    /// Named parameter placeholder (:name) for prepared statements
61    /// Used by many ORMs and applications for readability
62    NamedPlaceholder(String),
63    /// Semicolon (statement terminator)
64    Semicolon,
65    /// Comma (separator)
66    Comma,
67    /// Left parenthesis
68    LParen,
69    /// Right parenthesis
70    RParen,
71    /// End of input
72    Eof,
73}
74
75impl fmt::Display for Token {
76    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
77        match self {
78            Token::Keyword(kw) => write!(f, "Keyword({})", kw),
79            Token::Identifier(id) => write!(f, "Identifier({})", id),
80            Token::DelimitedIdentifier(id) => write!(f, "DelimitedIdentifier(\"{}\")", id),
81            Token::Number(n) => write!(f, "Number({})", n),
82            Token::String(s) => write!(f, "String('{}')", s),
83            Token::Symbol(c) => write!(f, "Symbol({})", c),
84            Token::Operator(op) => write!(f, "Operator({})", op),
85            Token::SessionVariable(v) => write!(f, "SessionVariable({})", v),
86            Token::UserVariable(v) => write!(f, "UserVariable({})", v),
87            Token::Placeholder => write!(f, "Placeholder"),
88            Token::NumberedPlaceholder(n) => write!(f, "NumberedPlaceholder(${})", n),
89            Token::NamedPlaceholder(name) => write!(f, "NamedPlaceholder(:{})", name),
90            Token::Semicolon => write!(f, "Semicolon"),
91            Token::Comma => write!(f, "Comma"),
92            Token::LParen => write!(f, "LParen"),
93            Token::RParen => write!(f, "RParen"),
94            Token::Eof => write!(f, "Eof"),
95        }
96    }
97}
vibesql_parser/token.rs

vibesql_parser/
token.rs
