vibesql_parser/
token.rs

1use std::fmt;
2
3use crate::keywords::Keyword;
4
5/// Multi-character operators that require heap allocation if stored as String.
6/// Using an enum eliminates allocation and enables fast matching.
7#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8pub enum MultiCharOperator {
9    /// <= (less than or equal)
10    LessEqual,
11    /// >= (greater than or equal)
12    GreaterEqual,
13    /// != (not equal)
14    NotEqual,
15    /// <> (not equal, SQL standard)
16    NotEqualAlt,
17    /// || (string concatenation)
18    Concat,
19    /// <-> (cosine distance - pgvector compatible)
20    CosineDistance,
21    /// <#> (negative inner product - pgvector compatible)
22    NegativeInnerProduct,
23    /// <=> (L2/Euclidean distance - pgvector compatible)
24    L2Distance,
25}
26
27impl fmt::Display for MultiCharOperator {
28    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
29        match self {
30            MultiCharOperator::LessEqual => write!(f, "<="),
31            MultiCharOperator::GreaterEqual => write!(f, ">="),
32            MultiCharOperator::NotEqual => write!(f, "!="),
33            MultiCharOperator::NotEqualAlt => write!(f, "<>"),
34            MultiCharOperator::Concat => write!(f, "||"),
35            MultiCharOperator::CosineDistance => write!(f, "<->"),
36            MultiCharOperator::NegativeInnerProduct => write!(f, "<#>"),
37            MultiCharOperator::L2Distance => write!(f, "<=>"),
38        }
39    }
40}
41
42/// SQL Token produced by the lexer.
43#[derive(Debug, Clone, PartialEq)]
44pub enum Token {
45    /// SQL keyword (SELECT, FROM, etc.)
46    Keyword(Keyword),
47    /// Identifier (table name, column name, etc.)
48    Identifier(String),
49    /// Delimited identifier ("columnName" - case-sensitive, can use reserved words)
50    DelimitedIdentifier(String),
51    /// Numeric literal (42, 3.14, etc.)
52    Number(String),
53    /// String literal ('hello')
54    String(String),
55    /// Single character symbols (+, -, *, /, =, <, >, etc.)
56    Symbol(char),
57    /// Multi-character operators (<=, >=, !=, <>, ||)
58    Operator(MultiCharOperator),
59    /// Session variable (@@variable, @@session.variable, @@global.variable)
60    SessionVariable(String),
61    /// User variable (@variable)
62    UserVariable(String),
63    /// Parameter placeholder (?) for prepared statements
64    /// The index is assigned during parsing (0-indexed, in order of appearance)
65    Placeholder,
66    /// Numbered parameter placeholder ($1, $2, etc.) for prepared statements
67    /// PostgreSQL-style: 1-indexed as written in SQL ($1 = first parameter)
68    NumberedPlaceholder(usize),
69    /// Named parameter placeholder (:name) for prepared statements
70    /// Used by many ORMs and applications for readability
71    NamedPlaceholder(String),
72    /// Semicolon (statement terminator)
73    Semicolon,
74    /// Comma (separator)
75    Comma,
76    /// Left parenthesis
77    LParen,
78    /// Right parenthesis
79    RParen,
80    /// End of input
81    Eof,
82}
83
84impl fmt::Display for Token {
85    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
86        match self {
87            Token::Keyword(kw) => write!(f, "Keyword({})", kw),
88            Token::Identifier(id) => write!(f, "Identifier({})", id),
89            Token::DelimitedIdentifier(id) => write!(f, "DelimitedIdentifier(\"{}\")", id),
90            Token::Number(n) => write!(f, "Number({})", n),
91            Token::String(s) => write!(f, "String('{}')", s),
92            Token::Symbol(c) => write!(f, "Symbol({})", c),
93            Token::Operator(op) => write!(f, "Operator({})", op),
94            Token::SessionVariable(v) => write!(f, "SessionVariable({})", v),
95            Token::UserVariable(v) => write!(f, "UserVariable({})", v),
96            Token::Placeholder => write!(f, "Placeholder"),
97            Token::NumberedPlaceholder(n) => write!(f, "NumberedPlaceholder(${})", n),
98            Token::NamedPlaceholder(name) => write!(f, "NamedPlaceholder(:{})", name),
99            Token::Semicolon => write!(f, "Semicolon"),
100            Token::Comma => write!(f, "Comma"),
101            Token::LParen => write!(f, "LParen"),
102            Token::RParen => write!(f, "RParen"),
103            Token::Eof => write!(f, "Eof"),
104        }
105    }
106}
vibesql_parser/token.rs

vibesql_parser/
token.rs
