oak_sql/lexer/
token_type.rs

1use oak_core::{Token, UniversalTokenRole};
2
3/// Represents a token in an SQL source file.
4pub type SqlToken = Token<SqlTokenType>;
5
6impl oak_core::TokenType for SqlTokenType {
7    type Role = UniversalTokenRole;
8    const END_OF_STREAM: Self = Self::Eof;
9
10    fn is_ignored(&self) -> bool {
11        matches!(self, SqlTokenType::Whitespace | SqlTokenType::Newline | SqlTokenType::Comment | SqlTokenType::LineComment | SqlTokenType::BlockComment)
12    }
13
14    fn is_comment(&self) -> bool {
15        matches!(self, SqlTokenType::Comment | SqlTokenType::LineComment | SqlTokenType::BlockComment)
16    }
17
18    fn is_whitespace(&self) -> bool {
19        matches!(self, SqlTokenType::Whitespace | SqlTokenType::Newline)
20    }
21
22    fn role(&self) -> Self::Role {
23        use UniversalTokenRole::*;
24        match self {
25            Self::Whitespace | Self::Newline => Whitespace,
26            Self::Comment | Self::LineComment | Self::BlockComment => Comment,
27            Self::NumberLiteral | Self::FloatLiteral | Self::StringLiteral | Self::BooleanLiteral | Self::NullLiteral => Literal,
28            Self::Identifier_ => Name,
29            Self::Select
30            | Self::From
31            | Self::Where
32            | Self::Insert
33            | Self::Into
34            | Self::Values
35            | Self::Update
36            | Self::Set
37            | Self::Delete
38            | Self::Create
39            | Self::Table
40            | Self::Drop
41            | Self::Alter
42            | Self::Add
43            | Self::Column
44            | Self::Primary
45            | Self::Key
46            | Self::Foreign
47            | Self::References
48            | Self::Index
49            | Self::Unique
50            | Self::Not
51            | Self::Null
52            | Self::Default
53            | Self::AutoIncrement
54            | Self::And
55            | Self::Or
56            | Self::In
57            | Self::Like
58            | Self::Between
59            | Self::Is
60            | Self::As
61            | Self::Join
62            | Self::Inner
63            | Self::Left
64            | Self::Right
65            | Self::Full
66            | Self::Outer
67            | Self::On
68            | Self::Group
69            | Self::By
70            | Self::Having
71            | Self::Order
72            | Self::Asc
73            | Self::Desc
74            | Self::Limit
75            | Self::Offset
76            | Self::Union
77            | Self::All
78            | Self::Distinct
79            | Self::Count
80            | Self::Sum
81            | Self::Avg
82            | Self::Min
83            | Self::Max
84            | Self::Explain
85            | Self::Vector
86            | Self::View
87            | Self::Database
88            | Self::Schema
89            | Self::True
90            | Self::False
91            | Self::Exists
92            | Self::Trigger
93            | Self::After
94            | Self::Delimiter
95            | Self::For
96            | Self::Each
97            | Self::Row
98            | Self::Check
99            | Self::Rename
100            | Self::To
101            | Self::Case
102            | Self::When
103            | Self::Then
104            | Self::Else
105            | Self::End
106            | Self::If
107            | Self::Begin
108            | Self::Commit
109            | Self::Rollback
110            | Self::Transaction
111            | Self::Conflict
112            | Self::Do
113            | Self::Nothing
114            | Self::Returning
115            | Self::Ilike
116            | Self::Strict
117            | Self::Without
118            | Self::Rowid => Keyword,
119            Self::Int | Self::Integer | Self::Varchar | Self::Char | Self::Text | Self::Date | Self::Time | Self::Timestamp | Self::Decimal | Self::Float | Self::Double | Self::Boolean | Self::Serial | Self::BigSerial => Keyword, /* Types are often keywords */
120            Self::Plus | Self::Minus | Self::Star | Self::Slash | Self::Percent | Self::Equal | Self::NotEqual | Self::Less | Self::Greater | Self::LessEqual | Self::GreaterEqual | Self::Concat => Operator,
121            Self::LeftParen | Self::RightParen | Self::LeftBracket | Self::RightBracket | Self::LeftBrace | Self::RightBrace | Self::Comma | Self::Semicolon | Self::Dot | Self::Colon | Self::Question => Punctuation,
122            Self::Error => Error,
123            _ => UniversalTokenRole::None,
124        }
125    }
126}
127
128/// Token types for the SQL language.
129#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
130#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
131pub enum SqlTokenType {
132    /// Root node.
133    Root,
134    /// EXPLAIN statement.
135    ExplainStatement,
136    /// TRANSACTION statement.
137    TransactionStatement,
138    /// PRAGMA statement.
139    PragmaStatement,
140    /// SHOW statement.
141    ShowStatement,
142    /// SELECT statement.
143    SelectStatement,
144    /// INSERT statement.
145    InsertStatement,
146    /// UPDATE statement.
147    UpdateStatement,
148    /// DELETE statement.
149    DeleteStatement,
150    /// CREATE statement.
151    CreateStatement,
152    /// DROP statement.
153    DropStatement,
154    /// ALTER statement.
155    AlterStatement,
156    /// SQL expression.
157    Expression,
158    /// Identifier.
159    Identifier,
160    /// Table name.
161    TableName,
162    /// Column name.
163    ColumnName,
164    /// JOIN clause.
165    JoinClause,
166    /// GROUP BY clause.
167    GroupByClause,
168    /// HAVING clause.
169    HavingClause,
170    /// ORDER BY clause.
171    OrderByClause,
172    /// LIMIT clause.
173    LimitClause,
174    /// Select item.
175    SelectItem,
176    /// Alias.
177    Alias,
178    /// Column definition.
179    ColumnDefinition,
180    /// Value list.
181    ValueList,
182    /// Assignment.
183    Assignment,
184    /// Alter action.
185    AlterAction,
186    /// Error node.
187    ErrorNode,
188
189    /// Whitespace.
190    Whitespace,
191    /// Newline.
192    Newline,
193
194    /// Comment.
195    Comment,
196    /// Line comment.
197    LineComment,
198    /// Block comment.
199    BlockComment,
200
201    /// Number literal.
202    NumberLiteral,
203    /// Float literal.
204    FloatLiteral,
205    /// String literal.
206    StringLiteral,
207    /// Boolean literal.
208    BooleanLiteral,
209    /// NULL literal.
210    NullLiteral,
211
212    /// Identifier token.
213    Identifier_,
214
215    /// `SELECT` keyword.
216    Select,
217    /// `FROM` keyword.
218    From,
219    /// `WHERE` keyword.
220    Where,
221    /// `INSERT` keyword.
222    Insert,
223    /// `INTO` keyword.
224    Into,
225    /// `VALUES` keyword.
226    Values,
227    /// `UPDATE` keyword.
228    Update,
229    /// `SET` keyword.
230    Set,
231    /// `DELETE` keyword.
232    Delete,
233    /// `CREATE` keyword.
234    Create,
235    /// `TABLE` keyword.
236    Table,
237    /// `DROP` keyword.
238    Drop,
239    /// `ALTER` keyword.
240    Alter,
241    /// `ADD` keyword.
242    Add,
243    /// `COLUMN` keyword.
244    Column,
245    /// `PRIMARY` keyword.
246    Primary,
247    /// `KEY` keyword.
248    Key,
249    /// `FOREIGN` keyword.
250    Foreign,
251    /// `REFERENCES` keyword.
252    References,
253    /// `INDEX` keyword.
254    Index,
255    /// `UNIQUE` keyword.
256    Unique,
257    /// `NOT` keyword.
258    Not,
259    /// `NULL` keyword.
260    Null,
261    /// `DEFAULT` keyword.
262    Default,
263    /// `AUTO_INCREMENT` keyword.
264    AutoIncrement,
265    /// `AND` keyword.
266    And,
267    /// `OR` keyword.
268    Or,
269    /// `IN` keyword.
270    In,
271    /// `LIKE` keyword.
272    Like,
273    /// `BETWEEN` keyword.
274    Between,
275    /// `IS` keyword.
276    Is,
277    /// `AS` keyword.
278    As,
279    /// `JOIN` keyword.
280    Join,
281    /// `INNER` keyword.
282    Inner,
283    /// `LEFT` keyword.
284    Left,
285    /// `RIGHT` keyword.
286    Right,
287    /// `FULL` keyword.
288    Full,
289    /// `OUTER` keyword.
290    Outer,
291    /// `ON` keyword.
292    On,
293    /// `GROUP` keyword.
294    Group,
295    /// `BY` keyword.
296    By,
297    /// `HAVING` keyword.
298    Having,
299    /// `ORDER` keyword.
300    Order,
301    /// `ASC` keyword.
302    Asc,
303    /// `DESC` keyword.
304    Desc,
305    /// `LIMIT` keyword.
306    Limit,
307    /// `OFFSET` keyword.
308    Offset,
309    /// `UNION` keyword.
310    Union,
311    /// `ALL` keyword.
312    All,
313    /// `DISTINCT` keyword.
314    Distinct,
315    /// `COUNT` keyword.
316    Count,
317    /// `SUM` keyword.
318    Sum,
319    /// `AVG` keyword.
320    Avg,
321    /// `MIN` keyword.
322    Min,
323    /// `MAX` keyword.
324    Max,
325    /// `EXPLAIN` keyword.
326    Explain,
327    /// `PRAGMA` keyword.
328    Pragma,
329    /// `SHOW` keyword.
330    Show,
331    /// `VIEW` keyword.
332    View,
333    /// `DATABASE` keyword.
334    Database,
335    /// `SCHEMA` keyword.
336    Schema,
337    /// `TRUE` keyword.
338    True,
339    /// `FALSE` keyword.
340    False,
341    /// `EXISTS` keyword.
342    Exists,
343    /// `TRIGGER` keyword.
344    Trigger,
345    /// `AFTER` keyword.
346    After,
347    /// `DELIMITER` keyword.
348    Delimiter,
349    /// `FOR` keyword.
350    For,
351    /// `EACH` keyword.
352    Each,
353    /// `ROW` keyword.
354    Row,
355    /// `CHECK` keyword.
356    Check,
357    /// `RENAME` keyword.
358    Rename,
359    /// `TO` keyword.
360    To,
361    /// `CASE` keyword.
362    Case,
363    /// `WHEN` keyword.
364    When,
365    /// `THEN` keyword.
366    Then,
367    /// `ELSE` keyword.
368    Else,
369    /// `END` keyword.
370    End,
371    /// `IF` keyword.
372    If,
373    /// `BEGIN` keyword.
374    Begin,
375    /// `COMMIT` keyword.
376    Commit,
377    /// `ROLLBACK` keyword.
378    Rollback,
379    /// `TRANSACTION` keyword.
380    Transaction,
381    /// `CONFLICT` keyword.
382    Conflict,
383    /// `DO` keyword.
384    Do,
385    /// `NOTHING` keyword.
386    Nothing,
387    /// `RETURNING` keyword.
388    Returning,
389    /// `VECTOR` keyword.
390    Vector,
391    /// `ILIKE` keyword.
392    Ilike,
393    /// `STRICT` keyword.
394    Strict,
395    /// `WITHOUT` keyword.
396    Without,
397    /// `ROWID` keyword.
398    Rowid,
399
400    /// `INT` type.
401    Int,
402    /// `INTEGER` type.
403    Integer,
404    /// `VARCHAR` type.
405    Varchar,
406    /// `CHAR` type.
407    Char,
408    /// `TEXT` type.
409    Text,
410    /// `DATE` type.
411    Date,
412    /// `TIME` type.
413    Time,
414    /// `TIMESTAMP` type.
415    Timestamp,
416    /// `DECIMAL` type.
417    Decimal,
418    /// `FLOAT` type.
419    Float,
420    /// `DOUBLE` type.
421    Double,
422    /// `BOOLEAN` type.
423    Boolean,
424    /// `SERIAL` type.
425    Serial,
426    /// `BIGSERIAL` type.
427    BigSerial,
428
429    /// Plus operator `+`.
430    Plus,
431    /// Minus operator `-`.
432    Minus,
433    /// Multiplication operator `*`.
434    Star,
435    /// Division operator `/`.
436    Slash,
437    /// Modulo operator `%`.
438    Percent,
439    /// Equality operator `=`.
440    Equal,
441    /// Inequality operator `<>`.
442    NotEqual,
443    /// Less than operator `<`.
444    Less,
445    /// Greater than operator `>`.
446    Greater,
447    /// Less than or equal operator `<=`.
448    LessEqual,
449    /// Greater than or equal operator `>=`.
450    GreaterEqual,
451    /// Concatenation operator `||`.
452    Concat,
453    /// Double Colon `::`.
454    DoubleColon,
455    /// Left parenthesis `(`.
456    LeftParen,
457    /// Right parenthesis `)`.
458    RightParen,
459    /// Left bracket `[`.
460    LeftBracket,
461    /// Right bracket `]`.
462    RightBracket,
463    /// Left brace `{`.
464    LeftBrace,
465    /// Right brace `}`.
466    RightBrace,
467    /// Comma `,`.
468    Comma,
469    /// Semicolon `;`.
470    Semicolon,
471    /// Dot `.`.
472    Dot,
473    /// Colon `:`.
474    Colon,
475    /// Question mark `?`.
476    Question,
477
478    /// Error token.
479    Error,
480    /// End of stream.
481    Eof,
482}
483
484impl From<crate::parser::element_type::SqlElementType> for SqlTokenType {
485    fn from(element: crate::parser::element_type::SqlElementType) -> Self {
486        use crate::parser::element_type::SqlElementType;
487        match element {
488            SqlElementType::Root => Self::Root,
489            SqlElementType::ExplainStatement => Self::ExplainStatement,
490            SqlElementType::Identifier => Self::Identifier,
491            SqlElementType::Expression => Self::Expression,
492            SqlElementType::ErrorNode => Self::ErrorNode,
493            SqlElementType::SelectStatement => Self::SelectStatement,
494            SqlElementType::VectorSearch => Self::Vector, // Map VectorSearch to Vector token for now or add a new one?
495            SqlElementType::InsertStatement => Self::InsertStatement,
496            SqlElementType::UpdateStatement => Self::UpdateStatement,
497            SqlElementType::DeleteStatement => Self::DeleteStatement,
498            SqlElementType::CreateStatement => Self::CreateStatement,
499            SqlElementType::DropStatement => Self::DropStatement,
500            SqlElementType::AlterStatement => Self::AlterStatement,
501            SqlElementType::JoinClause => Self::JoinClause,
502            SqlElementType::GroupByClause => Self::GroupByClause,
503            SqlElementType::HavingClause => Self::HavingClause,
504            SqlElementType::OrderByClause => Self::OrderByClause,
505            SqlElementType::LimitClause => Self::LimitClause,
506            SqlElementType::TableName => Self::TableName,
507            SqlElementType::ColumnName => Self::ColumnName,
508            SqlElementType::SelectItem => Self::SelectItem,
509            SqlElementType::Alias => Self::Alias,
510            SqlElementType::ColumnDefinition => Self::ColumnDefinition,
511            SqlElementType::ValueList => Self::ValueList,
512            SqlElementType::Assignment => Self::Assignment,
513            SqlElementType::AlterAction => Self::AlterAction,
514            _ => Self::ErrorNode,
515        }
516    }
517}
oak_sql/lexer/token_type.rs

oak_sql/lexer/
token_type.rs
