gitql_parser/
token.rs

1use std::fmt::Display;
2use std::fmt::Formatter;
3use std::fmt::Result;
4
5#[derive(PartialEq)]
6pub enum TokenKind {
7    Do,
8    Set,
9    Select,
10    Distinct,
11    From,
12    Group,
13    Where,
14    Having,
15    Qualify,
16    Limit,
17    Offset,
18    Order,
19    Using,
20    Like,
21    Glob,
22    Describe,
23    Show,
24    RegExp,
25    Array,
26    Cast,
27    Benchmark,
28    Join,
29    Left,
30    Right,
31    Cross,
32    Inner,
33    Outer,
34    Case,
35    When,
36    Then,
37    Else,
38    End,
39    Into,
40    Outfile,
41    Dumpfile,
42    Lines,
43    Fields,
44    Enclosed,
45    Terminated,
46    Between,
47    By,
48    In,
49    Is,
50    On,
51    Not,
52    As,
53    With,
54    Rollup,
55    OrKeyword,
56    AndKeyword,
57    XorKeyword,
58    Ascending,
59    Descending,
60    Symmetric,
61    Asymmetric,
62    Window,
63    Over,
64    Partition,
65    First,
66    Last,
67    Interval,
68
69    // Values
70    Symbol(String),
71    GlobalVariable(String),
72    String(String),
73    Integer(i64),
74    Float(f64),
75    True,
76    False,
77    Null,
78    Nulls,
79    Infinity,
80    NaN,
81
82    All,
83    Some,
84    Any,
85
86    Greater,
87    GreaterEqual,
88    Less,
89    LessEqual,
90    Equal,
91    Bang,
92    BangEqual,
93    NullSafeEqual,
94    AtRightArrow,
95    ArrowRightAt,
96    LeftParen,
97    RightParen,
98    LeftBracket,
99    RightBracket,
100    OrOr,
101    AndAnd,
102    BitwiseNot,
103    BitwiseXor,
104    BitwiseOr,
105    BitwiseAnd,
106    BitwiseRightShift,
107    BitwiseLeftShift,
108    Colon,
109    ColonColon,
110    ColonEqual,
111    Plus,
112    Minus,
113    Star,
114    Slash,
115    Percentage,
116    Caret,
117    Comma,
118    Dot,
119    Semicolon,
120}
121
122impl Display for TokenKind {
123    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
124        let literal = match self {
125            // Reserved Keywords
126            TokenKind::Do => "DO",
127            TokenKind::Set => "SET",
128            TokenKind::Select => "SELECT",
129            TokenKind::Distinct => "DISTINCT",
130            TokenKind::From => "FROM",
131            TokenKind::Group => "GROUP",
132            TokenKind::Where => "WHERE",
133            TokenKind::Having => "HAVING",
134            TokenKind::Qualify => "QUALIFY",
135            TokenKind::Limit => "LIMIT",
136            TokenKind::Offset => "OFFSET",
137            TokenKind::Order => "ORDER",
138            TokenKind::Using => "USING",
139            TokenKind::Like => "LIKE",
140            TokenKind::Glob => "GLOB",
141            TokenKind::Describe => "DESCRIBE",
142            TokenKind::Show => "SHOW",
143            TokenKind::RegExp => "REGEXP",
144            TokenKind::Array => "ARRAY",
145            TokenKind::Cast => "CAST",
146            TokenKind::Benchmark => "BENCHMARK",
147            TokenKind::Join => "JOIN",
148            TokenKind::Left => "LEFT",
149            TokenKind::Right => "RIGHT",
150            TokenKind::Cross => "CROSS",
151            TokenKind::Inner => "INNER",
152            TokenKind::Outer => "OUTER",
153            TokenKind::Case => "CASE",
154            TokenKind::When => "WHEN",
155            TokenKind::Then => "THEN",
156            TokenKind::Else => "ELSE",
157            TokenKind::End => "END",
158            TokenKind::Into => "INTO",
159            TokenKind::Outfile => "OUTFILE",
160            TokenKind::Dumpfile => "DUMPFILE",
161            TokenKind::Lines => "LINES",
162            TokenKind::Fields => "FIELDS",
163            TokenKind::Enclosed => "ENCLOSED",
164            TokenKind::Terminated => "TERMINATED",
165            TokenKind::Between => "BETWEEN",
166            TokenKind::By => "BY",
167            TokenKind::In => "IN",
168            TokenKind::Is => "IS",
169            TokenKind::On => "ON",
170            TokenKind::Not => "NOT",
171            TokenKind::As => "AS",
172            TokenKind::With => "WITH",
173            TokenKind::Rollup => "ROLLUP",
174            TokenKind::OrKeyword => "OR",
175            TokenKind::AndKeyword => "AND",
176            TokenKind::XorKeyword => "XOE",
177            TokenKind::Ascending => "ASC",
178            TokenKind::Descending => "DESC",
179            TokenKind::Symmetric => "SYMMETRIC",
180            TokenKind::Asymmetric => "ASYMMETRIC",
181            TokenKind::Window => "WINDOW",
182            TokenKind::Over => "OVER",
183            TokenKind::Partition => "PARTITION",
184            TokenKind::Nulls => "NULLS",
185            TokenKind::First => "FIRST",
186            TokenKind::Last => "LAST",
187            TokenKind::Interval => "INTERVAL",
188
189            // Values
190            TokenKind::Symbol(literal) => literal,
191            TokenKind::GlobalVariable(literal) => literal,
192            TokenKind::String(string) => string,
193            TokenKind::Integer(integer) => &integer.to_string(),
194            TokenKind::Float(float) => &float.to_string(),
195            TokenKind::True => "True",
196            TokenKind::False => "False",
197            TokenKind::Null => "Null",
198            TokenKind::Infinity => "Infinity",
199            TokenKind::NaN => "NaN",
200
201            // Group Operators
202            TokenKind::All => "ALL",
203            TokenKind::Some => "Some",
204            TokenKind::Any => "Any",
205
206            // Others
207            TokenKind::Greater => ">",
208            TokenKind::GreaterEqual => ">=",
209            TokenKind::Less => "<",
210            TokenKind::LessEqual => "<=",
211            TokenKind::Equal => "=",
212            TokenKind::Bang => "!",
213            TokenKind::BangEqual => "!=",
214            TokenKind::NullSafeEqual => "<=>",
215            TokenKind::AtRightArrow => "@>",
216            TokenKind::ArrowRightAt => "<@",
217            TokenKind::LeftParen => "(",
218            TokenKind::RightParen => ")",
219            TokenKind::LeftBracket => "[",
220            TokenKind::RightBracket => "]",
221            TokenKind::OrOr => "||",
222            TokenKind::AndAnd => "&&",
223            TokenKind::BitwiseNot => "~",
224            TokenKind::BitwiseXor => "#",
225            TokenKind::BitwiseOr => "|",
226            TokenKind::BitwiseAnd => "&",
227            TokenKind::BitwiseRightShift => ">>",
228            TokenKind::BitwiseLeftShift => "<<",
229            TokenKind::Colon => ":",
230            TokenKind::ColonColon => "::",
231            TokenKind::ColonEqual => ":=",
232            TokenKind::Plus => "+",
233            TokenKind::Minus => "-",
234            TokenKind::Star => "*",
235            TokenKind::Slash => "/",
236            TokenKind::Percentage => "%",
237            TokenKind::Caret => "^",
238            TokenKind::Comma => ",",
239            TokenKind::Dot => ".",
240            TokenKind::Semicolon => ";",
241        };
242        f.write_str(literal)
243    }
244}
245
246#[derive(Copy, Clone)]
247pub struct SourceLocation {
248    pub line_start: u32,
249    pub line_end: u32,
250    pub column_start: u32,
251    pub column_end: u32,
252}
253
254impl SourceLocation {
255    pub fn new(
256        line_start: u32,
257        line_end: u32,
258        column_start: u32,
259        column_end: u32,
260    ) -> SourceLocation {
261        SourceLocation {
262            line_start,
263            line_end,
264            column_start,
265            column_end,
266        }
267    }
268
269    pub fn expand_until(&mut self, location: SourceLocation) {
270        self.column_end = location.column_end;
271        self.line_end = location.line_end;
272    }
273}
274
275impl Display for SourceLocation {
276    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
277        f.write_fmt(format_args!(
278            "Loc(L {}:{}, C {}:{})",
279            self.line_start, self.line_end, self.column_start, self.column_end
280        ))
281    }
282}
283
284pub struct Token {
285    pub kind: TokenKind,
286    pub location: SourceLocation,
287}
288
289impl Token {
290    pub fn new(kind: TokenKind, location: SourceLocation) -> Token {
291        Token { kind, location }
292    }
293
294    pub fn new_symbol(symbol: String, location: SourceLocation) -> Token {
295        Token {
296            kind: resolve_symbol_kind(symbol),
297            location,
298        }
299    }
300
301    pub fn has_kind(&self, kind: TokenKind) -> bool {
302        self.kind == kind
303    }
304}
305
306impl Display for Token {
307    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
308        f.write_str(&self.kind.to_string())
309    }
310}
311
312fn resolve_symbol_kind(symbol: String) -> TokenKind {
313    match symbol.to_lowercase().as_str() {
314        // Reserved keywords
315        "do" => TokenKind::Do,
316        "set" => TokenKind::Set,
317        "select" => TokenKind::Select,
318        "distinct" => TokenKind::Distinct,
319        "from" => TokenKind::From,
320        "where" => TokenKind::Where,
321        "qualify" => TokenKind::Qualify,
322        "limit" => TokenKind::Limit,
323        "offset" => TokenKind::Offset,
324        "order" => TokenKind::Order,
325        "using" => TokenKind::Using,
326        "case" => TokenKind::Case,
327        "when" => TokenKind::When,
328        "then" => TokenKind::Then,
329        "else" => TokenKind::Else,
330        "end" => TokenKind::End,
331        "between" => TokenKind::Between,
332        "in" => TokenKind::In,
333        "is" => TokenKind::Is,
334        "on" => TokenKind::On,
335        "not" => TokenKind::Not,
336        "like" => TokenKind::Like,
337        "glob" => TokenKind::Glob,
338        "describe" => TokenKind::Describe,
339        "show" => TokenKind::Show,
340        "regexp" => TokenKind::RegExp,
341
342        "cast" => TokenKind::Cast,
343        "benchmark" => TokenKind::Benchmark,
344
345        "interval" => TokenKind::Interval,
346
347        // Select into
348        "into" => TokenKind::Into,
349        "outfile" => TokenKind::Outfile,
350        "dumpfile" => TokenKind::Dumpfile,
351        "lines" => TokenKind::Lines,
352        "fields" => TokenKind::Fields,
353        "enclosed" => TokenKind::Enclosed,
354        "terminated" => TokenKind::Terminated,
355
356        // Joins
357        "join" => TokenKind::Join,
358        "left" => TokenKind::Left,
359        "right" => TokenKind::Right,
360        "cross" => TokenKind::Cross,
361        "inner" => TokenKind::Inner,
362        "outer" => TokenKind::Outer,
363
364        // Grouping
365        "group" => TokenKind::Group,
366        "by" => TokenKind::By,
367        "having" => TokenKind::Having,
368        "with" => TokenKind::With,
369        "rollup" => TokenKind::Rollup,
370
371        // Between kind
372        "symmetric" => TokenKind::Symmetric,
373        "asymmetric" => TokenKind::Asymmetric,
374
375        // Integer division and Modulo operator
376        "div" => TokenKind::Slash,
377        "mod" => TokenKind::Percentage,
378
379        // Logical Operators
380        "or" => TokenKind::OrKeyword,
381        "and" => TokenKind::AndKeyword,
382        "xor" => TokenKind::XorKeyword,
383
384        // Group Operators
385        "all" => TokenKind::All,
386        "some" => TokenKind::Some,
387        "any" => TokenKind::Any,
388
389        // True, False and Null
390        "true" => TokenKind::True,
391        "false" => TokenKind::False,
392        "null" => TokenKind::Null,
393        "nulls" => TokenKind::Nulls,
394
395        // Infinity and NaN
396        "infinity" => TokenKind::Infinity,
397        "nan" => TokenKind::NaN,
398
399        // As for alias
400        "as" => TokenKind::As,
401
402        // Order by DES and ASC
403        "asc" => TokenKind::Ascending,
404        "desc" => TokenKind::Descending,
405
406        // Order by null ordering policy Null first and last
407        "first" => TokenKind::First,
408        "last" => TokenKind::Last,
409
410        // Array data type
411        "array" => TokenKind::Array,
412
413        // Over
414        "window" => TokenKind::Window,
415        "over" => TokenKind::Over,
416        "partition" => TokenKind::Partition,
417
418        // Identifier
419        _ => TokenKind::Symbol(symbol),
420    }
421}
gitql_parser/token.rs

gitql_parser/
token.rs
