gitql_parser/
token.rs

1use std::fmt::Display;
2use std::fmt::Formatter;
3use std::fmt::Result;
4
5#[derive(PartialEq)]
6pub enum TokenKind {
7    Do,
8    Set,
9    Select,
10    Distinct,
11    From,
12    Group,
13    Where,
14    Having,
15    Limit,
16    Offset,
17    Order,
18    Using,
19    Like,
20    Glob,
21    Describe,
22    Show,
23    RegExp,
24    Array,
25    Cast,
26    Benchmark,
27    Join,
28    Left,
29    Right,
30    Cross,
31    Inner,
32    Outer,
33    Case,
34    When,
35    Then,
36    Else,
37    End,
38    Into,
39    Outfile,
40    Dumpfile,
41    Lines,
42    Fields,
43    Enclosed,
44    Terminated,
45    Between,
46    By,
47    In,
48    Is,
49    On,
50    Not,
51    As,
52    With,
53    Rollup,
54    OrKeyword,
55    AndKeyword,
56    XorKeyword,
57    Ascending,
58    Descending,
59    Symmetric,
60    Asymmetric,
61    Window,
62    Over,
63    Partition,
64    First,
65    Last,
66    Interval,
67
68    // Values
69    Symbol(String),
70    GlobalVariable(String),
71    String(String),
72    Integer(i64),
73    Float(f64),
74    True,
75    False,
76    Null,
77    Nulls,
78    Infinity,
79    NaN,
80
81    All,
82    Some,
83    Any,
84
85    Greater,
86    GreaterEqual,
87    Less,
88    LessEqual,
89    Equal,
90    Bang,
91    BangEqual,
92    NullSafeEqual,
93    AtRightArrow,
94    ArrowRightAt,
95    LeftParen,
96    RightParen,
97    LeftBracket,
98    RightBracket,
99    OrOr,
100    AndAnd,
101    BitwiseNot,
102    BitwiseXor,
103    BitwiseOr,
104    BitwiseAnd,
105    BitwiseRightShift,
106    BitwiseLeftShift,
107    Colon,
108    ColonColon,
109    ColonEqual,
110    Plus,
111    Minus,
112    Star,
113    Slash,
114    Percentage,
115    Caret,
116    Comma,
117    Dot,
118    Semicolon,
119}
120
121impl Display for TokenKind {
122    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
123        let literal = match self {
124            // Reserved Keywords
125            TokenKind::Do => "DO",
126            TokenKind::Set => "SET",
127            TokenKind::Select => "SELECT",
128            TokenKind::Distinct => "DISTINCT",
129            TokenKind::From => "FROM",
130            TokenKind::Group => "GROUP",
131            TokenKind::Where => "WHERE",
132            TokenKind::Having => "HAVING",
133            TokenKind::Limit => "LIMIT",
134            TokenKind::Offset => "OFFSET",
135            TokenKind::Order => "ORDER",
136            TokenKind::Using => "USING",
137            TokenKind::Like => "LIKE",
138            TokenKind::Glob => "GLOB",
139            TokenKind::Describe => "DESCRIBE",
140            TokenKind::Show => "SHOW",
141            TokenKind::RegExp => "REGEXP",
142            TokenKind::Array => "ARRAY",
143            TokenKind::Cast => "CAST",
144            TokenKind::Benchmark => "BENCHMARK",
145            TokenKind::Join => "JOIN",
146            TokenKind::Left => "LEFT",
147            TokenKind::Right => "RIGHT",
148            TokenKind::Cross => "CROSS",
149            TokenKind::Inner => "INNER",
150            TokenKind::Outer => "OUTER",
151            TokenKind::Case => "CASE",
152            TokenKind::When => "WHEN",
153            TokenKind::Then => "THEN",
154            TokenKind::Else => "ELSE",
155            TokenKind::End => "END",
156            TokenKind::Into => "INTO",
157            TokenKind::Outfile => "OUTFILE",
158            TokenKind::Dumpfile => "DUMPFILE",
159            TokenKind::Lines => "LINES",
160            TokenKind::Fields => "FIELDS",
161            TokenKind::Enclosed => "ENCLOSED",
162            TokenKind::Terminated => "TERMINATED",
163            TokenKind::Between => "BETWEEN",
164            TokenKind::By => "BY",
165            TokenKind::In => "IN",
166            TokenKind::Is => "IS",
167            TokenKind::On => "ON",
168            TokenKind::Not => "NOT",
169            TokenKind::As => "AS",
170            TokenKind::With => "WITH",
171            TokenKind::Rollup => "ROLLUP",
172            TokenKind::OrKeyword => "OR",
173            TokenKind::AndKeyword => "AND",
174            TokenKind::XorKeyword => "XOE",
175            TokenKind::Ascending => "ASC",
176            TokenKind::Descending => "DESC",
177            TokenKind::Symmetric => "SYMMETRIC",
178            TokenKind::Asymmetric => "ASYMMETRIC",
179            TokenKind::Window => "WINDOW",
180            TokenKind::Over => "OVER",
181            TokenKind::Partition => "PARTITION",
182            TokenKind::Nulls => "NULLS",
183            TokenKind::First => "FIRST",
184            TokenKind::Last => "LAST",
185            TokenKind::Interval => "INTERVAL",
186
187            // Values
188            TokenKind::Symbol(literal) => literal,
189            TokenKind::GlobalVariable(literal) => literal,
190            TokenKind::String(string) => string,
191            TokenKind::Integer(integer) => &integer.to_string(),
192            TokenKind::Float(float) => &float.to_string(),
193            TokenKind::True => "True",
194            TokenKind::False => "False",
195            TokenKind::Null => "Null",
196            TokenKind::Infinity => "Infinity",
197            TokenKind::NaN => "NaN",
198
199            // Group Operators
200            TokenKind::All => "ALL",
201            TokenKind::Some => "Some",
202            TokenKind::Any => "Any",
203
204            // Others
205            TokenKind::Greater => ">",
206            TokenKind::GreaterEqual => ">=",
207            TokenKind::Less => "<",
208            TokenKind::LessEqual => "<=",
209            TokenKind::Equal => "=",
210            TokenKind::Bang => "!",
211            TokenKind::BangEqual => "!=",
212            TokenKind::NullSafeEqual => "<=>",
213            TokenKind::AtRightArrow => "@>",
214            TokenKind::ArrowRightAt => "<@",
215            TokenKind::LeftParen => "(",
216            TokenKind::RightParen => ")",
217            TokenKind::LeftBracket => "[",
218            TokenKind::RightBracket => "]",
219            TokenKind::OrOr => "||",
220            TokenKind::AndAnd => "&&",
221            TokenKind::BitwiseNot => "~",
222            TokenKind::BitwiseXor => "#",
223            TokenKind::BitwiseOr => "|",
224            TokenKind::BitwiseAnd => "&",
225            TokenKind::BitwiseRightShift => ">>",
226            TokenKind::BitwiseLeftShift => "<<",
227            TokenKind::Colon => ":",
228            TokenKind::ColonColon => "::",
229            TokenKind::ColonEqual => ":=",
230            TokenKind::Plus => "+",
231            TokenKind::Minus => "-",
232            TokenKind::Star => "*",
233            TokenKind::Slash => "/",
234            TokenKind::Percentage => "%",
235            TokenKind::Caret => "^",
236            TokenKind::Comma => ",",
237            TokenKind::Dot => ".",
238            TokenKind::Semicolon => ";",
239        };
240        f.write_str(literal)
241    }
242}
243
244#[derive(Copy, Clone)]
245pub struct SourceLocation {
246    pub line_start: u32,
247    pub line_end: u32,
248    pub column_start: u32,
249    pub column_end: u32,
250}
251
252impl SourceLocation {
253    pub fn new(
254        line_start: u32,
255        line_end: u32,
256        column_start: u32,
257        column_end: u32,
258    ) -> SourceLocation {
259        SourceLocation {
260            line_start,
261            line_end,
262            column_start,
263            column_end,
264        }
265    }
266
267    pub fn expand_until(&mut self, location: SourceLocation) {
268        self.column_end = location.column_end;
269        self.line_end = location.line_end;
270    }
271}
272
273impl Display for SourceLocation {
274    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
275        f.write_fmt(format_args!(
276            "Loc(L {}:{}, C {}:{})",
277            self.line_start, self.line_end, self.column_start, self.column_end
278        ))
279    }
280}
281
282pub struct Token {
283    pub kind: TokenKind,
284    pub location: SourceLocation,
285}
286
287impl Token {
288    pub fn new(kind: TokenKind, location: SourceLocation) -> Token {
289        Token { kind, location }
290    }
291
292    pub fn new_symbol(symbol: String, location: SourceLocation) -> Token {
293        Token {
294            kind: resolve_symbol_kind(symbol),
295            location,
296        }
297    }
298
299    pub fn has_kind(&self, kind: TokenKind) -> bool {
300        self.kind == kind
301    }
302}
303
304impl Display for Token {
305    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
306        f.write_str(&self.kind.to_string())
307    }
308}
309
310fn resolve_symbol_kind(symbol: String) -> TokenKind {
311    match symbol.to_lowercase().as_str() {
312        // Reserved keywords
313        "do" => TokenKind::Do,
314        "set" => TokenKind::Set,
315        "select" => TokenKind::Select,
316        "distinct" => TokenKind::Distinct,
317        "from" => TokenKind::From,
318        "where" => TokenKind::Where,
319        "limit" => TokenKind::Limit,
320        "offset" => TokenKind::Offset,
321        "order" => TokenKind::Order,
322        "using" => TokenKind::Using,
323        "case" => TokenKind::Case,
324        "when" => TokenKind::When,
325        "then" => TokenKind::Then,
326        "else" => TokenKind::Else,
327        "end" => TokenKind::End,
328        "between" => TokenKind::Between,
329        "in" => TokenKind::In,
330        "is" => TokenKind::Is,
331        "on" => TokenKind::On,
332        "not" => TokenKind::Not,
333        "like" => TokenKind::Like,
334        "glob" => TokenKind::Glob,
335        "describe" => TokenKind::Describe,
336        "show" => TokenKind::Show,
337        "regexp" => TokenKind::RegExp,
338
339        "cast" => TokenKind::Cast,
340        "benchmark" => TokenKind::Benchmark,
341
342        "interval" => TokenKind::Interval,
343
344        // Select into
345        "into" => TokenKind::Into,
346        "outfile" => TokenKind::Outfile,
347        "dumpfile" => TokenKind::Dumpfile,
348        "lines" => TokenKind::Lines,
349        "fields" => TokenKind::Fields,
350        "enclosed" => TokenKind::Enclosed,
351        "terminated" => TokenKind::Terminated,
352
353        // Joins
354        "join" => TokenKind::Join,
355        "left" => TokenKind::Left,
356        "right" => TokenKind::Right,
357        "cross" => TokenKind::Cross,
358        "inner" => TokenKind::Inner,
359        "outer" => TokenKind::Outer,
360
361        // Grouping
362        "group" => TokenKind::Group,
363        "by" => TokenKind::By,
364        "having" => TokenKind::Having,
365        "with" => TokenKind::With,
366        "rollup" => TokenKind::Rollup,
367
368        // Between kind
369        "symmetric" => TokenKind::Symmetric,
370        "asymmetric" => TokenKind::Asymmetric,
371
372        // Integer division and Modulo operator
373        "div" => TokenKind::Slash,
374        "mod" => TokenKind::Percentage,
375
376        // Logical Operators
377        "or" => TokenKind::OrKeyword,
378        "and" => TokenKind::AndKeyword,
379        "xor" => TokenKind::XorKeyword,
380
381        // Group Operators
382        "all" => TokenKind::All,
383        "some" => TokenKind::Some,
384        "any" => TokenKind::Any,
385
386        // True, False and Null
387        "true" => TokenKind::True,
388        "false" => TokenKind::False,
389        "null" => TokenKind::Null,
390        "nulls" => TokenKind::Nulls,
391
392        // Infinity and NaN
393        "infinity" => TokenKind::Infinity,
394        "nan" => TokenKind::NaN,
395
396        // As for alias
397        "as" => TokenKind::As,
398
399        // Order by DES and ASC
400        "asc" => TokenKind::Ascending,
401        "desc" => TokenKind::Descending,
402
403        // Order by null ordering policy Null first and last
404        "first" => TokenKind::First,
405        "last" => TokenKind::Last,
406
407        // Array data type
408        "array" => TokenKind::Array,
409
410        // Over
411        "window" => TokenKind::Window,
412        "over" => TokenKind::Over,
413        "partition" => TokenKind::Partition,
414
415        // Identifier
416        _ => TokenKind::Symbol(symbol),
417    }
418}
gitql_parser/token.rs

gitql_parser/
token.rs
