gitql_parser/
token.rs

1use std::fmt::Display;
2use std::fmt::Formatter;
3use std::fmt::Result;
4
5#[derive(PartialEq)]
6pub enum TokenKind {
7    Do,
8    Set,
9    Select,
10    Distinct,
11    From,
12    Group,
13    Where,
14    Having,
15    Qualify,
16    Limit,
17    Offset,
18    Order,
19    Using,
20    Like,
21    Glob,
22    Describe,
23    Show,
24    RegExp,
25    Array,
26    Cast,
27    Benchmark,
28    Join,
29    Left,
30    Right,
31    Cross,
32    Inner,
33    Outer,
34    Case,
35    When,
36    Then,
37    Else,
38    End,
39    Into,
40    Outfile,
41    Dumpfile,
42    Lines,
43    Fields,
44    Enclosed,
45    Terminated,
46    Between,
47    By,
48    In,
49    Is,
50    On,
51    Not,
52    As,
53    With,
54    Rollup,
55    OrKeyword,
56    AndKeyword,
57    XorKeyword,
58    Ascending,
59    Descending,
60    Symmetric,
61    Asymmetric,
62    Window,
63    Over,
64    Partition,
65    First,
66    Last,
67    Interval,
68
69    // Values
70    Row,
71    Symbol(String),
72    GlobalVariable(String),
73    String(String),
74    Integer(i64),
75    Float(f64),
76    True,
77    False,
78    Null,
79    Nulls,
80    Infinity,
81    NaN,
82
83    All,
84    Some,
85    Any,
86
87    Greater,
88    GreaterEqual,
89    Less,
90    LessEqual,
91    Equal,
92    Bang,
93    BangEqual,
94    NullSafeEqual,
95    AtRightArrow,
96    ArrowRightAt,
97    LeftParen,
98    RightParen,
99    LeftBracket,
100    RightBracket,
101    OrOr,
102    AndAnd,
103    BitwiseNot,
104    BitwiseXor,
105    BitwiseOr,
106    BitwiseAnd,
107    BitwiseRightShift,
108    BitwiseLeftShift,
109    Colon,
110    ColonColon,
111    ColonEqual,
112    Plus,
113    Minus,
114    Star,
115    Slash,
116    Percentage,
117    Caret,
118    Comma,
119    Dot,
120    Semicolon,
121}
122
123impl Display for TokenKind {
124    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
125        let literal = match self {
126            // Reserved Keywords
127            TokenKind::Do => "DO",
128            TokenKind::Set => "SET",
129            TokenKind::Select => "SELECT",
130            TokenKind::Distinct => "DISTINCT",
131            TokenKind::From => "FROM",
132            TokenKind::Group => "GROUP",
133            TokenKind::Where => "WHERE",
134            TokenKind::Having => "HAVING",
135            TokenKind::Qualify => "QUALIFY",
136            TokenKind::Limit => "LIMIT",
137            TokenKind::Offset => "OFFSET",
138            TokenKind::Order => "ORDER",
139            TokenKind::Using => "USING",
140            TokenKind::Like => "LIKE",
141            TokenKind::Glob => "GLOB",
142            TokenKind::Describe => "DESCRIBE",
143            TokenKind::Show => "SHOW",
144            TokenKind::RegExp => "REGEXP",
145            TokenKind::Array => "ARRAY",
146            TokenKind::Cast => "CAST",
147            TokenKind::Benchmark => "BENCHMARK",
148            TokenKind::Join => "JOIN",
149            TokenKind::Left => "LEFT",
150            TokenKind::Right => "RIGHT",
151            TokenKind::Cross => "CROSS",
152            TokenKind::Inner => "INNER",
153            TokenKind::Outer => "OUTER",
154            TokenKind::Case => "CASE",
155            TokenKind::When => "WHEN",
156            TokenKind::Then => "THEN",
157            TokenKind::Else => "ELSE",
158            TokenKind::End => "END",
159            TokenKind::Into => "INTO",
160            TokenKind::Outfile => "OUTFILE",
161            TokenKind::Dumpfile => "DUMPFILE",
162            TokenKind::Lines => "LINES",
163            TokenKind::Fields => "FIELDS",
164            TokenKind::Enclosed => "ENCLOSED",
165            TokenKind::Terminated => "TERMINATED",
166            TokenKind::Between => "BETWEEN",
167            TokenKind::By => "BY",
168            TokenKind::In => "IN",
169            TokenKind::Is => "IS",
170            TokenKind::On => "ON",
171            TokenKind::Not => "NOT",
172            TokenKind::As => "AS",
173            TokenKind::With => "WITH",
174            TokenKind::Rollup => "ROLLUP",
175            TokenKind::OrKeyword => "OR",
176            TokenKind::AndKeyword => "AND",
177            TokenKind::XorKeyword => "XOE",
178            TokenKind::Ascending => "ASC",
179            TokenKind::Descending => "DESC",
180            TokenKind::Symmetric => "SYMMETRIC",
181            TokenKind::Asymmetric => "ASYMMETRIC",
182            TokenKind::Window => "WINDOW",
183            TokenKind::Over => "OVER",
184            TokenKind::Partition => "PARTITION",
185            TokenKind::Nulls => "NULLS",
186            TokenKind::First => "FIRST",
187            TokenKind::Last => "LAST",
188            TokenKind::Interval => "INTERVAL",
189
190            // Values
191            TokenKind::Row => "Row",
192            TokenKind::Symbol(literal) => literal,
193            TokenKind::GlobalVariable(literal) => literal,
194            TokenKind::String(string) => string,
195            TokenKind::Integer(integer) => &integer.to_string(),
196            TokenKind::Float(float) => &float.to_string(),
197            TokenKind::True => "True",
198            TokenKind::False => "False",
199            TokenKind::Null => "Null",
200            TokenKind::Infinity => "Infinity",
201            TokenKind::NaN => "NaN",
202
203            // Group Operators
204            TokenKind::All => "ALL",
205            TokenKind::Some => "Some",
206            TokenKind::Any => "Any",
207
208            // Others
209            TokenKind::Greater => ">",
210            TokenKind::GreaterEqual => ">=",
211            TokenKind::Less => "<",
212            TokenKind::LessEqual => "<=",
213            TokenKind::Equal => "=",
214            TokenKind::Bang => "!",
215            TokenKind::BangEqual => "!=",
216            TokenKind::NullSafeEqual => "<=>",
217            TokenKind::AtRightArrow => "@>",
218            TokenKind::ArrowRightAt => "<@",
219            TokenKind::LeftParen => "(",
220            TokenKind::RightParen => ")",
221            TokenKind::LeftBracket => "[",
222            TokenKind::RightBracket => "]",
223            TokenKind::OrOr => "||",
224            TokenKind::AndAnd => "&&",
225            TokenKind::BitwiseNot => "~",
226            TokenKind::BitwiseXor => "#",
227            TokenKind::BitwiseOr => "|",
228            TokenKind::BitwiseAnd => "&",
229            TokenKind::BitwiseRightShift => ">>",
230            TokenKind::BitwiseLeftShift => "<<",
231            TokenKind::Colon => ":",
232            TokenKind::ColonColon => "::",
233            TokenKind::ColonEqual => ":=",
234            TokenKind::Plus => "+",
235            TokenKind::Minus => "-",
236            TokenKind::Star => "*",
237            TokenKind::Slash => "/",
238            TokenKind::Percentage => "%",
239            TokenKind::Caret => "^",
240            TokenKind::Comma => ",",
241            TokenKind::Dot => ".",
242            TokenKind::Semicolon => ";",
243        };
244        f.write_str(literal)
245    }
246}
247
248#[derive(Copy, Clone)]
249pub struct SourceLocation {
250    pub line_start: u32,
251    pub line_end: u32,
252    pub column_start: u32,
253    pub column_end: u32,
254}
255
256impl SourceLocation {
257    pub fn new(
258        line_start: u32,
259        line_end: u32,
260        column_start: u32,
261        column_end: u32,
262    ) -> SourceLocation {
263        SourceLocation {
264            line_start,
265            line_end,
266            column_start,
267            column_end,
268        }
269    }
270
271    pub fn expand_until(&mut self, location: SourceLocation) {
272        self.column_end = location.column_end;
273        self.line_end = location.line_end;
274    }
275}
276
277impl Display for SourceLocation {
278    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
279        f.write_fmt(format_args!(
280            "Loc(L {}:{}, C {}:{})",
281            self.line_start, self.line_end, self.column_start, self.column_end
282        ))
283    }
284}
285
286pub struct Token {
287    pub kind: TokenKind,
288    pub location: SourceLocation,
289}
290
291impl Token {
292    pub fn new(kind: TokenKind, location: SourceLocation) -> Token {
293        Token { kind, location }
294    }
295
296    pub fn new_symbol(symbol: String, location: SourceLocation) -> Token {
297        Token {
298            kind: resolve_symbol_kind(symbol),
299            location,
300        }
301    }
302
303    pub fn has_kind(&self, kind: TokenKind) -> bool {
304        self.kind == kind
305    }
306}
307
308impl Display for Token {
309    fn fmt(&self, f: &mut Formatter<'_>) -> Result {
310        f.write_str(&self.kind.to_string())
311    }
312}
313
314fn resolve_symbol_kind(symbol: String) -> TokenKind {
315    match symbol.to_lowercase().as_str() {
316        // Reserved keywords
317        "do" => TokenKind::Do,
318        "set" => TokenKind::Set,
319        "select" => TokenKind::Select,
320        "distinct" => TokenKind::Distinct,
321        "from" => TokenKind::From,
322        "where" => TokenKind::Where,
323        "qualify" => TokenKind::Qualify,
324        "limit" => TokenKind::Limit,
325        "offset" => TokenKind::Offset,
326        "order" => TokenKind::Order,
327        "using" => TokenKind::Using,
328        "case" => TokenKind::Case,
329        "when" => TokenKind::When,
330        "then" => TokenKind::Then,
331        "else" => TokenKind::Else,
332        "end" => TokenKind::End,
333        "between" => TokenKind::Between,
334        "in" => TokenKind::In,
335        "is" => TokenKind::Is,
336        "on" => TokenKind::On,
337        "not" => TokenKind::Not,
338        "like" => TokenKind::Like,
339        "glob" => TokenKind::Glob,
340        "describe" => TokenKind::Describe,
341        "show" => TokenKind::Show,
342        "regexp" => TokenKind::RegExp,
343
344        "cast" => TokenKind::Cast,
345        "benchmark" => TokenKind::Benchmark,
346
347        "interval" => TokenKind::Interval,
348
349        // Select into
350        "into" => TokenKind::Into,
351        "outfile" => TokenKind::Outfile,
352        "dumpfile" => TokenKind::Dumpfile,
353        "lines" => TokenKind::Lines,
354        "fields" => TokenKind::Fields,
355        "enclosed" => TokenKind::Enclosed,
356        "terminated" => TokenKind::Terminated,
357
358        // Joins
359        "join" => TokenKind::Join,
360        "left" => TokenKind::Left,
361        "right" => TokenKind::Right,
362        "cross" => TokenKind::Cross,
363        "inner" => TokenKind::Inner,
364        "outer" => TokenKind::Outer,
365
366        // Grouping
367        "group" => TokenKind::Group,
368        "by" => TokenKind::By,
369        "having" => TokenKind::Having,
370        "with" => TokenKind::With,
371        "rollup" => TokenKind::Rollup,
372
373        // Between kind
374        "symmetric" => TokenKind::Symmetric,
375        "asymmetric" => TokenKind::Asymmetric,
376
377        // Integer division and Modulo operator
378        "div" => TokenKind::Slash,
379        "mod" => TokenKind::Percentage,
380
381        // Logical Operators
382        "or" => TokenKind::OrKeyword,
383        "and" => TokenKind::AndKeyword,
384        "xor" => TokenKind::XorKeyword,
385
386        // Group Operators
387        "all" => TokenKind::All,
388        "some" => TokenKind::Some,
389        "any" => TokenKind::Any,
390
391        "row" => TokenKind::Row,
392
393        // True, False and Null
394        "true" => TokenKind::True,
395        "false" => TokenKind::False,
396        "null" => TokenKind::Null,
397        "nulls" => TokenKind::Nulls,
398
399        // Infinity and NaN
400        "infinity" => TokenKind::Infinity,
401        "inf" => TokenKind::Infinity,
402        "nan" => TokenKind::NaN,
403
404        // As for alias
405        "as" => TokenKind::As,
406
407        // Order by DES and ASC
408        "asc" => TokenKind::Ascending,
409        "desc" => TokenKind::Descending,
410
411        // Order by null ordering policy Null first and last
412        "first" => TokenKind::First,
413        "last" => TokenKind::Last,
414
415        // Array data type
416        "array" => TokenKind::Array,
417
418        // Over
419        "window" => TokenKind::Window,
420        "over" => TokenKind::Over,
421        "partition" => TokenKind::Partition,
422
423        // Identifier
424        _ => TokenKind::Symbol(symbol),
425    }
426}
gitql_parser/token.rs

gitql_parser/
token.rs
