databend_common_ast/parser/
token.rs

1// Copyright 2021 Datafuse Labs
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use logos::Lexer;
16use logos::Logos;
17use strum::IntoEnumIterator;
18use strum_macros::EnumIter;
19
20pub use self::TokenKind::*;
21use crate::ParseError;
22use crate::Range;
23use crate::Result;
24
25#[derive(Clone, PartialEq, Eq)]
26pub struct Token<'a> {
27    pub source: &'a str,
28    pub kind: TokenKind,
29    pub span: Range,
30}
31
32impl<'a> Token<'a> {
33    pub fn new_eoi(source: &'a str) -> Self {
34        Token {
35            source,
36            kind: TokenKind::EOI,
37            span: (source.len()..source.len()).into(),
38        }
39    }
40
41    pub fn text(&self) -> &'a str {
42        &self.source[std::ops::Range::from(self.span)]
43    }
44}
45
46impl std::fmt::Debug for Token<'_> {
47    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
48        write!(f, "{:?}({:?})", self.kind, self.span)
49    }
50}
51
52pub struct Tokenizer<'a> {
53    source: &'a str,
54    lexer: Lexer<'a, TokenKind>,
55    prev_token: Option<TokenKind>,
56    eoi: bool,
57}
58
59impl<'a> Tokenizer<'a> {
60    pub fn new(source: &'a str) -> Self {
61        Tokenizer {
62            source,
63            lexer: TokenKind::lexer(source),
64            eoi: false,
65            prev_token: None,
66        }
67    }
68
69    pub fn contains_token(query: &str, target_kind: TokenKind) -> bool {
70        let mut tokenizer = Tokenizer::new(query);
71        while let Some(Ok(token)) = tokenizer.next() {
72            if token.kind == target_kind {
73                return true;
74            }
75        }
76        false
77    }
78}
79
80impl<'a> Iterator for Tokenizer<'a> {
81    type Item = Result<Token<'a>>;
82
83    fn next(&mut self) -> Option<Self::Item> {
84        match self.lexer.next() {
85            Some(TokenKind::Error) => {
86                let span = Some((self.lexer.span().start..self.source.len()).into());
87                Some(Err(ParseError(
88                    span,
89                    "unable to recognize the rest tokens".to_string(),
90                )))
91            }
92            Some(kind) => {
93                // Skip hint-like comment that is in the invalid position.
94                if !matches!(
95                    self.prev_token,
96                    Some(
97                        TokenKind::INSERT
98                            | TokenKind::SELECT
99                            | TokenKind::REPLACE
100                            | TokenKind::UPDATE
101                            | TokenKind::DELETE
102                            | TokenKind::COPY
103                    )
104                ) && kind == TokenKind::HintPrefix
105                {
106                    loop {
107                        match self.next() {
108                            // Hint-like comment ended. Return the next token.
109                            Some(Ok(token)) if token.kind == TokenKind::HintSuffix => {
110                                return self.next();
111                            }
112                            // Do not skip EOI.
113                            Some(Ok(token)) if token.kind == TokenKind::EOI => {
114                                return Some(Ok(token));
115                            }
116                            // In the comment, skip the contents.
117                            Some(Ok(_)) => continue,
118                            Some(Err(err)) => return Some(Err(err)),
119                            None => return None,
120                        }
121                    }
122                }
123                self.prev_token = Some(kind);
124                Some(Ok(Token {
125                    source: self.source,
126                    kind,
127                    span: self.lexer.span().into(),
128                }))
129            }
130            None if !self.eoi => {
131                self.eoi = true;
132                Some(Ok(Token::new_eoi(self.source)))
133            }
134            None => None,
135        }
136    }
137}
138
139#[allow(non_camel_case_types)]
140#[derive(Logos, EnumIter, Clone, Copy, Debug, PartialEq, Eq, Hash)]
141pub enum TokenKind {
142    #[error]
143    Error,
144
145    EOI,
146
147    #[regex(r"[ \t\r\n\f]+", logos::skip)]
148    Whitespace,
149
150    #[regex(r"--[^\n\f]*", logos::skip)]
151    Comment,
152
153    #[regex(r"/\*[^\+]([^\*]|(\*[^/]))*\*/", logos::skip)]
154    CommentBlock,
155
156    #[regex(r#"[_a-zA-Z][_$a-zA-Z0-9]*"#)]
157    Ident,
158
159    #[regex(r#"\$[_a-zA-Z][_$a-zA-Z0-9]*"#)]
160    IdentVariable,
161
162    #[regex(r#"\$[0-9]+"#)]
163    ColumnPosition,
164
165    #[regex(r#"`[^`]*`"#)]
166    #[regex(r#""([^"\\]|\\.|"")*""#)]
167    #[regex(r#"'([^'\\]|\\.|'')*'"#)]
168    LiteralString,
169
170    #[regex(r#"\$\$([^\$]|(\$[^\$]))*\$\$"#)]
171    LiteralCodeString,
172
173    #[regex(r#"@([^\s`;'"()]|\\\s|\\'|\\"|\\\\)+"#)]
174    LiteralAtString,
175
176    #[regex(r"[xX]'[a-fA-F0-9]*'")]
177    PGLiteralHex,
178    #[regex(r"0[xX][a-fA-F0-9]+")]
179    MySQLLiteralHex,
180
181    #[regex(r"[0-9]+(_|[0-9])*")]
182    LiteralInteger,
183
184    #[regex(r"[0-9]+[eE][+-]?[0-9]+")]
185    #[regex(r"([0-9]*\.[0-9]+([eE][+-]?[0-9]+)?)|([0-9]+\.[0-9]*([eE][+-]?[0-9]+)?)")]
186    LiteralFloat,
187
188    // Symbols
189    #[token("/*+")]
190    HintPrefix,
191    #[token("*/")]
192    HintSuffix,
193    #[token("==")]
194    DoubleEq,
195    #[token("=")]
196    Eq,
197    #[token("<>")]
198    #[token("!=")]
199    NotEq,
200    #[token("<")]
201    Lt,
202    #[token(">")]
203    Gt,
204    #[token("<=")]
205    Lte,
206    #[token(">=")]
207    Gte,
208    #[token("<=>")]
209    Spaceship,
210    #[token("+")]
211    Plus,
212    #[token("-")]
213    Minus,
214    #[token("*")]
215    Multiply,
216    #[token("/")]
217    Divide,
218    #[token("//")]
219    IntDiv,
220    #[token("%")]
221    Modulo,
222    #[token("||")]
223    StringConcat,
224    #[token("(")]
225    LParen,
226    #[token(")")]
227    RParen,
228    #[token(",")]
229    Comma,
230    #[token(".")]
231    Dot,
232    #[token(":")]
233    Colon,
234    #[token("::")]
235    DoubleColon,
236    #[token(":=")]
237    ColonEqual,
238    #[token(";")]
239    SemiColon,
240    #[token("\\")]
241    Backslash,
242    #[token("[")]
243    LBracket,
244    #[token("]")]
245    RBracket,
246    #[token("^")]
247    Caret,
248    #[token("{")]
249    LBrace,
250    #[token("}")]
251    RBrace,
252    #[token("$")]
253    Dollar,
254    #[token("->")]
255    RArrow,
256    #[token("->>")]
257    LongRArrow,
258    #[token("=>")]
259    FatRArrow,
260    #[token("#>")]
261    HashRArrow,
262    #[token("#>>")]
263    HashLongRArrow,
264    /// A case insensitive match regular expression operator in PostgreSQL
265    #[token("~*")]
266    TildeAsterisk,
267    /// A case sensitive not match regular expression operator in PostgreSQL
268    #[token("!*")]
269    ExclamationMarkTilde,
270    /// A case insensitive not match regular expression operator in PostgreSQL
271    #[token("!~*")]
272    ExclamationMarkTildeAsterisk,
273    /// A bitwise and operator in PostgreSQL
274    #[token("&")]
275    BitWiseAnd,
276    /// A bitwise or operator in PostgreSQL
277    #[token("|")]
278    BitWiseOr,
279    /// A bitwise xor operator in PostgreSQL
280    #[token("#")]
281    BitWiseXor,
282    /// A bitwise not operator in PostgreSQL
283    #[token("~")]
284    BitWiseNot,
285    /// A bitwise shift left operator in PostgreSQL
286    #[token("<<")]
287    ShiftLeft,
288    /// A bitwise shift right operator in PostgreSQL
289    #[token(">>")]
290    ShiftRight,
291    /// Exclamation Mark `!` used for PostgreSQL factorial operator
292    #[token("!")]
293    Factorial,
294    /// Double Exclamation Mark `!!` used for PostgreSQL prefix factorial operator
295    #[token("!!")]
296    DoubleExclamationMark,
297    /// AtSign `@` used for PostgreSQL abs operator
298    #[token("@")]
299    Abs,
300    /// A square root math operator in PostgreSQL
301    #[token("|/")]
302    SquareRoot,
303    /// A cube root math operator in PostgreSQL
304    #[token("||/")]
305    CubeRoot,
306    /// Placeholder used in prepared stmt
307    /// Also used as JSON operator.
308    #[token("?")]
309    Placeholder,
310    /// Used as JSON operator.
311    #[token("?|")]
312    QuestionOr,
313    /// Used as JSON operator.
314    #[token("?&")]
315    QuestionAnd,
316    /// Used as JSON operator.
317    #[token("<@")]
318    ArrowAt,
319    /// Used as JSON operator.
320    #[token("@>")]
321    AtArrow,
322    /// Used as JSON operator.
323    #[token("@?")]
324    AtQuestion,
325    /// Used as JSON operator.
326    #[token("@@")]
327    AtAt,
328    /// Used as JSON operator.
329    #[token("#-")]
330    HashMinus,
331
332    // Keywords
333    //
334    // Steps to add keyword:
335    // 1. Add the keyword to token kind variants by alphabetical order.
336    // 2. Search in this file to see if the new keyword is a commented out reserved keyword. If
337    //    so, uncomment the keyword in the reserved list.
338    #[token("ACCOUNT", ignore(ascii_case))]
339    ACCOUNT,
340    #[token("ALL", ignore(ascii_case))]
341    ALL,
342    #[token("ALLOWED_IP_LIST", ignore(ascii_case))]
343    ALLOWED_IP_LIST,
344    #[token("ADD", ignore(ascii_case))]
345    ADD,
346    #[token("AFTER", ignore(ascii_case))]
347    AFTER,
348    #[token("AGGREGATING", ignore(ascii_case))]
349    AGGREGATING,
350    #[token("ANY", ignore(ascii_case))]
351    ANY,
352    #[token("APPEND_ONLY", ignore(ascii_case))]
353    APPEND_ONLY,
354    #[token("ARGS", ignore(ascii_case))]
355    ARGS,
356    #[token("AUTO", ignore(ascii_case))]
357    AUTO,
358    #[token("SOME", ignore(ascii_case))]
359    SOME,
360    #[token("ALTER", ignore(ascii_case))]
361    ALTER,
362    #[token("ALWAYS", ignore(ascii_case))]
363    ALWAYS,
364    #[token("ANALYZE", ignore(ascii_case))]
365    ANALYZE,
366    #[token("AND", ignore(ascii_case))]
367    AND,
368    #[token("ARRAY", ignore(ascii_case))]
369    ARRAY,
370    #[token("AS", ignore(ascii_case))]
371    AS,
372    #[token("ASOF", ignore(ascii_case))]
373    ASOF,
374    #[token("AST", ignore(ascii_case))]
375    AST,
376    #[token("AT", ignore(ascii_case))]
377    AT,
378    #[token("ASC", ignore(ascii_case))]
379    ASC,
380    #[token("ANTI", ignore(ascii_case))]
381    ANTI,
382    #[token("ASYNC", ignore(ascii_case))]
383    ASYNC,
384    #[token("ATTACH", ignore(ascii_case))]
385    ATTACH,
386    #[token("AVRO", ignore(ascii_case))]
387    AVRO,
388    #[token("BEFORE", ignore(ascii_case))]
389    BEFORE,
390    #[token("BETWEEN", ignore(ascii_case))]
391    BETWEEN,
392    #[token("BIGINT", ignore(ascii_case))]
393    BIGINT,
394    #[token("BINARY", ignore(ascii_case))]
395    BINARY,
396    #[token("BREAK", ignore(ascii_case))]
397    BREAK,
398    #[token("LONGBLOB", ignore(ascii_case))]
399    LONGBLOB,
400    #[token("MEDIUMBLOB", ignore(ascii_case))]
401    MEDIUMBLOB,
402    #[token("TINYBLOB", ignore(ascii_case))]
403    TINYBLOB,
404    #[token("BLOB", ignore(ascii_case))]
405    BLOB,
406    #[token("BINARY_FORMAT", ignore(ascii_case))]
407    BINARY_FORMAT,
408    #[token("BITMAP", ignore(ascii_case))]
409    BITMAP,
410    #[token("BLOCKED_IP_LIST", ignore(ascii_case))]
411    BLOCKED_IP_LIST,
412    #[token("BOOL", ignore(ascii_case))]
413    BOOL,
414    #[token("BOOLEAN", ignore(ascii_case))]
415    BOOLEAN,
416    #[token("BOTH", ignore(ascii_case))]
417    BOTH,
418    #[token("BY", ignore(ascii_case))]
419    BY,
420    #[token("BROTLI", ignore(ascii_case))]
421    BROTLI,
422    #[token("BZ2", ignore(ascii_case))]
423    BZ2,
424    #[token("BLOCK", ignore(ascii_case))]
425    BLOCK,
426    #[token("CALL", ignore(ascii_case))]
427    CALL,
428    #[token("CASE", ignore(ascii_case))]
429    CASE,
430    #[token("CASE_SENSITIVE", ignore(ascii_case))]
431    CASE_SENSITIVE,
432    #[token("CAST", ignore(ascii_case))]
433    CAST,
434    #[token("CATALOG", ignore(ascii_case))]
435    CATALOG,
436    #[token("CATALOGS", ignore(ascii_case))]
437    CATALOGS,
438    #[token("CENTURY", ignore(ascii_case))]
439    CENTURY,
440    #[token("CHANGES", ignore(ascii_case))]
441    CHANGES,
442    #[token("CLUSTER", ignore(ascii_case))]
443    CLUSTER,
444    #[token("COMMENT", ignore(ascii_case))]
445    COMMENT,
446    #[token("COMMENTS", ignore(ascii_case))]
447    COMMENTS,
448    #[token("COMPACT", ignore(ascii_case))]
449    COMPACT,
450    #[token("CONNECTION", ignore(ascii_case))]
451    CONNECTION,
452    #[token("CONNECTIONS", ignore(ascii_case))]
453    CONNECTIONS,
454    #[token("CONSUME", ignore(ascii_case))]
455    CONSUME,
456    #[token("CONTENT_TYPE", ignore(ascii_case))]
457    CONTENT_TYPE,
458    #[token("CONTINUE", ignore(ascii_case))]
459    CONTINUE,
460    #[token("CHAR", ignore(ascii_case))]
461    CHAR,
462    #[token("COLUMN", ignore(ascii_case))]
463    COLUMN,
464    #[token("CACHE", ignore(ascii_case))]
465    CACHE,
466    #[token("COLUMN_MATCH_MODE", ignore(ascii_case))]
467    COLUMN_MATCH_MODE,
468    #[token("COLUMNS", ignore(ascii_case))]
469    COLUMNS,
470    #[token("CHARACTER", ignore(ascii_case))]
471    CHARACTER,
472    #[token("CONFLICT", ignore(ascii_case))]
473    CONFLICT,
474    #[token("COMPRESSION", ignore(ascii_case))]
475    COMPRESSION,
476    #[token("COPY_OPTIONS", ignore(ascii_case))]
477    COPY_OPTIONS,
478    #[token("COPY", ignore(ascii_case))]
479    COPY,
480    #[token("COUNT", ignore(ascii_case))]
481    COUNT,
482    #[token("CREDENTIAL", ignore(ascii_case))]
483    CREDENTIAL,
484    #[token("CREATE", ignore(ascii_case))]
485    CREATE,
486    #[token("CROSS", ignore(ascii_case))]
487    CROSS,
488    #[token("CSV", ignore(ascii_case))]
489    CSV,
490    #[token("CURRENT", ignore(ascii_case))]
491    CURRENT,
492    #[token("CURRENT_TIMESTAMP", ignore(ascii_case))]
493    CURRENT_TIMESTAMP,
494    #[token("DATABASE", ignore(ascii_case))]
495    DATABASE,
496    #[token("DATABASES", ignore(ascii_case))]
497    DATABASES,
498    #[token("DATA", ignore(ascii_case))]
499    DATA,
500    #[token("DATE", ignore(ascii_case))]
501    DATE,
502    #[token("DATE_ADD", ignore(ascii_case))]
503    DATE_ADD,
504    #[token("DATEADD", ignore(ascii_case))]
505    DATEADD,
506    #[token("DATE_DIFF", ignore(ascii_case))]
507    DATE_DIFF,
508    #[token("DATEDIFF", ignore(ascii_case))]
509    DATEDIFF,
510    #[token("DATE_PART", ignore(ascii_case))]
511    DATE_PART,
512    #[token("DATEPART", ignore(ascii_case))]
513    DATEPART,
514    #[token("DATE_SUB", ignore(ascii_case))]
515    DATE_SUB,
516    #[token("DATE_BETWEEN", ignore(ascii_case))]
517    DATE_BETWEEN,
518    #[token("DATEBETWEEN", ignore(ascii_case))]
519    DATEBETWEEN,
520    #[token("DATESUB", ignore(ascii_case))]
521    DATESUB,
522    #[token("DATE_TRUNC", ignore(ascii_case))]
523    DATE_TRUNC,
524    #[token("TRUNC", ignore(ascii_case))]
525    TRUNC,
526    #[token("DATETIME", ignore(ascii_case))]
527    DATETIME,
528    #[token("DAY", ignore(ascii_case))]
529    DAY,
530    #[token("DECADE", ignore(ascii_case))]
531    DECADE,
532    #[token("DECIMAL", ignore(ascii_case))]
533    DECIMAL,
534    #[token("DECLARE", ignore(ascii_case))]
535    DECLARE,
536    #[token("DEFAULT", ignore(ascii_case))]
537    DEFAULT,
538    #[token("DEFLATE", ignore(ascii_case))]
539    DEFLATE,
540    #[token("DELETE", ignore(ascii_case))]
541    DELETE,
542    #[token("DESC", ignore(ascii_case))]
543    DESC,
544    #[token("DETAILED_OUTPUT", ignore(ascii_case))]
545    DETAILED_OUTPUT,
546    #[token("DESCRIBE", ignore(ascii_case))]
547    DESCRIBE,
548    #[token("DISABLE", ignore(ascii_case))]
549    DISABLE,
550    #[token("DISABLE_VARIANT_CHECK", ignore(ascii_case))]
551    DISABLE_VARIANT_CHECK,
552    #[token("DISTINCT", ignore(ascii_case))]
553    DISTINCT,
554    #[token("RESPECT", ignore(ascii_case))]
555    RESPECT,
556    #[token("IGNORE", ignore(ascii_case))]
557    IGNORE,
558    #[token("DIV", ignore(ascii_case))]
559    DIV,
560    #[token("DOUBLE_SHA1_PASSWORD", ignore(ascii_case))]
561    DOUBLE_SHA1_PASSWORD,
562    #[token("DO", ignore(ascii_case))]
563    DO,
564    #[token("DOUBLE", ignore(ascii_case))]
565    DOUBLE,
566    #[token("DOW", ignore(ascii_case))]
567    DOW,
568    #[token("WEEK", ignore(ascii_case))]
569    WEEK,
570    #[token("DELTA", ignore(ascii_case))]
571    DELTA,
572    #[token("DOY", ignore(ascii_case))]
573    DOY,
574    #[token("DOWNLOAD", ignore(ascii_case))]
575    DOWNLOAD,
576    #[token("DOWNSTREAM", ignore(ascii_case))]
577    DOWNSTREAM,
578    #[token("DROP", ignore(ascii_case))]
579    DROP,
580    #[token("DRY", ignore(ascii_case))]
581    DRY,
582    #[token("DYNAMIC", ignore(ascii_case))]
583    DYNAMIC,
584    #[token("EXCEPT", ignore(ascii_case))]
585    EXCEPT,
586    #[token("EXCLUDE", ignore(ascii_case))]
587    EXCLUDE,
588    #[token("ELSE", ignore(ascii_case))]
589    ELSE,
590    #[token("EMPTY_FIELD_AS", ignore(ascii_case))]
591    EMPTY_FIELD_AS,
592    #[token("ENABLE", ignore(ascii_case))]
593    ENABLE,
594    #[token("ENABLE_VIRTUAL_HOST_STYLE", ignore(ascii_case))]
595    ENABLE_VIRTUAL_HOST_STYLE,
596    #[token("END", ignore(ascii_case))]
597    END,
598    #[token("ENDPOINT", ignore(ascii_case))]
599    ENDPOINT,
600    #[token("ENGINE", ignore(ascii_case))]
601    ENGINE,
602    #[token("ENGINES", ignore(ascii_case))]
603    ENGINES,
604    #[token("EPOCH", ignore(ascii_case))]
605    EPOCH,
606    #[token("YEARWEEK", ignore(ascii_case))]
607    YEARWEEK,
608    #[token("MICROSECOND", ignore(ascii_case))]
609    MICROSECOND,
610    #[token("ERROR_ON_COLUMN_COUNT_MISMATCH", ignore(ascii_case))]
611    ERROR_ON_COLUMN_COUNT_MISMATCH,
612    #[token("ESCAPE", ignore(ascii_case))]
613    ESCAPE,
614    #[token("EXCEPTION_BACKTRACE", ignore(ascii_case))]
615    EXCEPTION_BACKTRACE,
616    #[token("EXISTS", ignore(ascii_case))]
617    EXISTS,
618    #[token("EXPLAIN", ignore(ascii_case))]
619    EXPLAIN,
620    #[token("EXPIRE", ignore(ascii_case))]
621    EXPIRE,
622    #[token("EXTRACT", ignore(ascii_case))]
623    EXTRACT,
624    #[token("ELSEIF", ignore(ascii_case))]
625    ELSEIF,
626    #[token("FALSE", ignore(ascii_case))]
627    FALSE,
628    #[token("FIELDS", ignore(ascii_case))]
629    FIELDS,
630    #[token("FIELD_DELIMITER", ignore(ascii_case))]
631    FIELD_DELIMITER,
632    #[token("NAN_DISPLAY", ignore(ascii_case))]
633    NAN_DISPLAY,
634    #[token("NULL_DISPLAY", ignore(ascii_case))]
635    NULL_DISPLAY,
636    #[token("NULL_IF", ignore(ascii_case))]
637    NULL_IF,
638    #[token("FILE_FORMAT", ignore(ascii_case))]
639    FILE_FORMAT,
640    #[token("FILE", ignore(ascii_case))]
641    FILE,
642    #[token("FILES", ignore(ascii_case))]
643    FILES,
644    #[token("FINAL", ignore(ascii_case))]
645    FINAL,
646    #[token("FLASHBACK", ignore(ascii_case))]
647    FLASHBACK,
648    #[token("FLOAT", ignore(ascii_case))]
649    FLOAT,
650    #[token("FLOAT32", ignore(ascii_case))]
651    FLOAT32,
652    #[token("FLOAT64", ignore(ascii_case))]
653    FLOAT64,
654    #[token("FOR", ignore(ascii_case))]
655    FOR,
656    #[token("FORCE", ignore(ascii_case))]
657    FORCE,
658    #[token("FORMAT", ignore(ascii_case))]
659    FORMAT,
660    #[token("FOLLOWING", ignore(ascii_case))]
661    FOLLOWING,
662    #[token("FORMAT_NAME", ignore(ascii_case))]
663    FORMAT_NAME,
664    #[token("FORMATS", ignore(ascii_case))]
665    FORMATS,
666    #[token("FRAGMENTS", ignore(ascii_case))]
667    FRAGMENTS,
668    #[token("FRIDAY", ignore(ascii_case))]
669    FRIDAY,
670    #[token("FROM", ignore(ascii_case))]
671    FROM,
672    #[token("FULL", ignore(ascii_case))]
673    FULL,
674    #[token("FUNCTION", ignore(ascii_case))]
675    FUNCTION,
676    #[token("FUNCTIONS", ignore(ascii_case))]
677    FUNCTIONS,
678    #[token("TABLE_FUNCTIONS", ignore(ascii_case))]
679    TABLE_FUNCTIONS,
680    #[token("SET_VAR", ignore(ascii_case))]
681    SET_VAR,
682    #[token("FUSE", ignore(ascii_case))]
683    FUSE,
684    #[token("GET", ignore(ascii_case))]
685    GET,
686    #[token("GENERATED", ignore(ascii_case))]
687    GENERATED,
688    #[token("GEOMETRY", ignore(ascii_case))]
689    GEOMETRY,
690    #[token("GEOGRAPHY", ignore(ascii_case))]
691    GEOGRAPHY,
692    #[token("GLOBAL", ignore(ascii_case))]
693    GLOBAL,
694    #[token("GRAPH", ignore(ascii_case))]
695    GRAPH,
696    #[token("GROUP", ignore(ascii_case))]
697    GROUP,
698    #[token("GZIP", ignore(ascii_case))]
699    GZIP,
700    #[token("HAVING", ignore(ascii_case))]
701    HAVING,
702    #[token("HIGH", ignore(ascii_case))]
703    HIGH,
704    #[token("HILBERT", ignore(ascii_case))]
705    HILBERT,
706    #[token("HISTORY", ignore(ascii_case))]
707    HISTORY,
708    #[token("HIVE", ignore(ascii_case))]
709    HIVE,
710    #[token("HOUR", ignore(ascii_case))]
711    HOUR,
712    #[token("HOURS", ignore(ascii_case))]
713    HOURS,
714    #[token("ICEBERG", ignore(ascii_case))]
715    ICEBERG,
716    #[token("INTERSECT", ignore(ascii_case))]
717    INTERSECT,
718    #[token("IDENTIFIED", ignore(ascii_case))]
719    IDENTIFIED,
720    #[token("IDENTIFIER", ignore(ascii_case))]
721    IDENTIFIER,
722    #[token("IF", ignore(ascii_case))]
723    IF,
724    #[token("IMMUTABLE", ignore(ascii_case))]
725    IMMUTABLE,
726    #[token("IN", ignore(ascii_case))]
727    IN,
728    #[token("INCLUDE_QUERY_ID", ignore(ascii_case))]
729    INCLUDE_QUERY_ID,
730    #[token("INCREMENTAL", ignore(ascii_case))]
731    INCREMENTAL,
732    #[token("INDEX", ignore(ascii_case))]
733    INDEX,
734    #[token("INFORMATION", ignore(ascii_case))]
735    INFORMATION,
736    #[token("INITIALIZE", ignore(ascii_case))]
737    INITIALIZE,
738    #[token("INNER", ignore(ascii_case))]
739    INNER,
740    #[token("INSERT", ignore(ascii_case))]
741    INSERT,
742    #[token("INT", ignore(ascii_case))]
743    INT,
744    #[token("INT16", ignore(ascii_case))]
745    INT16,
746    #[token("INT32", ignore(ascii_case))]
747    INT32,
748    #[token("INT64", ignore(ascii_case))]
749    INT64,
750    #[token("INT8", ignore(ascii_case))]
751    INT8,
752    #[token("INTEGER", ignore(ascii_case))]
753    INTEGER,
754    #[token("INTERVAL", ignore(ascii_case))]
755    INTERVAL,
756    #[token("INTO", ignore(ascii_case))]
757    INTO,
758    #[token("INVERTED", ignore(ascii_case))]
759    INVERTED,
760    #[token("PREVIOUS_DAY", ignore(ascii_case))]
761    PREVIOUS_DAY,
762    #[token("PROCEDURE", ignore(ascii_case))]
763    PROCEDURE,
764    #[token("PROCEDURES", ignore(ascii_case))]
765    PROCEDURES,
766    #[token("IMMEDIATE", ignore(ascii_case))]
767    IMMEDIATE,
768    #[token("IS", ignore(ascii_case))]
769    IS,
770    #[token("ISODOW", ignore(ascii_case))]
771    ISODOW,
772    #[token("ISOWEEK", ignore(ascii_case))]
773    ISOWEEK,
774    #[token("ISOYEAR", ignore(ascii_case))]
775    ISOYEAR,
776    #[token("JOIN", ignore(ascii_case))]
777    JOIN,
778    #[token("JSON", ignore(ascii_case))]
779    JSON,
780    #[token("JULIAN", ignore(ascii_case))]
781    JULIAN,
782    #[token("JWT", ignore(ascii_case))]
783    JWT,
784    #[token("KEY", ignore(ascii_case))]
785    KEY,
786    #[token("KILL", ignore(ascii_case))]
787    KILL,
788    #[token("LAST_DAY", ignore(ascii_case))]
789    LAST_DAY,
790    #[token("LATERAL", ignore(ascii_case))]
791    LATERAL,
792    #[token("LINEAR", ignore(ascii_case))]
793    LINEAR,
794    #[token("LOCATION_PREFIX", ignore(ascii_case))]
795    LOCATION_PREFIX,
796    #[token("LOCKS", ignore(ascii_case))]
797    LOCKS,
798    #[token("LOGICAL", ignore(ascii_case))]
799    LOGICAL,
800    #[token("LOOP", ignore(ascii_case))]
801    LOOP,
802    #[token("SECONDARY", ignore(ascii_case))]
803    SECONDARY,
804    #[token("ROLES", ignore(ascii_case))]
805    ROLES,
806    /// L1DISTANCE and L2DISTANCE op, from https://github.com/pgvector/pgvector
807    #[token("<+>")]
808    L1DISTANCE,
809    #[token("<->")]
810    L2DISTANCE,
811    #[token("LEADING", ignore(ascii_case))]
812    LEADING,
813    #[token("LEFT", ignore(ascii_case))]
814    LEFT,
815    #[token("LET", ignore(ascii_case))]
816    LET,
817    #[token("LIKE", ignore(ascii_case))]
818    LIKE,
819    #[token("LIMIT", ignore(ascii_case))]
820    LIMIT,
821    #[token("LIST", ignore(ascii_case))]
822    LIST,
823    #[token("LOW", ignore(ascii_case))]
824    LOW,
825    #[token("LZO", ignore(ascii_case))]
826    LZO,
827    #[token("MASKING", ignore(ascii_case))]
828    MASKING,
829    #[token("MAP", ignore(ascii_case))]
830    MAP,
831    #[token("MAX_FILE_SIZE", ignore(ascii_case))]
832    MAX_FILE_SIZE,
833    #[token("MASTER_KEY", ignore(ascii_case))]
834    MASTER_KEY,
835    #[token("MEDIUM", ignore(ascii_case))]
836    MEDIUM,
837    #[token("MEMO", ignore(ascii_case))]
838    MEMO,
839    #[token("MEMORY", ignore(ascii_case))]
840    MEMORY,
841    #[token("METRICS", ignore(ascii_case))]
842    METRICS,
843    #[token("MICROSECONDS", ignore(ascii_case))]
844    MICROSECONDS,
845    #[token("MILLENNIUM", ignore(ascii_case))]
846    MILLENNIUM,
847    #[token("MILLISECONDS", ignore(ascii_case))]
848    MILLISECONDS,
849    #[token("MINUTE", ignore(ascii_case))]
850    MINUTE,
851    #[token("MONTH", ignore(ascii_case))]
852    MONTH,
853    #[token("MODIFY", ignore(ascii_case))]
854    MODIFY,
855    #[token("MATERIALIZED", ignore(ascii_case))]
856    MATERIALIZED,
857    #[token("MUST_CHANGE_PASSWORD", ignore(ascii_case))]
858    MUST_CHANGE_PASSWORD,
859    #[token("NEXT_DAY", ignore(ascii_case))]
860    NEXT_DAY,
861    #[token("NON_DISPLAY", ignore(ascii_case))]
862    NON_DISPLAY,
863    #[token("NATURAL", ignore(ascii_case))]
864    NATURAL,
865    #[token("NETWORK", ignore(ascii_case))]
866    NETWORK,
867    #[token("DISABLED", ignore(ascii_case))]
868    DISABLED,
869    #[token("NDJSON", ignore(ascii_case))]
870    NDJSON,
871    #[token("NGRAM", ignore(ascii_case))]
872    NGRAM,
873    #[token("NO_PASSWORD", ignore(ascii_case))]
874    NO_PASSWORD,
875    #[token("NONE", ignore(ascii_case))]
876    NONE,
877    #[token("NOSCAN", ignore(ascii_case))]
878    NOSCAN,
879    #[token("NOT", ignore(ascii_case))]
880    NOT,
881    #[token("NOTENANTSETTING", ignore(ascii_case))]
882    NOTENANTSETTING,
883    #[token("DEFAULT_ROLE", ignore(ascii_case))]
884    DEFAULT_ROLE,
885    #[token("NULL", ignore(ascii_case))]
886    NULL,
887    #[token("NULLABLE", ignore(ascii_case))]
888    NULLABLE,
889    #[token("NUMERIC", ignore(ascii_case))]
890    NUMERIC,
891    #[token("OBJECT", ignore(ascii_case))]
892    OBJECT,
893    #[token("OF", ignore(ascii_case))]
894    OF,
895    #[token("OFFSET", ignore(ascii_case))]
896    OFFSET,
897    #[token("ON", ignore(ascii_case))]
898    ON,
899    #[token("ON_CREATE", ignore(ascii_case))]
900    ON_CREATE,
901    #[token("ON_SCHEDULE", ignore(ascii_case))]
902    ON_SCHEDULE,
903    #[token("OPTIMIZE", ignore(ascii_case))]
904    OPTIMIZE,
905    #[token("OPTIONS", ignore(ascii_case))]
906    OPTIONS,
907    #[token("OR", ignore(ascii_case))]
908    OR,
909    #[token("ORC", ignore(ascii_case))]
910    ORC,
911    #[token("ORDER", ignore(ascii_case))]
912    ORDER,
913    #[token("OUTPUT_HEADER", ignore(ascii_case))]
914    OUTPUT_HEADER,
915    #[token("OUTER", ignore(ascii_case))]
916    OUTER,
917    #[token("ON_ERROR", ignore(ascii_case))]
918    ON_ERROR,
919    #[token("OVER", ignore(ascii_case))]
920    OVER,
921    #[token("OVERWRITE", ignore(ascii_case))]
922    OVERWRITE,
923    #[token("PARTITION", ignore(ascii_case))]
924    PARTITION,
925    #[token("PROPERTIES", ignore(ascii_case))]
926    PROPERTIES,
927    #[token("PARQUET", ignore(ascii_case))]
928    PARQUET,
929    #[token("PASSWORD", ignore(ascii_case))]
930    PASSWORD,
931    #[token("PASSWORD_MIN_LENGTH", ignore(ascii_case))]
932    PASSWORD_MIN_LENGTH,
933    #[token("PASSWORD_MAX_LENGTH", ignore(ascii_case))]
934    PASSWORD_MAX_LENGTH,
935    #[token("PASSWORD_MIN_UPPER_CASE_CHARS", ignore(ascii_case))]
936    PASSWORD_MIN_UPPER_CASE_CHARS,
937    #[token("PASSWORD_MIN_LOWER_CASE_CHARS", ignore(ascii_case))]
938    PASSWORD_MIN_LOWER_CASE_CHARS,
939    #[token("PASSWORD_MIN_NUMERIC_CHARS", ignore(ascii_case))]
940    PASSWORD_MIN_NUMERIC_CHARS,
941    #[token("PASSWORD_MIN_SPECIAL_CHARS", ignore(ascii_case))]
942    PASSWORD_MIN_SPECIAL_CHARS,
943    #[token("PASSWORD_MIN_AGE_DAYS", ignore(ascii_case))]
944    PASSWORD_MIN_AGE_DAYS,
945    #[token("PASSWORD_MAX_AGE_DAYS", ignore(ascii_case))]
946    PASSWORD_MAX_AGE_DAYS,
947    #[token("PASSWORD_MAX_RETRIES", ignore(ascii_case))]
948    PASSWORD_MAX_RETRIES,
949    #[token("PASSWORD_LOCKOUT_TIME_MINS", ignore(ascii_case))]
950    PASSWORD_LOCKOUT_TIME_MINS,
951    #[token("PASSWORD_HISTORY", ignore(ascii_case))]
952    PASSWORD_HISTORY,
953    #[token("PATTERN", ignore(ascii_case))]
954    PATTERN,
955    #[token("PERF", ignore(ascii_case))]
956    PERF,
957    #[token("PIPELINE", ignore(ascii_case))]
958    PIPELINE,
959    #[token("PLAINTEXT_PASSWORD", ignore(ascii_case))]
960    PLAINTEXT_PASSWORD,
961    #[token("POLICIES", ignore(ascii_case))]
962    POLICIES,
963    #[token("POLICY", ignore(ascii_case))]
964    POLICY,
965    #[token("POSITION", ignore(ascii_case))]
966    POSITION,
967    #[token("PROCESSLIST", ignore(ascii_case))]
968    PROCESSLIST,
969    #[token("PRIORITY", ignore(ascii_case))]
970    PRIORITY,
971    #[token("PURGE", ignore(ascii_case))]
972    PURGE,
973    #[token("PUT", ignore(ascii_case))]
974    PUT,
975    #[token("PARTIAL", ignore(ascii_case))]
976    PARTIAL,
977    #[token("QUARTER", ignore(ascii_case))]
978    QUARTER,
979    #[token("QUERY", ignore(ascii_case))]
980    QUERY,
981    #[token("QUOTE", ignore(ascii_case))]
982    QUOTE,
983    #[token("QUOTED_IDENTIFIERS", ignore(ascii_case))]
984    QUOTED_IDENTIFIERS,
985    #[token("RANGE", ignore(ascii_case))]
986    RANGE,
987    #[token("RAWDEFLATE", ignore(ascii_case))]
988    RAWDEFLATE,
989    #[token("READ_ONLY", ignore(ascii_case))]
990    READ_ONLY,
991    #[token("RECLUSTER", ignore(ascii_case))]
992    RECLUSTER,
993    #[token("RECORD_DELIMITER", ignore(ascii_case))]
994    RECORD_DELIMITER,
995    #[token("REFERENCE_USAGE", ignore(ascii_case))]
996    REFERENCE_USAGE,
997    #[token("REFRESH", ignore(ascii_case))]
998    REFRESH,
999    #[token("REGEXP", ignore(ascii_case))]
1000    REGEXP,
1001    #[token("RENAME", ignore(ascii_case))]
1002    RENAME,
1003    #[token("REPLACE", ignore(ascii_case))]
1004    REPLACE,
1005    #[token("RETURN_FAILED_ONLY", ignore(ascii_case))]
1006    RETURN_FAILED_ONLY,
1007    #[token("REVERSE", ignore(ascii_case))]
1008    REVERSE,
1009    #[token("SAMPLE", ignore(ascii_case))]
1010    SAMPLE,
1011    #[token("MERGE", ignore(ascii_case))]
1012    MERGE,
1013    #[token("MATCHED", ignore(ascii_case))]
1014    MATCHED,
1015    #[token("MISSING_FIELD_AS", ignore(ascii_case))]
1016    MISSING_FIELD_AS,
1017    #[token("NULL_FIELD_AS", ignore(ascii_case))]
1018    NULL_FIELD_AS,
1019    #[token("UNMATCHED", ignore(ascii_case))]
1020    UNMATCHED,
1021    #[token("ROW", ignore(ascii_case))]
1022    ROW,
1023    #[token("ROWS", ignore(ascii_case))]
1024    ROWS,
1025    #[token("ROW_TAG", ignore(ascii_case))]
1026    ROW_TAG,
1027    #[token("GRANT", ignore(ascii_case))]
1028    GRANT,
1029    #[token("REAL", ignore(ascii_case))]
1030    REAL,
1031    #[token("REPEAT", ignore(ascii_case))]
1032    REPEAT,
1033    #[token("ROLE", ignore(ascii_case))]
1034    ROLE,
1035    #[token("PRECEDING", ignore(ascii_case))]
1036    PRECEDING,
1037    #[token("PRECISION", ignore(ascii_case))]
1038    PRECISION,
1039    #[token("PRESIGN", ignore(ascii_case))]
1040    PRESIGN,
1041    #[token("PRIVILEGES", ignore(ascii_case))]
1042    PRIVILEGES,
1043    #[token("QUALIFY", ignore(ascii_case))]
1044    QUALIFY,
1045    #[token("REMOVE", ignore(ascii_case))]
1046    REMOVE,
1047    #[token("RETAIN", ignore(ascii_case))]
1048    RETAIN,
1049    #[token("REVOKE", ignore(ascii_case))]
1050    REVOKE,
1051    #[token("RECURSIVE", ignore(ascii_case))]
1052    RECURSIVE,
1053    #[token("RETURN", ignore(ascii_case))]
1054    RETURN,
1055    #[token("RETURNS", ignore(ascii_case))]
1056    RETURNS,
1057    #[token("RESULTSET", ignore(ascii_case))]
1058    RESULTSET,
1059    #[token("RUN", ignore(ascii_case))]
1060    RUN,
1061    #[token("GRANTS", ignore(ascii_case))]
1062    GRANTS,
1063    #[token("REFRESH_MODE", ignore(ascii_case))]
1064    REFRESH_MODE,
1065    #[token("RIGHT", ignore(ascii_case))]
1066    RIGHT,
1067    #[token("RLIKE", ignore(ascii_case))]
1068    RLIKE,
1069    #[token("RAW", ignore(ascii_case))]
1070    RAW,
1071    #[token("OPTIMIZED", ignore(ascii_case))]
1072    OPTIMIZED,
1073    #[token("DECORRELATED", ignore(ascii_case))]
1074    DECORRELATED,
1075    #[token("SATURDAY", ignore(ascii_case))]
1076    SATURDAY,
1077    #[token("SCHEMA", ignore(ascii_case))]
1078    SCHEMA,
1079    #[token("SCHEMAS", ignore(ascii_case))]
1080    SCHEMAS,
1081    #[token("SECOND", ignore(ascii_case))]
1082    SECOND,
1083    #[token("MILLISECOND", ignore(ascii_case))]
1084    MILLISECOND,
1085    #[token("SELECT", ignore(ascii_case))]
1086    SELECT,
1087    #[token("PIVOT", ignore(ascii_case))]
1088    PIVOT,
1089    #[token("UNPIVOT", ignore(ascii_case))]
1090    UNPIVOT,
1091    #[token("SEGMENT", ignore(ascii_case))]
1092    SEGMENT,
1093    #[token("SET", ignore(ascii_case))]
1094    SET,
1095    #[token("UNSET", ignore(ascii_case))]
1096    UNSET,
1097    #[token("SESSION", ignore(ascii_case))]
1098    SESSION,
1099    #[token("SETTINGS", ignore(ascii_case))]
1100    SETTINGS,
1101    #[token("VARIABLES", ignore(ascii_case))]
1102    VARIABLES,
1103    #[token("STAGES", ignore(ascii_case))]
1104    STAGES,
1105    #[token("STATISTIC", ignore(ascii_case))]
1106    STATISTIC,
1107    #[token("SUMMARY", ignore(ascii_case))]
1108    SUMMARY,
1109    #[token("SHA256_PASSWORD", ignore(ascii_case))]
1110    SHA256_PASSWORD,
1111    #[token("SHOW", ignore(ascii_case))]
1112    SHOW,
1113    #[token("SINCE", ignore(ascii_case))]
1114    SINCE,
1115    #[token("SIGNED", ignore(ascii_case))]
1116    SIGNED,
1117    #[token("SINGLE", ignore(ascii_case))]
1118    SINGLE,
1119    #[token("SIZE_LIMIT", ignore(ascii_case))]
1120    SIZE_LIMIT,
1121    #[token("MAX_FILES", ignore(ascii_case))]
1122    MAX_FILES,
1123    #[token("MONDAY", ignore(ascii_case))]
1124    MONDAY,
1125    #[token("SKIP_HEADER", ignore(ascii_case))]
1126    SKIP_HEADER,
1127    #[token("SMALLINT", ignore(ascii_case))]
1128    SMALLINT,
1129    #[token("SNAPPY", ignore(ascii_case))]
1130    SNAPPY,
1131    #[token("SNAPSHOT", ignore(ascii_case))]
1132    SNAPSHOT,
1133    #[token("SPLIT_SIZE", ignore(ascii_case))]
1134    SPLIT_SIZE,
1135    #[token("STAGE", ignore(ascii_case))]
1136    STAGE,
1137    #[token("SYNTAX", ignore(ascii_case))]
1138    SYNTAX,
1139    #[token("USAGE", ignore(ascii_case))]
1140    USAGE,
1141    #[token("ACCESS", ignore(ascii_case))]
1142    ACCESS,
1143    #[token("USE_LOGIC_TYPE", ignore(ascii_case))]
1144    USE_LOGIC_TYPE,
1145    #[token("USE_RAW_PATH", ignore(ascii_case))]
1146    USE_RAW_PATH,
1147    #[token("UPDATE", ignore(ascii_case))]
1148    UPDATE,
1149    #[token("UPLOAD", ignore(ascii_case))]
1150    UPLOAD,
1151    #[token("SEQUENCE", ignore(ascii_case))]
1152    SEQUENCE,
1153    #[token("SEQUENCES", ignore(ascii_case))]
1154    SEQUENCES,
1155    #[token("SHARE", ignore(ascii_case))]
1156    SHARE,
1157    #[token("SHARES", ignore(ascii_case))]
1158    SHARES,
1159    #[token("SUPER", ignore(ascii_case))]
1160    SUPER,
1161    #[token("STATUS", ignore(ascii_case))]
1162    STATUS,
1163    #[token("STORED", ignore(ascii_case))]
1164    STORED,
1165    #[token("STREAM", ignore(ascii_case))]
1166    STREAM,
1167    #[token("STREAMS", ignore(ascii_case))]
1168    STREAMS,
1169    #[token("STRING", ignore(ascii_case))]
1170    STRING,
1171    #[token("SUBSTRING", ignore(ascii_case))]
1172    SUBSTRING,
1173    #[token("SUBSTR", ignore(ascii_case))]
1174    SUBSTR,
1175    #[token("SEMI", ignore(ascii_case))]
1176    SEMI,
1177    #[token("SOUNDS", ignore(ascii_case))]
1178    SOUNDS,
1179    #[token("SYNC", ignore(ascii_case))]
1180    SYNC,
1181    #[token("SYSTEM", ignore(ascii_case))]
1182    SYSTEM,
1183    #[token("STORAGE_TYPE", ignore(ascii_case))]
1184    STORAGE_TYPE,
1185    #[token("TABLE", ignore(ascii_case))]
1186    TABLE,
1187    #[token("TABLES", ignore(ascii_case))]
1188    TABLES,
1189    #[token("TARGET_LAG", ignore(ascii_case))]
1190    TARGET_LAG,
1191    #[token("TEXT", ignore(ascii_case))]
1192    TEXT,
1193    #[token("LONGTEXT", ignore(ascii_case))]
1194    LONGTEXT,
1195    #[token("MEDIUMTEXT", ignore(ascii_case))]
1196    MEDIUMTEXT,
1197    #[token("TINYTEXT", ignore(ascii_case))]
1198    TINYTEXT,
1199    #[token("TENANTSETTING", ignore(ascii_case))]
1200    TENANTSETTING,
1201    #[token("TENANTS", ignore(ascii_case))]
1202    TENANTS,
1203    #[token("TENANT", ignore(ascii_case))]
1204    TENANT,
1205    #[token("THEN", ignore(ascii_case))]
1206    THEN,
1207    #[token("THURSDAY", ignore(ascii_case))]
1208    THURSDAY,
1209    #[token("TIMESTAMP", ignore(ascii_case))]
1210    TIMESTAMP,
1211    #[token("TIMEZONE_HOUR", ignore(ascii_case))]
1212    TIMEZONE_HOUR,
1213    #[token("TIMEZONE_MINUTE", ignore(ascii_case))]
1214    TIMEZONE_MINUTE,
1215    #[token("TIMEZONE", ignore(ascii_case))]
1216    TIMEZONE,
1217    #[token("TINYINT", ignore(ascii_case))]
1218    TINYINT,
1219    #[token("TO", ignore(ascii_case))]
1220    TO,
1221    #[token("TOKEN", ignore(ascii_case))]
1222    TOKEN,
1223    #[token("TRAILING", ignore(ascii_case))]
1224    TRAILING,
1225    #[token("TRANSIENT", ignore(ascii_case))]
1226    TRANSIENT,
1227    #[token("TRIM", ignore(ascii_case))]
1228    TRIM,
1229    #[token("TRUE", ignore(ascii_case))]
1230    TRUE,
1231    #[token("TRUNCATE", ignore(ascii_case))]
1232    TRUNCATE,
1233    #[token("TRY_CAST", ignore(ascii_case))]
1234    TRY_CAST,
1235    #[token("TSV", ignore(ascii_case))]
1236    TSV,
1237    #[token("TUESDAY", ignore(ascii_case))]
1238    TUESDAY,
1239    #[token("TUPLE", ignore(ascii_case))]
1240    TUPLE,
1241    #[token("TYPE", ignore(ascii_case))]
1242    TYPE,
1243    #[token("UNBOUNDED", ignore(ascii_case))]
1244    UNBOUNDED,
1245    #[token("UNION", ignore(ascii_case))]
1246    UNION,
1247    #[token("UINT16", ignore(ascii_case))]
1248    UINT16,
1249    #[token("UINT32", ignore(ascii_case))]
1250    UINT32,
1251    #[token("UINT64", ignore(ascii_case))]
1252    UINT64,
1253    #[token("UINT8", ignore(ascii_case))]
1254    UINT8,
1255    #[token("UNDROP", ignore(ascii_case))]
1256    UNDROP,
1257    #[token("UNSIGNED", ignore(ascii_case))]
1258    UNSIGNED,
1259    #[token("URL", ignore(ascii_case))]
1260    URL,
1261    #[token("METHOD", ignore(ascii_case))]
1262    METHOD,
1263    #[token("AUTHORIZATION_HEADER", ignore(ascii_case))]
1264    AUTHORIZATION_HEADER,
1265    #[token("USE", ignore(ascii_case))]
1266    USE,
1267    #[token("USER", ignore(ascii_case))]
1268    USER,
1269    #[token("USERS", ignore(ascii_case))]
1270    USERS,
1271    #[token("USING", ignore(ascii_case))]
1272    USING,
1273    #[token("VACUUM", ignore(ascii_case))]
1274    VACUUM,
1275    #[token("VALUES", ignore(ascii_case))]
1276    VALUES,
1277    #[token("VARBINARY", ignore(ascii_case))]
1278    VARBINARY,
1279    #[token("VARCHAR", ignore(ascii_case))]
1280    VARCHAR,
1281    #[token("VARIANT", ignore(ascii_case))]
1282    VARIANT,
1283    #[token("VARIABLE", ignore(ascii_case))]
1284    VARIABLE,
1285    #[token("VECTOR", ignore(ascii_case))]
1286    VECTOR,
1287    #[token("VERBOSE", ignore(ascii_case))]
1288    VERBOSE,
1289    #[token("VOLATILE", ignore(ascii_case))]
1290    VOLATILE,
1291    #[token("GRAPHICAL", ignore(ascii_case))]
1292    GRAPHICAL,
1293    #[token("VIEW", ignore(ascii_case))]
1294    VIEW,
1295    #[token("VIEWS", ignore(ascii_case))]
1296    VIEWS,
1297    #[token("VIRTUAL", ignore(ascii_case))]
1298    VIRTUAL,
1299    #[token("WHEN", ignore(ascii_case))]
1300    WHEN,
1301    #[token("WHERE", ignore(ascii_case))]
1302    WHERE,
1303    #[token("WHILE", ignore(ascii_case))]
1304    WHILE,
1305    #[token("WINDOW", ignore(ascii_case))]
1306    WINDOW,
1307    #[token("WITH", ignore(ascii_case))]
1308    WITH,
1309    #[token("WITHIN", ignore(ascii_case))]
1310    WITHIN,
1311    #[token("XML", ignore(ascii_case))]
1312    XML,
1313    #[token("XOR", ignore(ascii_case))]
1314    XOR,
1315    #[token("XZ", ignore(ascii_case))]
1316    XZ,
1317    #[token("YEAR", ignore(ascii_case))]
1318    YEAR,
1319    #[token("ZIP", ignore(ascii_case))]
1320    ZIP,
1321    #[token("ZSTD", ignore(ascii_case))]
1322    ZSTD,
1323    #[token("NULLIF", ignore(ascii_case))]
1324    NULLIF,
1325    #[token("COALESCE", ignore(ascii_case))]
1326    COALESCE,
1327    #[token("RANDOM", ignore(ascii_case))]
1328    RANDOM,
1329    #[token("IFNULL", ignore(ascii_case))]
1330    IFNULL,
1331    #[token("NULLS", ignore(ascii_case))]
1332    NULLS,
1333    #[token("FIRST", ignore(ascii_case))]
1334    FIRST,
1335    #[token("LAST", ignore(ascii_case))]
1336    LAST,
1337    #[token("IGNORE_RESULT", ignore(ascii_case))]
1338    IGNORE_RESULT,
1339    #[token("GROUPING", ignore(ascii_case))]
1340    GROUPING,
1341    #[token("SETS", ignore(ascii_case))]
1342    SETS,
1343    #[token("CUBE", ignore(ascii_case))]
1344    CUBE,
1345    #[token("ROLLUP", ignore(ascii_case))]
1346    ROLLUP,
1347    #[token("INDEXES", ignore(ascii_case))]
1348    INDEXES,
1349    #[token("ADDRESS", ignore(ascii_case))]
1350    ADDRESS,
1351    #[token("OWNERSHIP", ignore(ascii_case))]
1352    OWNERSHIP,
1353    #[token("READ", ignore(ascii_case))]
1354    READ,
1355    #[token("WRITE", ignore(ascii_case))]
1356    WRITE,
1357    #[token("UDF", ignore(ascii_case))]
1358    UDF,
1359    #[token("HANDLER", ignore(ascii_case))]
1360    HANDLER,
1361    #[token("HEADERS", ignore(ascii_case))]
1362    HEADERS,
1363    #[token("LANGUAGE", ignore(ascii_case))]
1364    LANGUAGE,
1365    #[token("IMPORTS", ignore(ascii_case))]
1366    IMPORTS,
1367    #[token("PACKAGES", ignore(ascii_case))]
1368    PACKAGES,
1369    #[token("STATE", ignore(ascii_case))]
1370    STATE,
1371    #[token("TASK", ignore(ascii_case))]
1372    TASK,
1373    #[token("TASKS", ignore(ascii_case))]
1374    TASKS,
1375    #[token("TOP", ignore(ascii_case))]
1376    TOP,
1377    #[token("WAREHOUSE", ignore(ascii_case))]
1378    WAREHOUSE,
1379    #[token("SCHEDULE", ignore(ascii_case))]
1380    SCHEDULE,
1381    #[token("SUSPEND_TASK_AFTER_NUM_FAILURES", ignore(ascii_case))]
1382    SUSPEND_TASK_AFTER_NUM_FAILURES,
1383    #[token("CRON", ignore(ascii_case))]
1384    CRON,
1385    #[token("EXECUTE", ignore(ascii_case))]
1386    EXECUTE,
1387    #[token("SUSPEND", ignore(ascii_case))]
1388    SUSPEND,
1389    #[token("RESUME", ignore(ascii_case))]
1390    RESUME,
1391    #[token("PIPE", ignore(ascii_case))]
1392    PIPE,
1393    #[token("NOTIFICATION", ignore(ascii_case))]
1394    NOTIFICATION,
1395    #[token("INTEGRATION", ignore(ascii_case))]
1396    INTEGRATION,
1397    #[token("ENABLED", ignore(ascii_case))]
1398    ENABLED,
1399    #[token("WEBHOOK", ignore(ascii_case))]
1400    WEBHOOK,
1401    #[token("WEDNESDAY", ignore(ascii_case))]
1402    WEDNESDAY,
1403    #[token("ERROR_INTEGRATION", ignore(ascii_case))]
1404    ERROR_INTEGRATION,
1405    #[token("AUTO_INGEST", ignore(ascii_case))]
1406    AUTO_INGEST,
1407    #[token("PIPE_EXECUTION_PAUSED", ignore(ascii_case))]
1408    PIPE_EXECUTION_PAUSED,
1409    #[token("PREFIX", ignore(ascii_case))]
1410    PREFIX,
1411    #[token("MODIFIED_AFTER", ignore(ascii_case))]
1412    MODIFIED_AFTER,
1413    #[token("UNTIL", ignore(ascii_case))]
1414    UNTIL,
1415    #[token("BEGIN", ignore(ascii_case))]
1416    BEGIN,
1417    #[token("TRANSACTION", ignore(ascii_case))]
1418    TRANSACTION,
1419    #[token("COMMIT", ignore(ascii_case))]
1420    COMMIT,
1421    #[token("ABORT", ignore(ascii_case))]
1422    ABORT,
1423    #[token("ROLLBACK", ignore(ascii_case))]
1424    ROLLBACK,
1425    #[token("TEMPORARY", ignore(ascii_case))]
1426    TEMPORARY,
1427    #[token("TEMP", ignore(ascii_case))]
1428    TEMP,
1429    #[token("SECONDS", ignore(ascii_case))]
1430    SECONDS,
1431    #[token("DAYS", ignore(ascii_case))]
1432    DAYS,
1433    #[token("DICTIONARY", ignore(ascii_case))]
1434    DICTIONARY,
1435    #[token("DICTIONARIES", ignore(ascii_case))]
1436    DICTIONARIES,
1437    #[token("PRIMARY", ignore(ascii_case))]
1438    PRIMARY,
1439    #[token("SOURCE", ignore(ascii_case))]
1440    SOURCE,
1441    #[token("SQL", ignore(ascii_case))]
1442    SQL,
1443    #[token("SUNDAY", ignore(ascii_case))]
1444    SUNDAY,
1445    #[token("WAREHOUSES", ignore(ascii_case))]
1446    WAREHOUSES,
1447    #[token("INSPECT", ignore(ascii_case))]
1448    INSPECT,
1449    #[token("ASSIGN", ignore(ascii_case))]
1450    ASSIGN,
1451    #[token("NODES", ignore(ascii_case))]
1452    NODES,
1453    #[token("UNASSIGN", ignore(ascii_case))]
1454    UNASSIGN,
1455    #[token("ONLINE", ignore(ascii_case))]
1456    ONLINE,
1457    #[token("WORKLOAD", ignore(ascii_case))]
1458    WORKLOAD,
1459    #[token("GROUPS", ignore(ascii_case))]
1460    GROUPS,
1461    #[token("REPORT", ignore(ascii_case))]
1462    REPORT,
1463    #[token("ISSUE", ignore(ascii_case))]
1464    ISSUE,
1465}
1466
1467// Reference: https://www.postgresql.org/docs/current/sql-keywords-appendix.html
1468impl TokenKind {
1469    pub fn is_literal(&self) -> bool {
1470        matches!(
1471            self,
1472            LiteralInteger
1473                | LiteralFloat
1474                | LiteralString
1475                | LiteralCodeString
1476                | PGLiteralHex
1477                | MySQLLiteralHex
1478        )
1479    }
1480
1481    pub fn is_keyword(&self) -> bool {
1482        !matches!(
1483            self,
1484            Ident
1485                | LiteralString
1486                | LiteralCodeString
1487                | PGLiteralHex
1488                | MySQLLiteralHex
1489                | LiteralInteger
1490                | LiteralFloat
1491                | HintPrefix
1492                | HintSuffix
1493                | DoubleEq
1494                | Eq
1495                | NotEq
1496                | Lt
1497                | Gt
1498                | Lte
1499                | Gte
1500                | Spaceship
1501                | Plus
1502                | Minus
1503                | Multiply
1504                | Divide
1505                | IntDiv
1506                | Modulo
1507                | StringConcat
1508                | LParen
1509                | RParen
1510                | Comma
1511                | Dot
1512                | Colon
1513                | DoubleColon
1514                | ColonEqual
1515                | SemiColon
1516                | Backslash
1517                | LBracket
1518                | RBracket
1519                | BitWiseAnd
1520                | BitWiseOr
1521                | Caret
1522                | Factorial
1523                | LBrace
1524                | RBrace
1525                | Dollar
1526                | RArrow
1527                | LongRArrow
1528                | HashRArrow
1529                | HashLongRArrow
1530                | FatRArrow
1531                | BitWiseXor
1532                | BitWiseNot
1533                | TildeAsterisk
1534                | ExclamationMarkTilde
1535                | ExclamationMarkTildeAsterisk
1536                | ShiftLeft
1537                | ShiftRight
1538                | DoubleExclamationMark
1539                | Abs
1540                | SquareRoot
1541                | CubeRoot
1542                | L1DISTANCE
1543                | L2DISTANCE
1544                | Placeholder
1545                | QuestionOr
1546                | QuestionAnd
1547                | ArrowAt
1548                | AtArrow
1549                | AtQuestion
1550                | AtAt
1551                | HashMinus
1552                | EOI
1553        )
1554    }
1555
1556    pub fn is_reserved_function_name(&self) -> bool {
1557        matches!(
1558            self,
1559            TokenKind::ALL
1560            // | TokenKind::ANALYSE
1561            | TokenKind::ANALYZE
1562            | TokenKind::AND
1563            // | TokenKind::ANY
1564            | TokenKind::ASC
1565            // | TokenKind::ASYMMETRIC
1566            | TokenKind::BETWEEN
1567            | TokenKind::BIGINT
1568            // | TokenKind::BIT
1569            | TokenKind::BOOLEAN
1570            | TokenKind::BOTH
1571            | TokenKind::CASE
1572            | TokenKind::CAST
1573            // | TokenKind::CHECK
1574            // | TokenKind::COALESCE
1575            // | TokenKind::COLLATE
1576            // | TokenKind::COLUMN
1577            // | TokenKind::CONSTRAINT
1578            // | TokenKind::CURRENT_CATALOG
1579            // | TokenKind::CURRENT_DATE
1580            // | TokenKind::CURRENT_ROLE
1581            // | TokenKind::CURRENT_TIME
1582            // | TokenKind::CURRENT_TIMESTAMP
1583            // | TokenKind::CURRENT_USER
1584            // | TokenKind::DEC
1585            // | TokenKind::DECIMAL
1586            | TokenKind::DEFAULT
1587            // | TokenKind::DEFERRABLE
1588            | TokenKind::DESC
1589            | TokenKind::DISTINCT
1590            | TokenKind::DO
1591            | TokenKind::ELSE
1592            | TokenKind::END
1593            | TokenKind::EXISTS
1594            | TokenKind::EXTRACT
1595            | TokenKind::DATE_PART
1596            | TokenKind::DATEPART
1597            | TokenKind::DATEDIFF
1598            | TokenKind::DATESUB
1599            | TokenKind::DATEBETWEEN
1600            | TokenKind::FALSE
1601            | TokenKind::FLOAT
1602            // | TokenKind::FOREIGN
1603            // | TokenKind::GREATEST
1604            // | TokenKind::GROUPING
1605            | TokenKind::CUBE
1606            | TokenKind::ROLLUP
1607            // | TokenKind::IFNULL
1608            | TokenKind::IN
1609            | TokenKind::IDENTIFIER
1610            // | TokenKind::INITIALLY
1611            // | TokenKind::INOUT
1612            | TokenKind::INT
1613            | TokenKind::INTEGER
1614            | TokenKind::INTERVAL
1615            | TokenKind::LATERAL
1616            | TokenKind::LEADING
1617            // | TokenKind::LEAST
1618            // | TokenKind::LOCALTIME
1619            // | TokenKind::LOCALTIMESTAMP
1620            // | TokenKind::NATIONAL
1621            // | TokenKind::NCHAR
1622            // | TokenKind::NONE
1623            // | TokenKind::NORMALIZE
1624            | TokenKind::NOT
1625            | TokenKind::NULL
1626            // | TokenKind::NULLIF
1627            // | TokenKind::NUMERIC
1628            // | TokenKind::ONLY
1629            | TokenKind::OR
1630            // | TokenKind::OUT
1631            // | TokenKind::OVERLAY
1632            // | TokenKind::PLACING
1633            | TokenKind::POSITION
1634            // | TokenKind::PRIMARY
1635            // | TokenKind::REAL
1636            // | TokenKind::REFERENCES
1637            // | TokenKind::ROW
1638            | TokenKind::SELECT
1639            // | TokenKind::SESSION_USER
1640            // | TokenKind::SETOF
1641            | TokenKind::SHARE
1642            | TokenKind::SHARES
1643            | TokenKind::SMALLINT
1644            | TokenKind::SOME
1645            | TokenKind::SUBSTRING
1646            | TokenKind::SUBSTR
1647            // | TokenKind::SYMMETRIC
1648            | TokenKind::TABLE
1649            | TokenKind::THEN
1650            // | TokenKind::TIME
1651            // | TokenKind::TIMESTAMP
1652            | TokenKind::TRAILING
1653            // | TokenKind::TREAT
1654            // | TokenKind::TRIM
1655            | TokenKind::TRUE
1656            | TokenKind::TRY_CAST
1657            // | TokenKind::UNIQUE
1658            //| TokenKind::USER
1659            | TokenKind::USING
1660            | TokenKind::VALUES
1661            | TokenKind::VARCHAR
1662            // | TokenKind::VARIADIC
1663            // | TokenKind::XMLATTRIBUTES
1664            // | TokenKind::XMLCONCAT
1665            // | TokenKind::XMLELEMENT
1666            // | TokenKind::XMLEXISTS
1667            // | TokenKind::XMLFOREST
1668            // | TokenKind::XMLNAMESPACES
1669            // | TokenKind::XMLPARSE
1670            // | TokenKind::XMLPI
1671            // | TokenKind::XMLROOT
1672            // | TokenKind::XMLSERIALIZE
1673            // | TokenKind::XMLTABLE
1674            | TokenKind::WHEN
1675            // | TokenKind::ARRAY
1676            | TokenKind::AS
1677            // | TokenKind::CHAR
1678            | TokenKind::CHARACTER
1679            | TokenKind::CREATE
1680            | TokenKind::ATTACH
1681            | TokenKind::EXCEPT
1682            // | TokenKind::FETCH
1683            | TokenKind::FOR
1684            | TokenKind::FROM
1685            // | TokenKind::GRANT
1686            | TokenKind::GROUP
1687            | TokenKind::HAVING
1688            | TokenKind::INTERSECT
1689            | TokenKind::INTO
1690            | TokenKind::LIMIT
1691            | TokenKind::OFFSET
1692            | TokenKind::ON
1693            | TokenKind::OF
1694            | TokenKind::ORDER
1695            | TokenKind::OVER
1696            | TokenKind::QUALIFY
1697            | TokenKind::ROWS
1698            // | TokenKind::PRECISION
1699            // | TokenKind::RETURNING
1700            | TokenKind::TO
1701            | TokenKind::UNION
1702            | TokenKind::WHERE
1703            // | TokenKind::WINDOW
1704            | TokenKind::WITH
1705            | TokenKind::DATE_ADD
1706            | TokenKind::DATEADD
1707            | TokenKind::DATE_DIFF
1708            | TokenKind::DATE_SUB
1709            | TokenKind::DATE_BETWEEN
1710            | TokenKind::DATE_TRUNC
1711            | TokenKind::TRUNC
1712            | TokenKind::LAST_DAY
1713            | TokenKind::PREVIOUS_DAY
1714            | TokenKind::NEXT_DAY
1715            | TokenKind::IGNORE_RESULT
1716        )
1717    }
1718
1719    pub fn is_reserved_ident(&self, after_as: bool) -> bool {
1720        match self {
1721            | TokenKind::ALL
1722            // | TokenKind::ANALYSE
1723            | TokenKind::ANALYZE
1724            | TokenKind::AND
1725            | TokenKind::ANY
1726            | TokenKind::FUNCTION
1727            | TokenKind::PROCEDURE
1728            | TokenKind::SEQUENCE
1729            | TokenKind::ASC
1730            | TokenKind::ANTI
1731            // | TokenKind::ASYMMETRIC
1732            // | TokenKind::AUTHORIZATION
1733            // | TokenKind::BINARY
1734            | TokenKind::BOTH
1735            | TokenKind::BLOCK
1736            | TokenKind::CASE
1737            | TokenKind::CAST
1738            // | TokenKind::CHECK
1739            // | TokenKind::COLLATE
1740            // | TokenKind::COLLATION
1741            // | TokenKind::COLUMN
1742            // | TokenKind::CONCURRENTLY
1743            // | TokenKind::CONSTRAINT
1744            | TokenKind::CONNECTION
1745            | TokenKind::CROSS
1746            // | TokenKind::CURRENT_CATALOG
1747            // | TokenKind::CURRENT_DATE
1748            // | TokenKind::CURRENT_ROLE
1749            // | TokenKind::CURRENT_SCHEMA
1750            // | TokenKind::CURRENT_TIME
1751            | TokenKind::CURRENT_TIMESTAMP
1752            // | TokenKind::CURRENT_USER
1753            // | TokenKind::DEFERRABLE
1754            | TokenKind::DESC
1755            | TokenKind::DISTINCT
1756            | TokenKind::DO
1757            | TokenKind::ELSE
1758            | TokenKind::END
1759            | TokenKind::FALSE
1760            // | TokenKind::FOREIGN
1761            // | TokenKind::FREEZE
1762            | TokenKind::FOR
1763            | TokenKind::FULL
1764            // | TokenKind::ILIKE
1765            | TokenKind::IN
1766            | TokenKind::IDENTIFIER
1767            // | TokenKind::INITIALLY
1768            | TokenKind::INNER
1769            | TokenKind::IS
1770            | TokenKind::JOIN
1771            | TokenKind::LATERAL
1772            | TokenKind::LEADING
1773            | TokenKind::LEFT
1774            | TokenKind::LIKE
1775            // | TokenKind::LOCALTIME
1776            // | TokenKind::LOCALTIMESTAMP
1777            | TokenKind::NATURAL
1778            | TokenKind::NOT
1779            | TokenKind::NULL
1780            // | TokenKind::ONLY
1781            | TokenKind::OR
1782            // | TokenKind::PLACING
1783            // | TokenKind::PRIMARY
1784            // | TokenKind::REFERENCES
1785            | TokenKind::RIGHT
1786            | TokenKind::SELECT
1787            | TokenKind::PIVOT
1788            | TokenKind::UNPIVOT
1789            // | TokenKind::SESSION_USER
1790            // | TokenKind::SIMILAR
1791            | TokenKind::SOME
1792            | TokenKind::SEMI
1793            | TokenKind::SET
1794            | TokenKind::SAMPLE
1795            // | TokenKind::SYMMETRIC
1796            // | TokenKind::TABLESAMPLE
1797            | TokenKind::THEN
1798            | TokenKind::TRAILING
1799            | TokenKind::TRUE
1800            // | TokenKind::UNIQUE
1801            //| TokenKind::USER
1802            | TokenKind::USING
1803            // | TokenKind::VARIADIC
1804            // | TokenKind::VERBOSE
1805            | TokenKind::WHEN => true,
1806            | TokenKind::ARRAY
1807            | TokenKind::AS
1808            | TokenKind::ASOF
1809            | TokenKind::BETWEEN
1810            | TokenKind::CREATE
1811            | TokenKind::ATTACH
1812            | TokenKind::EXCEPT
1813            // | TokenKind::FETCH
1814            | TokenKind::FROM
1815            | TokenKind::GRANT
1816            | TokenKind::GROUP
1817            | TokenKind::HAVING
1818            | TokenKind::INTERSECT
1819            | TokenKind::INTO
1820            // | TokenKind::ISNULL
1821            | TokenKind::LIMIT
1822            | TokenKind::FORMAT
1823            // | TokenKind::NOTNULL
1824            | TokenKind::OFFSET
1825            | TokenKind::ON
1826            | TokenKind::OF
1827            | TokenKind::ORDER
1828            | TokenKind::OVER
1829            | TokenKind::PARTITION
1830            | TokenKind::PROPERTIES
1831            | TokenKind::QUALIFY
1832            | TokenKind::ROWS
1833            | TokenKind::RANGE
1834            // | TokenKind::OVERLAPS
1835            // | TokenKind::RETURNING
1836            | TokenKind::STAGE
1837            | TokenKind::UDF
1838            | TokenKind::SHARE
1839            | TokenKind::SHARES
1840            | TokenKind::TO
1841            | TokenKind::UNION
1842            | TokenKind::WHERE
1843            | TokenKind::WINDOW
1844            | TokenKind::WITH
1845            | TokenKind::IGNORE_RESULT
1846            | TokenKind::MASKING
1847            | TokenKind::POLICY
1848            | TokenKind::TASK
1849            | TokenKind::PIPE
1850            | TokenKind::STREAM
1851            | TokenKind::NOTIFICATION
1852            if !after_as => true,
1853            _ => false
1854        }
1855    }
1856
1857    pub(crate) fn is_grant_reserved_ident(&self, after_as: bool, in_grant: bool) -> bool {
1858        match self {
1859            TokenKind::WAREHOUSE if in_grant => true,
1860            _ => self.is_reserved_ident(after_as),
1861        }
1862    }
1863}
1864
1865pub fn all_reserved_keywords() -> Vec<String> {
1866    let mut result = Vec::new();
1867    for token in TokenKind::iter() {
1868        result.push(format!("{:?}", token));
1869    }
1870    result
1871}
databend_common_ast/parser/token.rs

databend_common_ast/parser/
token.rs
