Enum sqlparser::tokenizer::Token

pub enum Token {
    EOF,
    Word(Word),
    Number(String, bool),
    Char(char),
    SingleQuotedString(String),
    DoubleQuotedString(String),
    TripleSingleQuotedString(String),
    TripleDoubleQuotedString(String),
    DollarQuotedString(DollarQuotedString),
    SingleQuotedByteStringLiteral(String),
    DoubleQuotedByteStringLiteral(String),
    TripleSingleQuotedByteStringLiteral(String),
    TripleDoubleQuotedByteStringLiteral(String),
    SingleQuotedRawStringLiteral(String),
    DoubleQuotedRawStringLiteral(String),
    TripleSingleQuotedRawStringLiteral(String),
    TripleDoubleQuotedRawStringLiteral(String),
    NationalStringLiteral(String),
    EscapedStringLiteral(String),
    HexStringLiteral(String),
    Comma,
    Whitespace(Whitespace),
    DoubleEq,
    Eq,
    Neq,
    Lt,
    Gt,
    LtEq,
    GtEq,
    Spaceship,
    Plus,
    Minus,
    Mul,
    Div,
    DuckIntDiv,
    Mod,
    StringConcat,
    LParen,
    RParen,
    Period,
    Colon,
    DoubleColon,
    Assignment,
    SemiColon,
    Backslash,
    LBracket,
    RBracket,
    Ampersand,
    Pipe,
    Caret,
    LBrace,
    RBrace,
    RArrow,
    Sharp,
    Tilde,
    TildeAsterisk,
    ExclamationMarkTilde,
    ExclamationMarkTildeAsterisk,
    DoubleTilde,
    DoubleTildeAsterisk,
    ExclamationMarkDoubleTilde,
    ExclamationMarkDoubleTildeAsterisk,
    ShiftLeft,
    ShiftRight,
    Overlap,
    ExclamationMark,
    DoubleExclamationMark,
    AtSign,
    CaretAt,
    PGSquareRoot,
    PGCubeRoot,
    Placeholder(String),
    Arrow,
    LongArrow,
    HashArrow,
    HashLongArrow,
    AtArrow,
    ArrowAt,
    HashMinus,
    AtQuestion,
    AtAt,
    Question,
    QuestionAnd,
    QuestionPipe,
}

SQL Token enumeration

Variants

EOF

An end-of-file marker, not a real token

Word(Word)

A keyword (like SELECT) or an optionally quoted SQL identifier

Number(String, bool)

An unsigned numeric literal

Char(char)

A character that could not be tokenized

SingleQuotedString(String)

Single quoted string: i.e: ‘string’

DoubleQuotedString(String)

Double quoted string: i.e: “string”

TripleSingleQuotedString(String)

Triple single quoted strings: Example ‘’‘abc’‘’ BigQuery

TripleDoubleQuotedString(String)

Triple double quoted strings: Example “”“abc”“” BigQuery

DollarQuotedString(DollarQuotedString)

Dollar quoted string: i.e: $$string$$ or $tag_name$string$tag_name$

SingleQuotedByteStringLiteral(String)

Byte string literal: i.e: b’string’ or B’string’ (note that some backends, such as PostgreSQL, may treat this syntax as a bit string literal instead, i.e: b’10010101’)

DoubleQuotedByteStringLiteral(String)

Byte string literal: i.e: b“string“ or B“string“

TripleSingleQuotedByteStringLiteral(String)

Triple single quoted literal with byte string prefix. Example B'''abc''' BigQuery

TripleDoubleQuotedByteStringLiteral(String)

Triple double quoted literal with byte string prefix. Example B"""abc""" BigQuery

SingleQuotedRawStringLiteral(String)

Single quoted literal with raw string prefix. Example R'abc' BigQuery

DoubleQuotedRawStringLiteral(String)

Double quoted literal with raw string prefix. Example R"abc" BigQuery

TripleSingleQuotedRawStringLiteral(String)

Triple single quoted literal with raw string prefix. Example R'''abc''' BigQuery

TripleDoubleQuotedRawStringLiteral(String)

Triple double quoted literal with raw string prefix. Example R"""abc""" BigQuery

NationalStringLiteral(String)

“National” string literal: i.e: N’string’

EscapedStringLiteral(String)

“escaped” string literal, which are an extension to the SQL standard: i.e: e’first \n second’ or E ‘first \n second’

HexStringLiteral(String)

Hexadecimal string literal: i.e.: X’deadbeef’

Comma

Comma

Whitespace(Whitespace)

Whitespace (space, tab, etc)

DoubleEq

Double equals sign ==

Eq

Equality operator =

Neq

Not Equals operator <> (or != in some dialects)

Lt

Less Than operator <

Gt

Greater Than operator >

LtEq

Less Than Or Equals operator <=

GtEq

Greater Than Or Equals operator >=

Spaceship

Spaceship operator <=>

Plus

Plus operator +

Minus

Minus operator -

Mul

Multiplication operator *

Div

Division operator /

DuckIntDiv

Integer division operator // in DuckDB

Mod

Modulo Operator %

StringConcat

String concatenation ||

LParen

Left parenthesis (

RParen

Right parenthesis )

Period

Period (used for compound identifiers or projections into nested types)

Colon

Colon :

DoubleColon

DoubleColon :: (used for casting in PostgreSQL)

Assignment

Assignment := (used for keyword argument in DuckDB macros and some functions, and for variable declarations in DuckDB and Snowflake)

SemiColon

SemiColon ; used as separator for COPY and payload

Backslash

Backslash \ used in terminating the COPY payload with \.

LBracket

Left bracket [

RBracket

Right bracket ]

Ampersand

Ampersand &

Pipe

Pipe |

Caret

Caret ^

LBrace

Left brace {

RBrace

Right brace }

RArrow

Right Arrow =>

Sharp

Sharp # used for PostgreSQL Bitwise XOR operator

Tilde

Tilde ~ used for PostgreSQL Bitwise NOT operator or case sensitive match regular expression operator

TildeAsterisk

~* , a case insensitive match regular expression operator in PostgreSQL

ExclamationMarkTilde

!~ , a case sensitive not match regular expression operator in PostgreSQL

ExclamationMarkTildeAsterisk

!~* , a case insensitive not match regular expression operator in PostgreSQL

DoubleTilde

~~, a case sensitive match pattern operator in PostgreSQL

DoubleTildeAsterisk

~~*, a case insensitive match pattern operator in PostgreSQL

ExclamationMarkDoubleTilde

!~~, a case sensitive not match pattern operator in PostgreSQL

ExclamationMarkDoubleTildeAsterisk

!~~*, a case insensitive not match pattern operator in PostgreSQL

ShiftLeft

<<, a bitwise shift left operator in PostgreSQL

ShiftRight

>>, a bitwise shift right operator in PostgreSQL

Overlap

&&, an overlap operator in PostgreSQL

ExclamationMark

Exclamation Mark ! used for PostgreSQL factorial operator

DoubleExclamationMark

Double Exclamation Mark !! used for PostgreSQL prefix factorial operator

AtSign

AtSign @ used for PostgreSQL abs operator

CaretAt

^@, a “starts with” string operator in PostgreSQL

PGSquareRoot

|/, a square root math operator in PostgreSQL

PGCubeRoot

||/, a cube root math operator in PostgreSQL

Placeholder(String)

? or $ , a prepared statement arg placeholder

Arrow

->, used as a operator to extract json field in PostgreSQL

LongArrow

->>, used as a operator to extract json field as text in PostgreSQL

HashArrow

#>, extracts JSON sub-object at the specified path

HashLongArrow

#>>, extracts JSON sub-object at the specified path as text

AtArrow

jsonb @> jsonb -> boolean: Test whether left json contains the right json

ArrowAt

jsonb <@ jsonb -> boolean: Test whether right json contains the left json

HashMinus

jsonb #- text[] -> jsonb: Deletes the field or array element at the specified path, where path elements can be either field keys or array indexes.

AtQuestion

jsonb @? jsonpath -> boolean: Does JSON path return any item for the specified JSON value?

AtAt

jsonb @@ jsonpath → boolean: Returns the result of a JSON path predicate check for the specified JSON value. Only the first item of the result is taken into account. If the result is not Boolean, then NULL is returned.

Question

jsonb ? text -> boolean: Checks whether the string exists as a top-level key within the jsonb object

QuestionAnd

jsonb ?& text[] -> boolean: Check whether all members of the text array exist as top-level keys within the jsonb object

QuestionPipe

jsonb ?| text[] -> boolean: Check whether any member of the text array exists as top-level keys within the jsonb object

Implementations

impl Token

pub fn make_keyword(keyword: &str) -> Self

pub fn make_word(word: &str, quote_style: Option<char>) -> Self

Trait Implementations

impl Clone for Token

fn clone(&self) -> Token

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for Token

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Display for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Hash for Token

fn hash<__H: Hasher>(&self, state: &mut __H)

Feeds this value into the given Hasher.

fn hash_slice<H>(data: &[Self], state: &mut H)

where H: Hasher, Self: Sized,

Feeds a slice of this type into the given Hasher.

impl Ord for Token

fn cmp(&self, other: &Token) -> Ordering

This method returns an Ordering between self and other.

fn max(self, other: Self) -> Self

where Self: Sized,

Compares and returns the maximum of two values.

fn min(self, other: Self) -> Self

where Self: Sized,

Compares and returns the minimum of two values.

fn clamp(self, min: Self, max: Self) -> Self

where Self: Sized + PartialOrd,

Restrict a value to a certain interval.

impl PartialEq<Token> for TokenWithLocation

fn eq(&self, other: &Token) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl PartialEq<TokenWithLocation> for Token

fn eq(&self, other: &TokenWithLocation) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl PartialEq for Token

fn eq(&self, other: &Token) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

This method tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl PartialOrd for Token

fn partial_cmp(&self, other: &Token) -> Option<Ordering>

This method returns an ordering between self and other values if one exists.

fn lt(&self, other: &Rhs) -> bool

This method tests less than (for self and other) and is used by the < operator.

fn le(&self, other: &Rhs) -> bool

This method tests less than or equal to (for self and other) and is used by the <= operator.

fn gt(&self, other: &Rhs) -> bool

This method tests greater than (for self and other) and is used by the > operator.

fn ge(&self, other: &Rhs) -> bool

This method tests greater than or equal to (for self and other) and is used by the >= operator.

impl Serialize for Token

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Visit for Token

fn visit<V: Visitor>(&self, visitor: &mut V) -> ControlFlow<V::Break>

impl VisitMut for Token

fn visit<V: VisitorMut>(&mut self, visitor: &mut V) -> ControlFlow<V::Break>

impl Eq for Token

impl StructuralPartialEq for Token