Enum Token 

pub enum Token {
    As(Span),
    Class(Span),
    Declare(Span),
    Else(Span),
    Fn(Span),
    For(Span),
    If(Span),
    Import(Span),
    Is(Span),
    Instance(Span),
    Match(Span),
    Pub(Span),
    Type(Span),
    When(Span),
    Then(Span),
    With(Span),
    Where(Span),
    Add(Span),
    And(Span),
    Concat(Span),
    Div(Span),
    Dot(Span),
    Eq(Span),
    Ne(Span),
    Ge(Span),
    Gt(Span),
    Le(Span),
    Lt(Span),
    Mod(Span),
    Mul(Span),
    Or(Span),
    Sub(Span),
    ArrowL(Span),
    ArrowR(Span),
    Assign(Span),
    BackSlash(Span),
    BraceL(Span),
    BraceR(Span),
    BracketL(Span),
    BracketR(Span),
    Colon(Span),
    ColonColon(Span),
    Comma(Span),
    CommentL(Span),
    CommentR(Span),
    DotDot(Span),
    HashTag(Span),
    In(Span),
    Let(Span),
    Rec(Span),
    ParenL(Span),
    ParenR(Span),
    Pipe(Span),
    Question(Span),
    SemiColon(Span),
    Whitespace(Span),
    WhitespaceNewline(Span),
    Bool(bool, Span),
    Float(f64, Span),
    Int(u64, Span),
    Null(Span),
    String(String, Span),
    HttpsUrl(String, Span),
    Ident(String, Span),
    Eof(Span),
}

A Token represents a lexical token in the Rex programming language. It includes the values of literals, identifiers, and comments.

Variants

As(Span)

Class(Span)

Declare(Span)

Else(Span)

Fn(Span)

For(Span)

If(Span)

Import(Span)

Is(Span)

Instance(Span)

Match(Span)

Pub(Span)

Type(Span)

When(Span)

Then(Span)

With(Span)

Where(Span)

Add(Span)

And(Span)

Concat(Span)

Div(Span)

Dot(Span)

Eq(Span)

Ne(Span)

Ge(Span)

Gt(Span)

Le(Span)

Lt(Span)

Mod(Span)

Mul(Span)

Or(Span)

Sub(Span)

ArrowL(Span)

ArrowR(Span)

Assign(Span)

BackSlash(Span)

BraceL(Span)

BraceR(Span)

BracketL(Span)

BracketR(Span)

Colon(Span)

ColonColon(Span)

Comma(Span)

CommentL(Span)

CommentR(Span)

DotDot(Span)

HashTag(Span)

In(Span)

Let(Span)

Rec(Span)

ParenL(Span)

ParenR(Span)

Pipe(Span)

Question(Span)

SemiColon(Span)

Whitespace(Span)

WhitespaceNewline(Span)

Bool(bool, Span)

Float(f64, Span)

Int(u64, Span)

Null(Span)

String(String, Span)

HttpsUrl(String, Span)

Ident(String, Span)

Eof(Span)

Implementations

impl Token

pub fn tokenize(input: &str) -> Result<Tokens, LexicalError>

pub fn regex() -> Result<&'static Regex, LexicalError>

Get the regular expression that can capture all Tokens. The regular expression can capture Unicode characters.

Return

An unwrapped Regex object.

pub fn precedence(&self) -> Precedence

pub fn is_whitespace(&self) -> bool

Trait Implementations

impl Clone for Token

fn clone(&self) -> Token

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for Token

fn deserialize<__D>( __deserializer: __D, ) -> Result<Token, <__D as Deserializer<'de>>::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Display for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl PartialEq for Token

fn eq(&self, other: &Token) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for Token

fn serialize<__S>( &self, __serializer: __S, ) -> Result<<__S as Serializer>::Ok, <__S as Serializer>::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl Spanned for Token

fn span(&self) -> &Span

fn span_mut(&mut self) -> &mut Span

fn set_span_begin(&mut self, begin: Position)

fn set_span_end(&mut self, end: Position)

impl StructuralPartialEq for Token