Enum Token 

pub enum Token {
    LexMarker,
    Indentation,
    Indent(Vec<(Token, Range<usize>)>),
    Dedent(Vec<(Token, Range<usize>)>),
    BlankLine(Option<String>),
    Whitespace(usize),
    Dash,
    Period,
    OpenParen,
    CloseParen,
    Colon,
    ExclamationMark,
    QuestionMark,
    Semicolon,
    InvertedExclamationMark,
    InvertedQuestionMark,
    Ellipsis,
    IdeographicFullStop,
    FullwidthExclamationMark,
    FullwidthQuestionMark,
    ExclamationQuestionMark,
    QuestionExclamationMark,
    ArabicQuestionMark,
    ArabicFullStop,
    ArabicTripleDot,
    ArabicComma,
    Danda,
    DoubleDanda,
    BengaliCurrencyNumeratorFour,
    EthiopianFullStop,
    ArmenianFullStop,
    TibetanShad,
    ThaiFongman,
    MyanmarComma,
    MyanmarFullStop,
    Comma,
    Quote,
    Equals,
    Number(String),
    Text(String),
}

All possible tokens in the lex format

Variants

LexMarker

Indentation

Indent(Vec<(Token, Range<usize>)>)

Dedent(Vec<(Token, Range<usize>)>)

BlankLine(Option<String>)

Whitespace(usize)

Dash

Period

OpenParen

CloseParen

Colon

ExclamationMark

QuestionMark

Semicolon

InvertedExclamationMark

InvertedQuestionMark

Ellipsis

IdeographicFullStop

FullwidthExclamationMark

FullwidthQuestionMark

ExclamationQuestionMark

QuestionExclamationMark

ArabicQuestionMark

ArabicFullStop

ArabicTripleDot

ArabicComma

Danda

DoubleDanda

BengaliCurrencyNumeratorFour

EthiopianFullStop

ArmenianFullStop

TibetanShad

ThaiFongman

MyanmarComma

MyanmarFullStop

Comma

Quote

Equals

Number(String)

Text(String)

Implementations

impl Token

pub fn simple_name(&self) -> &'static str

Get an uppercase identifier describing this token variant. Used by CLI simple output.

pub fn is_indent(&self) -> bool

Check if this token represents indentation

pub fn is_indent_level(&self) -> bool

Check if this token represents semantic indentation level

pub fn is_dedent_level(&self) -> bool

Check if this token represents semantic dedentation level

pub fn is_whitespace(&self) -> bool

Check if this token is whitespace (including indentation)

pub fn is_sequence_marker(&self) -> bool

Check if this token is a sequence marker

pub fn is_number(&self) -> bool

Check if this token is a number

pub fn is_text(&self) -> bool

Check if this token is text content

pub fn is_end_punctuation(&self) -> bool

Trait Implementations

impl Clone for Token

fn clone(&self) -> Token

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for Token

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Display for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Hash for Token

fn hash<__H: Hasher>(&self, state: &mut __H)

Feeds this value into the given Hasher.

fn hash_slice<H>(data: &[Self], state: &mut H)

where H: Hasher, Self: Sized,

Feeds a slice of this type into the given Hasher.

impl<'s> Logos<'s> for Token

type Error = ()

Error type returned by the lexer. This can be set using #[logos(error = MyError)]. Defaults to () if not set.

type Extras = ()

Associated type Extras for the particular lexer. This can be set using #[logos(extras = MyExtras)] and accessed inside callbacks.

type Source = str

Source type this token can be lexed from. This will default to str, unless one of the defined patterns explicitly uses non-unicode byte values or byte slices, in which case that implementation will use [u8].

fn lex(lex: &mut Lexer<'s, Self>)

The heart of Logos. Called by the Lexer. The implementation for this function is generated by the logos-derive crate.

fn lexer(source: &'source Self::Source) -> Lexer<'source, Self>

where Self::Extras: Default,

Create a new instance of a Lexer that will produce tokens implementing this Logos.

fn lexer_with_extras( source: &'source Self::Source, extras: Self::Extras, ) -> Lexer<'source, Self>

Create a new instance of a Lexer with the provided Extras that will produce tokens implementing this Logos.

impl PartialEq for Token

fn eq(&self, other: &Token) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Serialize for Token

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.

impl ToLexString for Token

fn to_lex_string(&self) -> String

impl Eq for Token

impl StructuralPartialEq for Token