Enum Token

pub enum Token {
    Brace(Substr, Span),
    CommentStart(Substr, Span),
    CommentEnd(Substr, Span),
    Type(Substr, Span),
    Directive(Substr, Span),
    Pragma(Substr, Span),
    PreprocessorInsertion(Substr, Span),
    Keyword(Substr, Span),
    Ident(Substr, Span),
    FloatLiteral(Substr, Span),
    IntLiteral(Substr, Span),
    Path(Substr, Span),
    Operator(Substr, Span),
    Punct(Substr, Span),
    Unrecognized(Substr, Span),
    Attribute(Substr, Span),
    Function(Substr, Span),
    Param(Substr, Span),
    Struct(Substr, Span),
    Field(Substr, Span),
    Module(Substr, Span),
    Plain(Substr, Span),
}

Variants

Brace(Substr, Span)

CommentStart(Substr, Span)

CommentEnd(Substr, Span)

Type(Substr, Span)

Directive(Substr, Span)

Pragma(Substr, Span)

PreprocessorInsertion(Substr, Span)

Keyword(Substr, Span)

Ident(Substr, Span)

FloatLiteral(Substr, Span)

IntLiteral(Substr, Span)

Path(Substr, Span)

Operator(Substr, Span)

Punct(Substr, Span)

Unrecognized(Substr, Span)

Attribute(Substr, Span)

Function(Substr, Span)

Param(Substr, Span)

Struct(Substr, Span)

Field(Substr, Span)

Module(Substr, Span)

Plain(Substr, Span)

Implementations

impl Token

pub fn as_inner(&self) -> (Substr, Span)

pub fn brace(lexeme: Substr, span: Span) -> Self

pub fn comment_start(lexeme: Substr, span: Span) -> Self

pub fn comment_end(lexeme: Substr, span: Span) -> Self

pub fn type_(lexeme: Substr, span: Span) -> Self

pub fn directive(lexeme: Substr, span: Span) -> Self

pub fn pragma(lexeme: Substr, span: Span) -> Self

pub fn preprocessor_insertion(lexeme: Substr, span: Span) -> Self

pub fn keyword(lexeme: Substr, span: Span) -> Self

pub fn ident(lexeme: Substr, span: Span) -> Self

pub fn float_literal(lexeme: Substr, span: Span) -> Self

pub fn int_literal(lexeme: Substr, span: Span) -> Self

pub fn path_(lexeme: Substr, span: Span) -> Self

pub fn operator(lexeme: Substr, span: Span) -> Self

pub fn punct(lexeme: Substr, span: Span) -> Self

pub fn unrecognized(lexeme: Substr, span: Span) -> Self

pub fn attribute(lexeme: Substr, span: Span) -> Self

pub fn function(lexeme: Substr, span: Span) -> Self

pub fn param(lexeme: Substr, span: Span) -> Self

pub fn struct_(lexeme: Substr, span: Span) -> Self

pub fn field(lexeme: Substr, span: Span) -> Self

pub fn module(lexeme: Substr, span: Span) -> Self

pub fn plain(lexeme: Substr, span: Span) -> Self

pub fn match_brace(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_comment_start(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_comment_end(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_directive(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_pragma(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_preprocessor_insertion( input: &str, ) -> Option<(usize, usize, TokenKind)>

pub fn match_ident(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_float_literal(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_int_literal(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_path(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_operator(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_punct(input: &str) -> Option<(usize, usize, TokenKind)>

pub fn match_unrecognized(input: &str) -> Option<(usize, usize, TokenKind)>

Trait Implementations

impl Clone for Token

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl DebugLisp for Token

fn fmt(&self, f: &mut Formatter<'_>, _: usize) -> Result

impl Display for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl From<Token> for Text

fn from(value: Token) -> Self

Converts to this type from the input type.

impl Hash for Token

fn hash<H>(&self, state: &mut H)

where H: Hasher,

Feeds this value into the given Hasher.

fn hash_slice<H>(data: &[Self], state: &mut H)

where H: Hasher, Self: Sized,

Feeds a slice of this type into the given Hasher.

impl PartialEq for Token

fn eq(&self, other: &Self) -> bool

Tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Rhs) -> bool

Tests for !=. The default implementation is almost always sufficient, and should not be overridden without very good reason.

impl Spanned for Token

fn span(&self) -> Span

impl Token for Token

type Kind = TokenKind

fn find<S>(input: S) -> Option<(usize, usize, Self::Kind)>

where S: AsRef<str>,

Find the first match for this token in a str slice, returning the start and end byte offsets and the matching Kind.

fn from_parts(kind: Self::Kind, substr: Substr, span: Span) -> Self

Construct a token from its constituent parts.

fn multilines() -> &'static HashSet<TokenKind>

Returns the set of this token’s variants which should be treated as multi-line patterns (i.e., the . character in regex patterns should match \n characters).

fn discards() -> &'static HashSet<TokenKind>

Returns the set of this token’s variants which should be discarded by the lexer when scanning.

fn lexeme(&self) -> Substr

Returns the actual text content of a token.

fn kind(&self) -> TokenKind

Returns the Kind of this token. Used in ParseStreamer methods like check_kind and consume_kind. Effectively a more user-friendly version of std::mem::discriminant.

fn as_matchable(&self) -> (TokenKind, &str, Span)

Decomposes the token into its constituent Kind, lexeme and Span) parts, with the lexeme as a &str slice for compatibility with pattern matching.

impl Eq for Token