Struct lib_ruby_parser::Token

#[repr(C)]pub struct Token {
    pub token_type: i32,
    pub token_value: Bytes,
    pub loc: LocPtr,
    pub lex_state_before: LexState,
    pub lex_state_after: LexState,
}

A token that is emitted by a lexer and consumed by a parser

Fields

token_type: i32

Numeric representation of the token type, e.g. 42 (for example) for tINTEGER

token_value: Bytes

Value of the token, e.g “42” for 42

loc: LocPtr

Location of the token

lex_state_before: LexState

Lex state before reading the token

lex_state_after: LexState

Lex state after reading the token

Implementations

impl Token

pub fn as_bytes(&self) -> &[u8]

Returns a byte array of the token value

pub fn as_bytes_mut(&mut self) -> &mut [u8]

Returns a mut byte array of the token value

pub fn into_bytes(self) -> List<u8>

Consumes a token and returns an owned byte array of the token value

pub fn as_str_lossy(&self) -> Result<&str, Utf8Error>

Converts token value into &str

pub fn to_string_lossy(&self) -> String

Converts token to a string, replaces unknown chars to U+FFFD

pub fn to_string(&self) -> Result<String, FromUtf8Error>

Converts token to a string

pub fn into_string(self) -> Result<String, FromUtf8Error>

Consumes a token and converts it into a string

Trait Implementations

impl Clone for Token

fn clone(&self) -> Token

Returns a copy of the value.

pub fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.