Struct lib_ruby_parser::Token

pub struct Token {
    pub token_type: i32,
    pub token_value: TokenValue,
    pub loc: Loc,
}

A token that is emitted by a lexer and consumed by a parser

Fields

token_type: i32

token_value: TokenValue

loc: Loc

Implementations

impl Token

pub fn to_string_lossy(&self) -> String

Converts Token to a string, replaces unknown chars to U+FFFD

pub fn to_bytes(&self) -> Vec<u8>

Converts Token to a vector of bytes

pub fn into_string_lossy(self) -> String

pub fn into_bytes(self) -> Vec<u8>

Trait Implementations

impl Clone for Token

pub fn clone(&self) -> Token

Returns a copy of the value.

pub fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

pub fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.