Struct lib_ruby_parser::Token

#[repr(C)]
pub struct Token { /* fields omitted */ }

A token that is emitted by a lexer and consumed by a parser

Trait Implementations

impl Clone for Token

fn clone(&self) -> Token

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl PartialEq<Token> for Token

fn eq(&self, other: &Token) -> bool

This method tests for self and other values to be equal, and is used by ==.

fn ne(&self, other: &Token) -> bool

This method tests for !=.

impl TokenTrait for Token

fn new(
    token_type: i32,
    token_value: Bytes,
    loc: Loc,
    lex_state_before: LexState,
    lex_state_after: LexState
) -> Self

Constructor

fn token_type(&self) -> i32

Returns type of the token

fn token_value(&self) -> &Bytes

Returns value of the token

fn set_token_value(&mut self, token_value: Bytes)

Sets token value

fn into_token_value(self) -> Bytes

Consumes self, returns owned values of the token

fn loc(&self) -> Loc

Returns location of the token

fn lex_state_before(&self) -> LexState

Returns lex state before reading the token

fn lex_state_after(&self) -> LexState

Returns lex state after reading the token

fn as_bytes(&self) -> &[u8]

Returns a byte array of the token value

fn into_bytes(self) -> Vec<u8> where
    Self: Sized, 

Consumes a token and returns an owned byte array of the token value

fn as_str_lossy(&self) -> Result<&str, Utf8Error>

Converts token value into &str

fn to_string_lossy(&self) -> String

Converts token to a string, replaces unknown chars to U+FFFD

fn to_string(&self) -> Result<String, FromUtf8Error>

Converts token to a string

fn into_string(self) -> Result<String, FromUtf8Error> where
    Self: Sized, 

Consumes a token and converts it into a string

fn token_name(&self) -> &'static str

Returns name of the token

impl Eq for Token

impl StructuralEq for Token

impl StructuralPartialEq for Token