Enum erl_tokenize::Token

pub enum Token {
    Atom(AtomToken),
    Char(CharToken),
    Comment(CommentToken),
    Float(FloatToken),
    Integer(IntegerToken),
    Keyword(KeywordToken),
    String(StringToken),
    Symbol(SymbolToken),
    Variable(VariableToken),
    Whitespace(WhitespaceToken),
}

Token.

Variants

Atom(AtomToken)Char(CharToken)Comment(CommentToken)Float(FloatToken)Integer(IntegerToken)Keyword(KeywordToken)String(StringToken)Symbol(SymbolToken)Variable(VariableToken)Whitespace(WhitespaceToken)

Methods

impl Token

fn from_text(text: &str, pos: Position) -> Result<Self>

Tries to convert from any prefixes of the text to a token.

Examples

use erl_tokenize::{Token, Position};
use erl_tokenize::values::Symbol;

let pos = Position::new();

// Atom
let token = Token::from_text("foo", pos.clone()).unwrap();
assert_eq!(token.as_atom_token().map(|t| t.value()), Some("foo"));

// Symbol
let token = Token::from_text("[foo]", pos.clone()).unwrap();
assert_eq!(token.as_symbol_token().map(|t| t.value()), Some(Symbol::OpenSquare));

fn text(&self) -> &str

Returns the original textual representation of this token.

Examples

use erl_tokenize::{Token, Position};

let pos = Position::new();

// Comment
assert_eq!(Token::from_text("% foo", pos.clone()).unwrap().text(), "% foo");

// Char
assert_eq!(Token::from_text(r#"$\t"#, pos.clone()).unwrap().text(), r#"$\t"#);

fn is_lexical_token(&self) -> bool

Returns true if this is a lexical token, otherwise false.

fn is_hidden_token(&self) -> bool

Returns true if this is a hidden token, otherwise false.

fn into_lexical_token(self) -> Result<LexicalToken, Self>

Tries to convert into LexicalToken.

fn into_hidden_token(self) -> Result<HiddenToken, Self>

Tries to convert into HiddenToken.

fn as_atom_token(&self) -> Option<&AtomToken>

Tries to return the reference to the inner AtomToken.

fn as_char_token(&self) -> Option<&CharToken>

Tries to return the reference to the inner CharToken.

fn as_float_token(&self) -> Option<&FloatToken>

Tries to return the reference to the inner FloatToken.

fn as_integer_token(&self) -> Option<&IntegerToken>

Tries to return the reference to the inner IntegerToken.

fn as_keyword_token(&self) -> Option<&KeywordToken>

Tries to return the reference to the inner KeywordToken.

fn as_string_token(&self) -> Option<&StringToken>

Tries to return the reference to the inner StringToken.

fn as_symbol_token(&self) -> Option<&SymbolToken>

Tries to return the reference to the inner SymbolToken.

fn as_variable_token(&self) -> Option<&VariableToken>

Tries to return the reference to the inner VariableToken.

fn as_comment_token(&self) -> Option<&CommentToken>

Tries to return the reference to the inner CommentToken.

fn as_whitespace_token(&self) -> Option<&WhitespaceToken>

Tries to return the reference to the inner WhitespaceToken.

fn into_atom_token(self) -> Result<AtomToken, Self>

Tries to return the inner AtomToken.

fn into_char_token(self) -> Result<CharToken, Self>

Tries to return the inner CharToken.

fn into_float_token(self) -> Result<FloatToken, Self>

Tries to return the inner FloatToken.

fn into_integer_token(self) -> Result<IntegerToken, Self>

Tries to return the inner IntegerToken.

fn into_keyword_token(self) -> Result<KeywordToken, Self>

Tries to return the inner KeywordToken.

fn into_string_token(self) -> Result<StringToken, Self>

Tries to return the inner StringToken.

fn into_symbol_token(self) -> Result<SymbolToken, Self>

Tries to return the inner SymbolToken.

fn into_variable_token(self) -> Result<VariableToken, Self>

Tries to return the inner VariableToken.

fn into_comment_token(self) -> Result<CommentToken, Self>

Tries to return the inner CommentToken.

fn into_whitespace_token(self) -> Result<WhitespaceToken, Self>

Tries to return the inner WhitespaceToken.

Trait Implementations

impl Debug for Token

fn fmt(&self, __arg_0: &mut Formatter) -> Result

Formats the value using the given formatter.

impl Clone for Token

fn clone(&self) -> Token

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl From<AtomToken> for Token

fn from(f: AtomToken) -> Self

Performs the conversion.

impl From<CharToken> for Token

fn from(f: CharToken) -> Self

Performs the conversion.

impl From<CommentToken> for Token

fn from(f: CommentToken) -> Self

Performs the conversion.

impl From<FloatToken> for Token

fn from(f: FloatToken) -> Self

Performs the conversion.

impl From<IntegerToken> for Token

fn from(f: IntegerToken) -> Self

Performs the conversion.

impl From<KeywordToken> for Token

fn from(f: KeywordToken) -> Self

Performs the conversion.

impl From<StringToken> for Token

fn from(f: StringToken) -> Self

Performs the conversion.

impl From<SymbolToken> for Token

fn from(f: SymbolToken) -> Self

Performs the conversion.

impl From<VariableToken> for Token

fn from(f: VariableToken) -> Self

Performs the conversion.

impl From<WhitespaceToken> for Token

fn from(f: WhitespaceToken) -> Self

Performs the conversion.

impl From<HiddenToken> for Token

fn from(f: HiddenToken) -> Self

Performs the conversion.

impl From<LexicalToken> for Token

fn from(f: LexicalToken) -> Self

Performs the conversion.

impl PositionRange for Token

fn start_position(&self) -> Position

Returns the (inclusive) start position of this.

fn end_position(&self) -> Position

Returns the (exclusive) end position of this.

impl Display for Token

fn fmt(&self, f: &mut Formatter) -> Result

Formats the value using the given formatter.