Enum erl_tokenize::Token

pub enum Token<'a> {
    Atom(AtomToken<'a>),
    Char(CharToken<'a>),
    Comment(CommentToken<'a>),
    Float(FloatToken<'a>),
    Integer(IntegerToken<'a>),
    Keyword(KeywordToken<'a>),
    String(StringToken<'a>),
    Symbol(SymbolToken<'a>),
    Variable(VariableToken<'a>),
    Whitespace(WhitespaceToken<'a>),
}

Token.

Variants

Atom(AtomToken<'a>)Char(CharToken<'a>)Comment(CommentToken<'a>)Float(FloatToken<'a>)Integer(IntegerToken<'a>)Keyword(KeywordToken<'a>)String(StringToken<'a>)Symbol(SymbolToken<'a>)Variable(VariableToken<'a>)Whitespace(WhitespaceToken<'a>)

Methods

impl<'a> Token<'a>

fn from_text(text: &'a str) -> Result<Self>

Tries to convert from any prefixes of the text to a token.

Examples

use erl_tokenize::{Token, TokenValue};
use erl_tokenize::values::Symbol;

// Atom
assert_eq!(Token::from_text("foo").unwrap().value(), TokenValue::Atom("foo"));

// Symbol
assert_eq!(Token::from_text("[foo]").unwrap().value(),
           TokenValue::Symbol(Symbol::OpenSquare));

fn value(&self) -> TokenValue

Returns the value of this token.

Examples

use erl_tokenize::{Token, TokenValue};

// Comment
assert_eq!(Token::from_text("% foo").unwrap().value(), TokenValue::Comment(" foo"));

// Float
assert_eq!(Token::from_text("1.23").unwrap().value(), TokenValue::Float(1.23));

fn text(&self) -> &'a str

Returns the original textual representation of this token.

Examples

use erl_tokenize::Token;

// Comment
assert_eq!(Token::from_text("% foo").unwrap().text(), "% foo");

// Char
assert_eq!(Token::from_text(r#"$\t"#).unwrap().text(), r#"$\t"#);

fn kind(&self) -> TokenKind

Returns the kind of this token.

Examples

use erl_tokenize::{Token, TokenKind};

assert_eq!(Token::from_text("foo").unwrap().kind(), TokenKind::Atom);
assert_eq!(Token::from_text("123").unwrap().kind(), TokenKind::Integer);
assert_eq!(Token::from_text(" ").unwrap().kind(), TokenKind::Whitespace);

Trait Implementations

impl<'a> Debug for Token<'a>

fn fmt(&self, __arg_0: &mut Formatter) -> Result

Formats the value using the given formatter.

impl<'a> Clone for Token<'a>

fn clone(&self) -> Token<'a>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'a> From<AtomToken<'a>> for Token<'a>

fn from(f: AtomToken<'a>) -> Self

Performs the conversion.

impl<'a> From<CharToken<'a>> for Token<'a>

fn from(f: CharToken<'a>) -> Self

Performs the conversion.

impl<'a> From<CommentToken<'a>> for Token<'a>

fn from(f: CommentToken<'a>) -> Self

Performs the conversion.

impl<'a> From<FloatToken<'a>> for Token<'a>

fn from(f: FloatToken<'a>) -> Self

Performs the conversion.

impl<'a> From<IntegerToken<'a>> for Token<'a>

fn from(f: IntegerToken<'a>) -> Self

Performs the conversion.

impl<'a> From<KeywordToken<'a>> for Token<'a>

fn from(f: KeywordToken<'a>) -> Self

Performs the conversion.

impl<'a> From<StringToken<'a>> for Token<'a>

fn from(f: StringToken<'a>) -> Self

Performs the conversion.

impl<'a> From<SymbolToken<'a>> for Token<'a>

fn from(f: SymbolToken<'a>) -> Self

Performs the conversion.

impl<'a> From<VariableToken<'a>> for Token<'a>

fn from(f: VariableToken<'a>) -> Self

Performs the conversion.

impl<'a> From<WhitespaceToken<'a>> for Token<'a>

fn from(f: WhitespaceToken<'a>) -> Self

Performs the conversion.