Enum erl_tokenize::Token

pub enum Token {
    Atom(AtomToken),
    Char(CharToken),
    Comment(CommentToken),
    Float(FloatToken),
    Integer(IntegerToken),
    Keyword(KeywordToken),
    String(StringToken),
    Symbol(SymbolToken),
    Variable(VariableToken),
    Whitespace(WhitespaceToken),
}

Token.

Variants

Atom(AtomToken)Char(CharToken)Comment(CommentToken)Float(FloatToken)Integer(IntegerToken)Keyword(KeywordToken)String(StringToken)Symbol(SymbolToken)Variable(VariableToken)Whitespace(WhitespaceToken)

Methods

impl Token

fn from_text(text: &str, pos: Position) -> Result<Self>

Tries to convert from any prefixes of the text to a token.

Examples

use erl_tokenize::{Token, TokenValue, Position};
use erl_tokenize::values::Symbol;

let pos = Position::new();

// Atom
assert_eq!(Token::from_text("foo", pos.clone()).unwrap().value(), TokenValue::Atom("foo"));

// Symbol
assert_eq!(Token::from_text("[foo]", pos.clone()).unwrap().value(),
           TokenValue::Symbol(Symbol::OpenSquare));

fn value(&self) -> TokenValue

Returns the value of this token.

Examples

use erl_tokenize::{Token, TokenValue, Position};

let pos = Position::new();

// Comment
assert_eq!(Token::from_text("% foo", pos.clone()).unwrap().value(),
           TokenValue::Comment(" foo"));

// Float
assert_eq!(Token::from_text("1.23", pos.clone()).unwrap().value(),
           TokenValue::Float(1.23));

fn text(&self) -> &str

Returns the original textual representation of this token.

Examples

use erl_tokenize::{Token, Position};

let pos = Position::new();

// Comment
assert_eq!(Token::from_text("% foo", pos.clone()).unwrap().text(), "% foo");

// Char
assert_eq!(Token::from_text(r#"$\t"#, pos.clone()).unwrap().text(), r#"$\t"#);

fn position(&self) -> &Position

Returns the start position of the begnning of this token.

fn kind(&self) -> TokenKind

Returns the kind of this token.

Examples

use erl_tokenize::{Token, TokenKind, Position};

let pos = Position::new();
assert_eq!(Token::from_text("foo", pos.clone()).unwrap().kind(), TokenKind::Atom);
assert_eq!(Token::from_text("123", pos.clone()).unwrap().kind(), TokenKind::Integer);
assert_eq!(Token::from_text(" ", pos.clone()).unwrap().kind(), TokenKind::Whitespace);

Trait Implementations

impl Debug for Token

fn fmt(&self, __arg_0: &mut Formatter) -> Result

Formats the value using the given formatter.

impl Clone for Token

fn clone(&self) -> Token

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl From<AtomToken> for Token

fn from(f: AtomToken) -> Self

Performs the conversion.

impl From<CharToken> for Token

fn from(f: CharToken) -> Self

Performs the conversion.

impl From<CommentToken> for Token

fn from(f: CommentToken) -> Self

Performs the conversion.

impl From<FloatToken> for Token

fn from(f: FloatToken) -> Self

Performs the conversion.

impl From<IntegerToken> for Token

fn from(f: IntegerToken) -> Self

Performs the conversion.

impl From<KeywordToken> for Token

fn from(f: KeywordToken) -> Self

Performs the conversion.

impl From<StringToken> for Token

fn from(f: StringToken) -> Self

Performs the conversion.

impl From<SymbolToken> for Token

fn from(f: SymbolToken) -> Self

Performs the conversion.

impl From<VariableToken> for Token

fn from(f: VariableToken) -> Self

Performs the conversion.

impl From<WhitespaceToken> for Token

fn from(f: WhitespaceToken) -> Self

Performs the conversion.