Enum erl_tokenize::Token
[−]
[src]
pub enum Token { Atom(AtomToken), Char(CharToken), Comment(CommentToken), Float(FloatToken), Integer(IntegerToken), Keyword(KeywordToken), String(StringToken), Symbol(SymbolToken), Variable(VariableToken), Whitespace(WhitespaceToken), }
Token.
Variants
Atom(AtomToken)
Char(CharToken)
Comment(CommentToken)
Float(FloatToken)
Integer(IntegerToken)
Keyword(KeywordToken)
String(StringToken)
Symbol(SymbolToken)
Variable(VariableToken)
Whitespace(WhitespaceToken)
Methods
impl Token
[src]
fn from_text(text: &str) -> Result<Self>
Tries to convert from any prefixes of the text to a token.
Examples
use erl_tokenize::{Token, TokenValue}; use erl_tokenize::values::Symbol; // Atom assert_eq!(Token::from_text("foo").unwrap().value(), TokenValue::Atom("foo")); // Symbol assert_eq!(Token::from_text("[foo]").unwrap().value(), TokenValue::Symbol(Symbol::OpenSquare));
fn value(&self) -> TokenValue
Returns the value of this token.
Examples
use erl_tokenize::{Token, TokenValue}; // Comment assert_eq!(Token::from_text("% foo").unwrap().value(), TokenValue::Comment(" foo")); // Float assert_eq!(Token::from_text("1.23").unwrap().value(), TokenValue::Float(1.23));
fn text(&self) -> &str
Returns the original textual representation of this token.
Examples
use erl_tokenize::Token; // Comment assert_eq!(Token::from_text("% foo").unwrap().text(), "% foo"); // Char assert_eq!(Token::from_text(r#"$\t"#).unwrap().text(), r#"$\t"#);
fn kind(&self) -> TokenKind
Returns the kind of this token.
Examples
use erl_tokenize::{Token, TokenKind}; assert_eq!(Token::from_text("foo").unwrap().kind(), TokenKind::Atom); assert_eq!(Token::from_text("123").unwrap().kind(), TokenKind::Integer); assert_eq!(Token::from_text(" ").unwrap().kind(), TokenKind::Whitespace);
Trait Implementations
impl Debug for Token
[src]
impl Clone for Token
[src]
fn clone(&self) -> Token
Returns a copy of the value. Read more
fn clone_from(&mut self, source: &Self)
1.0.0
Performs copy-assignment from source
. Read more
impl From<AtomToken> for Token
[src]
impl From<CharToken> for Token
[src]
impl From<CommentToken> for Token
[src]
fn from(f: CommentToken) -> Self
Performs the conversion.
impl From<FloatToken> for Token
[src]
fn from(f: FloatToken) -> Self
Performs the conversion.
impl From<IntegerToken> for Token
[src]
fn from(f: IntegerToken) -> Self
Performs the conversion.
impl From<KeywordToken> for Token
[src]
fn from(f: KeywordToken) -> Self
Performs the conversion.
impl From<StringToken> for Token
[src]
fn from(f: StringToken) -> Self
Performs the conversion.
impl From<SymbolToken> for Token
[src]
fn from(f: SymbolToken) -> Self
Performs the conversion.
impl From<VariableToken> for Token
[src]
fn from(f: VariableToken) -> Self
Performs the conversion.
impl From<WhitespaceToken> for Token
[src]
fn from(f: WhitespaceToken) -> Self
Performs the conversion.