Enum erl_tokenize::Token
[−]
[src]
pub enum Token<'a> { Atom(AtomToken<'a>), Char(CharToken<'a>), Comment(CommentToken<'a>), Float(FloatToken<'a>), Integer(IntegerToken<'a>), Keyword(KeywordToken<'a>), String(StringToken<'a>), Symbol(SymbolToken<'a>), Variable(VariableToken<'a>), Whitespace(WhitespaceToken<'a>), }
Token.
Variants
Atom(AtomToken<'a>)
Char(CharToken<'a>)
Comment(CommentToken<'a>)
Float(FloatToken<'a>)
Integer(IntegerToken<'a>)
Keyword(KeywordToken<'a>)
String(StringToken<'a>)
Symbol(SymbolToken<'a>)
Variable(VariableToken<'a>)
Whitespace(WhitespaceToken<'a>)
Methods
impl<'a> Token<'a>
[src]
fn from_text(text: &'a str) -> Result<Self>
Tries to convert from any prefixes of the text to a token.
Examples
use erl_tokenize::{Token, TokenValue}; use erl_tokenize::values::Symbol; // Atom assert_eq!(Token::from_text("foo").unwrap().value(), TokenValue::Atom("foo")); // Symbol assert_eq!(Token::from_text("[foo]").unwrap().value(), TokenValue::Symbol(Symbol::OpenSquare));
fn value(&self) -> TokenValue
Returns the value of this token.
Examples
use erl_tokenize::{Token, TokenValue}; // Comment assert_eq!(Token::from_text("% foo").unwrap().value(), TokenValue::Comment(" foo")); // Float assert_eq!(Token::from_text("1.23").unwrap().value(), TokenValue::Float(1.23));
fn text(&self) -> &'a str
Returns the original textual representation of this token.
Examples
use erl_tokenize::Token; // Comment assert_eq!(Token::from_text("% foo").unwrap().text(), "% foo"); // Char assert_eq!(Token::from_text(r#"$\t"#).unwrap().text(), r#"$\t"#);
fn kind(&self) -> TokenKind
Returns the kind of this token.
Examples
use erl_tokenize::{Token, TokenKind}; assert_eq!(Token::from_text("foo").unwrap().kind(), TokenKind::Atom); assert_eq!(Token::from_text("123").unwrap().kind(), TokenKind::Integer); assert_eq!(Token::from_text(" ").unwrap().kind(), TokenKind::Whitespace);
Trait Implementations
impl<'a> Debug for Token<'a>
[src]
impl<'a> Clone for Token<'a>
[src]
fn clone(&self) -> Token<'a>
Returns a copy of the value. Read more
fn clone_from(&mut self, source: &Self)
1.0.0
Performs copy-assignment from source
. Read more
impl<'a> From<AtomToken<'a>> for Token<'a>
[src]
impl<'a> From<CharToken<'a>> for Token<'a>
[src]
impl<'a> From<CommentToken<'a>> for Token<'a>
[src]
fn from(f: CommentToken<'a>) -> Self
Performs the conversion.
impl<'a> From<FloatToken<'a>> for Token<'a>
[src]
fn from(f: FloatToken<'a>) -> Self
Performs the conversion.
impl<'a> From<IntegerToken<'a>> for Token<'a>
[src]
fn from(f: IntegerToken<'a>) -> Self
Performs the conversion.
impl<'a> From<KeywordToken<'a>> for Token<'a>
[src]
fn from(f: KeywordToken<'a>) -> Self
Performs the conversion.
impl<'a> From<StringToken<'a>> for Token<'a>
[src]
fn from(f: StringToken<'a>) -> Self
Performs the conversion.
impl<'a> From<SymbolToken<'a>> for Token<'a>
[src]
fn from(f: SymbolToken<'a>) -> Self
Performs the conversion.
impl<'a> From<VariableToken<'a>> for Token<'a>
[src]
fn from(f: VariableToken<'a>) -> Self
Performs the conversion.
impl<'a> From<WhitespaceToken<'a>> for Token<'a>
[src]
fn from(f: WhitespaceToken<'a>) -> Self
Performs the conversion.