Enum erl_tokenize::Token

pub enum Token {
    Atom(AtomToken),
    Char(CharToken),
    Comment(CommentToken),
    Float(FloatToken),
    Integer(IntegerToken),
    Keyword(KeywordToken),
    SigilString(SigilStringToken),
    String(StringToken),
    Symbol(SymbolToken),
    Variable(VariableToken),
    Whitespace(WhitespaceToken),
}

Token.

Variants

Atom(AtomToken)

Char(CharToken)

Comment(CommentToken)

Float(FloatToken)

Integer(IntegerToken)

Keyword(KeywordToken)

SigilString(SigilStringToken)

String(StringToken)

Symbol(SymbolToken)

Variable(VariableToken)

Whitespace(WhitespaceToken)

Implementations

impl Token

pub fn from_text(text: &str, pos: Position) -> Result<Self>

Tries to convert from any prefixes of the text to a token.

Examples

use erl_tokenize::{Token, Position};
use erl_tokenize::values::Symbol;

let pos = Position::new();

// Atom
let token = Token::from_text("foo", pos.clone()).unwrap();
assert_eq!(token.as_atom_token().map(|t| t.value()), Some("foo"));

// Symbol
let token = Token::from_text("[foo]", pos.clone()).unwrap();
assert_eq!(token.as_symbol_token().map(|t| t.value()), Some(Symbol::OpenSquare));

pub fn text(&self) -> &str

Returns the original textual representation of this token.

Examples

use erl_tokenize::{Token, Position};

let pos = Position::new();

// Comment
assert_eq!(Token::from_text("% foo", pos.clone()).unwrap().text(), "% foo");

// Char
assert_eq!(Token::from_text(r#"$\t"#, pos.clone()).unwrap().text(), r#"$\t"#);

pub fn is_lexical_token(&self) -> bool

Returns true if this is a lexical token, otherwise false.

pub fn is_hidden_token(&self) -> bool

Returns true if this is a hidden token, otherwise false.

pub fn into_lexical_token(self) -> Result<LexicalToken, Self>

Tries to convert into LexicalToken.

pub fn into_hidden_token(self) -> Result<HiddenToken, Self>

Tries to convert into HiddenToken.

pub fn as_atom_token(&self) -> Option<&AtomToken>

Tries to return the reference to the inner AtomToken.

pub fn as_char_token(&self) -> Option<&CharToken>

Tries to return the reference to the inner CharToken.

pub fn as_float_token(&self) -> Option<&FloatToken>

Tries to return the reference to the inner FloatToken.

pub fn as_integer_token(&self) -> Option<&IntegerToken>

Tries to return the reference to the inner IntegerToken.

pub fn as_keyword_token(&self) -> Option<&KeywordToken>

Tries to return the reference to the inner KeywordToken.

pub fn as_string_token(&self) -> Option<&StringToken>

Tries to return the reference to the inner StringToken.

pub fn as_symbol_token(&self) -> Option<&SymbolToken>

Tries to return the reference to the inner SymbolToken.

pub fn as_variable_token(&self) -> Option<&VariableToken>

Tries to return the reference to the inner VariableToken.

pub fn as_comment_token(&self) -> Option<&CommentToken>

Tries to return the reference to the inner CommentToken.

pub fn as_whitespace_token(&self) -> Option<&WhitespaceToken>

Tries to return the reference to the inner WhitespaceToken.

pub fn into_atom_token(self) -> Result<AtomToken, Self>

Tries to return the inner AtomToken.

pub fn into_char_token(self) -> Result<CharToken, Self>

Tries to return the inner CharToken.

pub fn into_float_token(self) -> Result<FloatToken, Self>

Tries to return the inner FloatToken.

pub fn into_integer_token(self) -> Result<IntegerToken, Self>

Tries to return the inner IntegerToken.

pub fn into_keyword_token(self) -> Result<KeywordToken, Self>

Tries to return the inner KeywordToken.

pub fn into_string_token(self) -> Result<StringToken, Self>

Tries to return the inner StringToken.

pub fn into_symbol_token(self) -> Result<SymbolToken, Self>

Tries to return the inner SymbolToken.

pub fn into_variable_token(self) -> Result<VariableToken, Self>

Tries to return the inner VariableToken.

pub fn into_comment_token(self) -> Result<CommentToken, Self>

Tries to return the inner CommentToken.

pub fn into_whitespace_token(self) -> Result<WhitespaceToken, Self>

Tries to return the inner WhitespaceToken.

Trait Implementations

impl Clone for Token

fn clone(&self) -> Token

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<'de> Deserialize<'de> for Token

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Display for Token

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl From<AtomToken> for Token

fn from(f: AtomToken) -> Self

Converts to this type from the input type.

impl From<CharToken> for Token

fn from(f: CharToken) -> Self

Converts to this type from the input type.

impl From<CommentToken> for Token

fn from(f: CommentToken) -> Self

Converts to this type from the input type.

impl From<FloatToken> for Token

fn from(f: FloatToken) -> Self

Converts to this type from the input type.

impl From<HiddenToken> for Token

fn from(f: HiddenToken) -> Self

Converts to this type from the input type.

impl From<IntegerToken> for Token

fn from(f: IntegerToken) -> Self

Converts to this type from the input type.

impl From<KeywordToken> for Token

fn from(f: KeywordToken) -> Self

Converts to this type from the input type.

impl From<LexicalToken> for Token

fn from(f: LexicalToken) -> Self

Converts to this type from the input type.

impl From<SigilStringToken> for Token

fn from(f: SigilStringToken) -> Self

Converts to this type from the input type.

impl From<StringToken> for Token

fn from(f: StringToken) -> Self

Converts to this type from the input type.

impl From<SymbolToken> for Token

fn from(f: SymbolToken) -> Self

Converts to this type from the input type.

impl From<VariableToken> for Token

fn from(f: VariableToken) -> Self

Converts to this type from the input type.

impl From<WhitespaceToken> for Token

fn from(f: WhitespaceToken) -> Self

Converts to this type from the input type.

impl PositionRange for Token

fn start_position(&self) -> Position

Returns the (inclusive) start position of this.

fn end_position(&self) -> Position

Returns the (exclusive) end position of this.

impl Serialize for Token

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.