[−][src]Enum polyglot_tokenizer::tokenizer::Token
Token is an enum whose variants represent each type of possible Token returned from the
Tokenizer
. Block Comments and Strings hold both the start and end indicator for the Tokens.
Line Comments hold the open indicator for the Tokens. See below for examples.
Examples
use polyglot_tokenizer::{Token, Tokenizer}; let content = "/* Block Comment */"; let tokens: Vec<Token> = Tokenizer::new(content).tokens().collect(); let expected = vec![Token::BlockComment("/*", " Block Comment ", "*/")]; assert_eq!(tokens, expected);
use polyglot_tokenizer::{Token, Tokenizer}; let content = "// Line Comment"; let tokens: Vec<Token> = Tokenizer::new(content).tokens().collect(); let expected = vec![Token::LineComment("//", "Line Comment")]; assert_eq!(tokens, expected);
Variants
Ident(&'a str)
Number(&'a str)
Symbol(&'a str)
Trait Implementations
impl<'a> Debug for Token<'a>
[src]
impl<'a> Eq for Token<'a>
[src]
impl<'a> PartialEq<Token<'a>> for Token<'a>
[src]
impl<'a> StructuralEq for Token<'a>
[src]
impl<'a> StructuralPartialEq for Token<'a>
[src]
Auto Trait Implementations
impl<'a> RefUnwindSafe for Token<'a>
impl<'a> Send for Token<'a>
impl<'a> Sync for Token<'a>
impl<'a> Unpin for Token<'a>
impl<'a> UnwindSafe for Token<'a>
Blanket Implementations
impl<T> Any for T where
T: 'static + ?Sized,
[src]
T: 'static + ?Sized,
impl<T> Borrow<T> for T where
T: ?Sized,
[src]
T: ?Sized,
impl<T> BorrowMut<T> for T where
T: ?Sized,
[src]
T: ?Sized,
fn borrow_mut(&mut self) -> &mut T
[src]
impl<T> From<T> for T
[src]
impl<T, U> Into<U> for T where
U: From<T>,
[src]
U: From<T>,
impl<T, U> TryFrom<U> for T where
U: Into<T>,
[src]
U: Into<T>,
type Error = Infallible
The type returned in the event of a conversion error.
fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>
[src]
impl<T, U> TryInto<U> for T where
U: TryFrom<T>,
[src]
U: TryFrom<T>,