[−][src]Struct polyglot_tokenizer::tokenizer::Tokenizer

pub struct Tokenizer<'a> { /* fields omitted */ }

The tokenizer is responsible for turning content into an iterator of Token.

Examples

use polyglot_tokenizer::{Token, Tokenizer};

let content = r#"let x = 5;"#;
let tokens: Vec<Token> = Tokenizer::new(content).tokens().collect();
let expected = vec![
  Token::Ident("let"),
  Token::Ident("x"),
  Token::Symbol("="),
  Token::Number("5"),
  Token::Symbol(";"),
];
assert_eq!(tokens, expected);

Implementations

impl<'a> Tokenizer<'a>[src]

pub fn new(content: &'a str) -> Self[src]

pub fn tokens(&self) -> Tokens<'a>ⓘ

Important traits for Tokens<'a>

impl<'a> Iterator for Tokens<'a> type Item = Token<'a>;
[src]

Auto Trait Implementations

impl<'a> RefUnwindSafe for Tokenizer<'a>

impl<'a> Send for Tokenizer<'a>

impl<'a> Sync for Tokenizer<'a>

impl<'a> Unpin for Tokenizer<'a>

impl<'a> UnwindSafe for Tokenizer<'a>

Blanket Implementations

impl<T> Any for T where
    T: 'static + ?Sized, [src]

impl<T> Borrow<T> for T where
    T: ?Sized, [src]

impl<T> BorrowMut<T> for T where
    T: ?Sized, [src]

impl<T> From<T> for T[src]

impl<T, U> Into<U> for T where
    U: From<T>, [src]

impl<T, U> TryFrom<U> for T where
    U: Into<T>, [src]

type Error = Infallible

The type returned in the event of a conversion error.

impl<T, U> TryInto<U> for T where
    U: TryFrom<T>, [src]

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.