Struct ress::Tokenizer[−][src]

pub struct Tokenizer<'a> { /* fields omitted */ }

This structure will perform the low level tokenization before the Scanner provides additional context

Implementations

impl<'a> Tokenizer<'a>[src]

pub fn new(stream: &'a str) -> Self[src]

Create a new tokenizer using the provided string reference to create a JsBuffer

pub fn next(
    &mut self,
    allow_html_comment_close: bool
) -> Result<RawItem, RawError>[src]

Get the next raw token from the js text

pub fn next_regex(&mut self, start_len: usize) -> Result<RawItem, RawError>[src]

get the next regex token from the js text, providing the lenth of the already consumed token (this will be either 1 or 2)

note: this should only be used after first getting / or /= from the next method.

pub fn skip_whitespace(&mut self) -> (usize, usize)[src]

Skip any whitespace that might be coming up

Auto Trait Implementations

impl<'a> RefUnwindSafe for Tokenizer<'a>

impl<'a> Send for Tokenizer<'a>

impl<'a> Sync for Tokenizer<'a>

impl<'a> Unpin for Tokenizer<'a>

impl<'a> UnwindSafe for Tokenizer<'a>

Blanket Implementations

impl<T> Any for T where
    T: 'static + ?Sized, [src]

impl<T> Borrow<T> for T where
    T: ?Sized, [src]

impl<T> BorrowMut<T> for T where
    T: ?Sized, [src]

impl<T> From<T> for T[src]

impl<T, U> Into<U> for T where
    U: From<T>, [src]

impl<T, U> TryFrom<U> for T where
    U: Into<T>, [src]

type Error = Infallible

The type returned in the event of a conversion error.

impl<T, U> TryInto<U> for T where
    U: TryFrom<T>, [src]

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.