Skip to main content

Tokenizer

Struct Tokenizer 

Source
pub struct Tokenizer<'a> { /* private fields */ }

Implementations§

Source§

impl<'a> Tokenizer<'a>

Source

pub fn new(input: &'a str) -> Tokenizer<'a>

A tokenizer over input, which should already have been through preprocess.

Source

pub fn set_deadline(&mut self, deadline: Instant)

Stops run once deadline has passed; Tokenizer::timed_out says whether it did.

Source

pub fn timed_out(&self) -> bool

Source

pub fn position(&self) -> usize

How far into the input it got.

Source

pub fn report_errors(&mut self)

Reports each parse error to the sink’s Sink::error. It costs a pass over the input, for the errors in it as a stream.

Source

pub fn keep_positions(&mut self)

Hands the sink each character reference as a run of its own, at the place it was written, rather than gathered up with the text around it: for a sink that keeps where each character came from.

Source

pub fn set_text_mode(&mut self, mode: TextMode)

Source

pub fn set_last_start_tag(&mut self, name: &str)

The tag an end tag must match to close RCDATA, RAWTEXT or script data, as when tokenizing a fragment.

Source

pub fn run<S>(&mut self, sink: &mut S)
where S: Sink,

Tokenizes the input to the end, handing each token to sink.

Auto Trait Implementations§

§

impl<'a> Forget for Tokenizer<'a>

§

impl<'a> Freeze for Tokenizer<'a>

§

impl<'a> Move for Tokenizer<'a>

§

impl<'a> RefUnwindSafe for Tokenizer<'a>

§

impl<'a> Send for Tokenizer<'a>

§

impl<'a> Sync for Tokenizer<'a>

§

impl<'a> Unpin for Tokenizer<'a>

§

impl<'a> UnsafeUnpin for Tokenizer<'a>

§

impl<'a> UnwindSafe for Tokenizer<'a>

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = !

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, !>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.