Struct Tokenizer 

pub struct Tokenizer<'a> { /* private fields */ }

Implementations

impl<'a> Tokenizer<'a>

pub fn new(source_code: &'a str, ignore_errors: bool) -> Tokenizer<'a>

pub fn position(&self) -> usize

pub fn unclosed(&self, what: &str)

pub fn end_of_file(&self) -> bool

pub fn back(&self, token: Token<'a>)

pub fn next_token(&self, ignore_unclosed: bool) -> Token<'a>

pub fn from_offset(&mut self, offset: usize) -> (usize, usize)

return (line, column), use rope for simplicity

Trait Implementations

impl<'a> Debug for Tokenizer<'a>

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.