Struct Lexer 

pub struct Lexer<'a, R> {
    pub nbr_groups: u32,
    pub initial_state: usize,
    pub first_end_state: usize,
    pub nbr_states: usize,
    pub ascii_to_group: &'a [u32],
    pub utf8_to_group: HashMap<char, u32>,
    pub seg_to_group: SegMap<u32>,
    pub state_table: &'a [usize],
    pub terminal_table: &'a [Terminal],
    /* private fields */
}

Lexical analyzer (lexer) based on tables, which scans a Read source and produces tokens.

The tokens can be extracted one by one with get_token() or from an iterator created by tokens().

Fields

nbr_groups: u32

initial_state: usize

first_end_state: usize

nbr_states: usize

ascii_to_group: &'a [u32]

utf8_to_group: HashMap<char, u32>

seg_to_group: SegMap<u32>

state_table: &'a [usize]

terminal_table: &'a [Terminal]

Implementations

impl<'a, R> Lexer<'a, R>

where R: Read,

pub fn new( nbr_groups: u32, initial_state: usize, first_end_state: usize, nbr_states: usize, ascii_to_group: &'a [u32], utf8_to_group: HashMap<char, u32>, seg_to_group: SegMap<u32>, state_table: &'a [usize], terminal_table: &'a [Terminal], ) -> Lexer<'a, R>

pub fn attach_stream(&mut self, input: CharReader<R>)

pub fn detach_stream(&mut self) -> Option<CharReader<R>>

pub fn set_tab_width(&mut self, width: u8)

pub fn get_tab_width(&self) -> u8

pub fn stream(&self) -> Option<&CharReader<R>>

pub fn is_open(&self) -> bool

pub fn tokens(&mut self) -> LexInterpretIter<'_, 'a, R>

pub fn get_token( &mut self, ) -> Result<Option<(u16, u16, String, PosSpan)>, LexerError>

pub fn update_pos(&mut self, c: char)

pub fn get_error(&self) -> &LexerError

pub fn has_error(&self) -> bool

pub fn is_eos(&self) -> bool