Struct BaseLexer 

pub struct BaseLexer<'input, 'arena, Ext, Input, TF>
where
    Ext: LexerRecog<'input, 'arena, TF, Self> + 'static,
    Input: CharStream<'input>,
    TF: TokenFactory<'input, 'arena> + 'arena,
    'input: 'arena,
{
    pub interpreter: Option<Box<LexerATNSimulator>>,
    pub input: Option<Input>,
    pub token_start_char_index: isize,
    pub token_start_line: u32,
    pub token_start_column: i32,
    pub token_type: i32,
    pub token: Option<&'arena mut TF::Tok>,
    pub channel: i32,
    pub mode_stack: Vec<usize>,
    pub mode: usize,
    pub text: Option<String>,
    /* private fields */
}

Default implementation of Lexer

Public fields in this struct are intended to be used by embedded actions

Fields

interpreter: Option<Box<LexerATNSimulator>>

LexerATNSimulator instance of this lexer

input: Option<Input>

CharStream used by this lexer

token_start_char_index: isize

token_start_line: u32

token_start_column: i32

token_type: i32

Overrides token type emitted by lexer for current token

token: Option<&'arena mut TF::Tok>

Make it Some to override token that is currently being generated by lexer

channel: i32

Channel lexer is currently assigning tokens to

mode_stack: Vec<usize>

stack of modes, which is used for pushMode,popMode lexer actions

mode: usize

Mode lexer is currently in

text: Option<String>

Make it Some to override text for token that is currently being generated by lexer

Implementations

impl<'input, 'arena, Ext, Input, TF> BaseLexer<'input, 'arena, Ext, Input, TF>

where Ext: LexerRecog<'input, 'arena, TF, Self> + 'static, Input: CharStream<'input>, TF: TokenFactory<'input, 'arena> + 'arena, 'input: 'arena,

pub fn get_char_index(&self) -> isize

Current position in input stream

pub fn get_text<'a>(&'a self) -> Cow<'a, str>

where 'input: 'a,

Current token text

pub fn set_text(&mut self, _text: impl Into<String>)

Used from lexer actions to override text of the token that will be emitted next

pub fn add_error_listener( &mut self, listener: Box<dyn ErrorListener<'input, 'arena, Self>>, )

Add error listener

pub fn remove_error_listeners(&mut self)

Remove and drop all error listeners

pub fn new_base_lexer(input: Input, recog: Ext, factory: TF) -> Self

Creates new lexer instance

Trait Implementations

impl<'input, 'arena, Ext, Input, TF> Deref for BaseLexer<'input, 'arena, Ext, Input, TF>

where Ext: LexerRecog<'input, 'arena, TF, Self> + 'static, Input: CharStream<'input>, TF: TokenFactory<'input, 'arena> + 'arena,

type Target = Ext

The resulting type after dereferencing.

fn deref(&self) -> &Self::Target

Dereferences the value.

impl<'input, 'arena, Ext, Input, TF> DerefMut for BaseLexer<'input, 'arena, Ext, Input, TF>

where Ext: LexerRecog<'input, 'arena, TF, Self> + 'static, Input: CharStream<'input>, TF: TokenFactory<'input, 'arena> + 'arena,

fn deref_mut(&mut self) -> &mut Self::Target

Mutably dereferences the value.

impl<'input, 'arena, L, Input, TF> Lexer<'input, 'arena, Input, TF> for BaseLexer<'input, 'arena, L, Input, TF>

where L: LexerRecog<'input, 'arena, TF, Self> + 'static, Input: CharStream<'input>, TF: TokenFactory<'input, 'arena> + 'arena, 'input: 'arena,

fn input(&mut self) -> &mut Input

Same as TokenStream::get_input_stream but returns concrete type instance important for proper inlining in hot code of LexerATNSimulator

fn set_channel(&mut self, v: i32)

Sets channel where current token will be pushed

fn push_mode(&mut self, m: usize)

Pushes current mode to internal mode stack and sets m as current lexer mode `pop_mode should be used to recover previous mode

fn pop_mode(&mut self) -> Option<usize>

Pops mode from internal mode stack

fn set_type(&mut self, t: i32)

Sets type of the current token Called from action to override token that will be emitted by lexer

fn set_mode(&mut self, m: usize)

Sets lexer mode discarding current one

fn more(&mut self)

Used to informs lexer that it should consider next token as a continuation of the current one

fn skip(&mut self)

Tells lexer to completely ignore and not emit current token.

impl<'input, 'arena, L, Input, TF> TokenSource<'input, 'arena, TF> for BaseLexer<'input, 'arena, L, Input, TF>

where L: LexerRecog<'input, 'arena, TF, Self> + 'static, Input: CharStream<'input>, TF: TokenFactory<'input, 'arena> + 'arena, 'input: 'arena,

fn next_token(&mut self) -> &'arena mut TF::Tok

Return a {@link Token} object from your input stream (usually a {@link CharStream}). Do not fail/return upon lexing error; keep chewing on the characters until you get a good one; errors are not passed through to the parser.

fn get_line(&self) -> u32

Get the line number for the current position in the input stream. The first line in the input is line 1.

fn get_char_position_in_line(&self) -> i32

Get the index into the current line for the current position in the input stream. The first character on a line has position 0.

fn get_input_stream(&mut self) -> Option<&mut dyn IntStream>

Returns underlying input stream

fn get_source_name(&self) -> String

Returns string identifier of underlying input e.g. file name

fn get_token_factory(&self) -> &TF

Gets the TokenFactory this token source is currently using for creating Token objects from the input.

fn get_dfa_string(&self) -> String