Struct BaseLexer

pub struct BaseLexer<'input, T: LexerRecog<'input, Self> + 'static, Input: CharStream<TF::From>, TF: TokenFactory<'input> = CommonTokenFactory> {
    pub interpreter: Option<Box<LexerATNSimulator>>,
    pub input: Option<Input>,
    pub token_start_char_index: isize,
    pub token_start_line: isize,
    pub token_start_column: isize,
    pub token_type: isize,
    pub token: Option<TF::Tok>,
    pub channel: isize,
    pub mode: usize,
    pub text: Option<<TF::Data as ToOwned>::Owned>,
    /* private fields */
}

Default implementation of Lexer

Public fields in this struct are intended to be used by embedded actions

Fields

interpreter: Option<Box<LexerATNSimulator>>

LexerATNSimulator instance of this lexer

input: Option<Input>

CharStream used by this lexer

token_start_char_index: isize

token_start_line: isize

token_start_column: isize

token_type: isize

Overrides token type emitted by lexer for current token

token: Option<TF::Tok>

Make it Some to override token that is currently being generated by lexer

channel: isize

Channel lexer is currently assigning tokens to

mode: usize

Mode lexer is currently in

text: Option<<TF::Data as ToOwned>::Owned>

Make it Some to override text for token that is currently being generated by lexer

Implementations

impl<'input, T, Input, TF> BaseLexer<'input, T, Input, TF>

where T: LexerRecog<'input, Self> + 'static, Input: CharStream<TF::From>, TF: TokenFactory<'input>,

pub fn get_char_index(&self) -> isize

Current position in input stream

pub fn get_text<'a>(&'a self) -> Cow<'a, TF::Data>

where 'input: 'a,

Current token text

pub fn set_text(&mut self, _text: <TF::Data as ToOwned>::Owned)

Used from lexer actions to override text of the token that will be emitted next

pub fn add_error_listener( &mut self, listener: Box<dyn ErrorListener<'input, Self>>, )

Add error listener

pub fn remove_error_listeners(&mut self)

Remove and drop all error listeners

pub fn new_base_lexer( input: Input, interpreter: LexerATNSimulator, recog: T, factory: &'input TF, ) -> Self

Creates new lexer instance

Trait Implementations

impl<'input, T, Input, TF> DerefMut for BaseLexer<'input, T, Input, TF>

where T: LexerRecog<'input, Self> + 'static, Input: CharStream<TF::From>, TF: TokenFactory<'input>,

fn deref_mut(&mut self) -> &mut Self::Target

Mutably dereferences the value.

impl<'input, T, Input, TF> Lexer<'input> for BaseLexer<'input, T, Input, TF>

where T: LexerRecog<'input, Self> + 'static, Input: CharStream<TF::From>, TF: TokenFactory<'input>,

type Input = Input

Concrete input stream used by this parser

fn input(&mut self) -> &mut Self::Input

Same as TokenStream::get_input_stream but returns concrete type instance important for proper inlining in hot code of LexerATNSimulator

fn set_channel(&mut self, v: isize)

Sets channel where current token will be pushed

fn push_mode(&mut self, m: usize)

Pushes current mode to internal mode stack and sets m as current lexer mode `pop_mode should be used to recover previous mode

fn pop_mode(&mut self) -> Option<usize>

Pops mode from internal mode stack

fn set_type(&mut self, t: isize)

Sets type of the current token Called from action to override token that will be emitted by lexer

fn set_mode(&mut self, m: usize)

Sets lexer mode discarding current one

fn more(&mut self)

Used to informs lexer that it should consider next token as a continuation of the current one

fn skip(&mut self)

Tells lexer to completely ignore and not emit current token.

impl<'input, T, Input, TF> TokenSource<'input> for BaseLexer<'input, T, Input, TF>

where T: LexerRecog<'input, Self> + 'static, Input: CharStream<TF::From>, TF: TokenFactory<'input>,

type TF = TF

TokenFactory this token source produce tokens with

fn next_token(&mut self) -> <Self::TF as TokenFactory<'input>>::Tok

Return a {@link Token} object from your input stream (usually a {@link CharStream}). Do not fail/return upon lexing error; keep chewing on the characters until you get a good one; errors are not passed through to the parser.

fn get_line(&self) -> isize

Get the line number for the current position in the input stream. The first line in the input is line 1.

fn get_char_position_in_line(&self) -> isize

Get the index into the current line for the current position in the input stream. The first character on a line has position 0.

fn get_input_stream(&mut self) -> Option<&mut dyn IntStream>

Returns underlying input stream

fn get_source_name(&self) -> String

Returns string identifier of underlying input e.g. file name

fn get_token_factory(&self) -> &'input TF

Gets the TokenFactory this token source is currently using for creating Token objects from the input.

impl<'input, T, Input, TF> Deref for BaseLexer<'input, T, Input, TF>

where T: LexerRecog<'input, Self> + 'static, Input: CharStream<TF::From>, TF: TokenFactory<'input>,

type Target = T

The resulting type after dereferencing.

fn deref(&self) -> &Self::Target

Dereferences the value.