Struct TermLexer 

pub struct TermLexer<I>
where
    I: FusedIterator<Item = u8>,
{
    pub opers: OperDefs,
    /* private fields */
}

Fields

opers: OperDefs

Implementations

impl<I> TermLexer<I>

where I: FusedIterator<Item = u8>,

pub fn try_new(input: I, opers: Option<OperDefs>) -> Result<Self>

Trait Implementations

impl<I> Lexer<Arena> for TermLexer<I>

where I: FusedIterator<Item = u8>,

type Input = I

The input stream type producing bytes for lexing.

type LexerData = LexData

The lexer data providing DFA tables and rule definitions. Must be provided by the implementer and must be the struct generated by parlex-gen’s lexer generator, Alex.

type Token = TermToken

The token type produced by the lexer. Must be provided by the implementor.

fn ctx(&self) -> &LexerCtx<Self::Input, Self::LexerData, Self::Token>

Returns a shared reference to the lexer context. Must be provided by the implementor and return &LexerCtx.

fn ctx_mut( &mut self, ) -> &mut LexerCtx<Self::Input, Self::LexerData, Self::Token>

Returns a mutable reference to the lexer context. Must be provided by the implementor and return &mut LexerCtx.

fn action( &mut self, arena: &mut Arena, rule: <Self::LexerData as LexerData>::LexerRule, ) -> Result<()>

Executes a lexer action based on the matched rule and user data. Must be provided by the implementor.

fn stats(&self) -> LexerStats

Returns current lexer statistics.

fn try_collect(&mut self, user_data: &mut U) -> Result<Vec<Self::Token>, Error>

Collects all tokens from the input until exhaustion.

fn try_next(&mut self, user_data: &mut U) -> Result<Option<Self::Token>, Error>

Attempts to fetch the next token from the input stream.

fn accum(&mut self)

Switches on accumulation of bytes into the buffer.

fn begin(&mut self, mode: <Self::LexerData as LexerData>::LexerMode)

Switches the lexer to a different mode.

fn yield_token(&mut self, token: Self::Token)

Emits a token into the output queue.

fn clear(&mut self)

Clears the current buffer and disables accumulation.

fn take_bytes(&mut self) -> Vec<u8>

Takes accumulated bytes from the main buffer.

fn take_bytes2(&mut self) -> Vec<u8>

Takes accumulated bytes from the secondary buffer.

fn take_str(&mut self) -> Result<SmartString<LazyCompact>, Error>

Takes accumulated bytes from the main buffer and converts them into a UTF-8 string.

fn take_str2(&mut self) -> Result<SmartString<LazyCompact>, Error>

Takes accumulated bytes from the secondary buffer and converts them into a UTF-8 string.

fn extend_buffer2_with_buffer(&mut self)

Extends the secondary buffer with the contents of the main buffer.