Trait TokenSpliterator 

pub trait TokenSpliterator: Iterator<Item = (u16, u16, String, PosSpan)> {
    // Provided methods
    fn split_channel0<F>(self, f: F) -> TokenSplit<Self, F> 
       where Self: Sized,
             F: FnMut((u16, u16, String, PosSpan)) { ... }
    fn split_channels<F>(self, channel: u16, f: F) -> TokenSplit<Self, F> 
       where Self: Sized,
             F: FnMut((u16, u16, String, PosSpan)) { ... }
    fn keep_channel0(self) -> impl Iterator<Item = (u16, String, PosSpan)>
       where Self: Sized { ... }
    fn keep_channel(
        self,
        channel: u16,
    ) -> TokenSplit<Self, fn((u16, u16, String, PosSpan))> 
       where Self: Sized { ... }
}

Provided Methods

fn split_channel0<F>(self, f: F) -> TokenSplit<Self, F>

where Self: Sized, F: FnMut((u16, u16, String, PosSpan)),

Splits the token iterator out of the lexer (Item: (TokenId, ChannelId, String, PosSpan)) based on the channel ID:

• the default channel 0 is output as another iterator on (token, string, pos_span), suitable for the parser
• other channels are consummed by the closure f, which takes the parameters (token, channel, string, pos_span)

Example

let tokens = lexer.tokens().split_channel0(|(tok, ch, text, pos_span)|
    println!("TOKEN: channel {ch}, discarded, pos {pos_span}, Id {tok:?}, \"{text}\"")
);
let result = parser.parse_stream(&mut listener, tokens);

fn split_channels<F>(self, channel: u16, f: F) -> TokenSplit<Self, F>

where Self: Sized, F: FnMut((u16, u16, String, PosSpan)),

Splits the token iterator out of the lexer (Item: (TokenId, ChannelId, String, PosSpan)) based on the channel ID:

• the channel channel is output as another iterator on (token, string, pos_span), suitable for the parser
• other channels are consummed by the closure f, which takes the parameters (token, channel, string, pos_span)

Example

let tokens = lexer.tokens().split_channels(2, |(tok, ch, text, pos_span)|
    println!("TOKEN: channel {ch}, discarded, pos {pos_span}, Id {tok:?}, \"{text}\"")
);
let result = parser.parse_stream(&mut listener, tokens);

fn keep_channel0(self) -> impl Iterator<Item = (u16, String, PosSpan)>

where Self: Sized,

Filters the token iterator out of the lexer (Item: (TokenId, ChannelId, String, PosSpan)) based on the channel ID:

• the default channel 0 is output as another iterator on (token, string, pos_span), suitable for the parser
• other channels are discarded.

Example

let tokens = lexer.tokens().keep_channel0();
let result = parser.parse_stream(&mut listener, tokens);

fn keep_channel( self, channel: u16, ) -> TokenSplit<Self, fn((u16, u16, String, PosSpan))>

where Self: Sized,

Filters the token iterator out of the lexer (Item: (TokenId, ChannelId, String, PosSpan)) based on the channel ID:

• channel channel is output as another iterator on (token, string, pos_span), suitable for the parser
• other channels are discarded.

Example

let tokens = lexer.tokens().keep_channel(2);
let result = parser.parse_stream(&mut listener, tokens);

Implementors

impl<I> TokenSpliterator for I

where I: Iterator<Item = (u16, u16, String, PosSpan)>,