Trait CharStream 

pub trait CharStream<P>
where
    P: PosnInCharStream,
{
    // Required methods
    fn range_as_bytes(&self, ofs: usize, n: usize) -> &[u8] ;
    fn matches_bytes(&self, state: &P, s: &[u8]) -> bool;
    fn get_text_span(&self, span: &StreamCharSpan<P>) -> &str
       where P: PosnInCharStream;
    fn get_text(&self, start: P, end: P) -> &str;
    fn matches_str(&self, pos: &P, pat: &str) -> bool;
    fn peek_at(&self, state: &P) -> Option<char>;
    fn consumed(&self, state: P, num_chars: usize) -> P;

    // Provided methods
    fn do_while<F: Fn(usize, char) -> bool>(
        &self,
        state: P,
        ch: char,
        f: &F,
    ) -> (P, Option<(P, usize)>) { ... }
    fn fold<T, F: Fn(&Self, T, &P, usize, char) -> (T, Option<P>)>(
        &self,
        state: P,
        ch: char,
        acc: T,
        f: &F,
    ) -> (P, Option<(P, usize, T)>) { ... }
    fn consumed_char(&self, state: P, ch: char) -> P
       where P: PosnInCharStream { ... }
    unsafe fn consumed_newline(&self, state: P, num_bytes: usize) -> P
       where P: PosnInCharStream { ... }
    fn consumed_ascii_str(&self, state: P, s: &str) -> P
       where P: PosnInCharStream { ... }
    unsafe fn consumed_chars(
        &self,
        state: P,
        num_bytes: usize,
        num_chars: usize,
    ) -> P
       where P: PosnInCharStream { ... }
    fn commit_consumed(&self, _up_to: &P) { ... }
}

The CharStream trait allows a stream of char to provide extraa methods

Requires P : PosnInCharStream

Required Methods

fn range_as_bytes(&self, ofs: usize, n: usize) -> &[u8]

Retrieve a range of bytes from the stream

fn matches_bytes(&self, state: &P, s: &[u8]) -> bool

Return true if the content of the stream at ‘state’ matches the byte slice

fn get_text_span(&self, span: &StreamCharSpan<P>) -> &str

where P: PosnInCharStream,

Get the text between the start of a span (inclusive) and the end of the span (exclusive).

fn get_text(&self, start: P, end: P) -> &str

Get the text between the start (inclusive) and the end (exclusive).

fn matches_str(&self, pos: &P, pat: &str) -> bool

Match the text at the offset with a str; return true if it matches, else false

fn peek_at(&self, state: &P) -> Option<char>

Peek at the next character in the stream, returning None if the state is the end of the stream

fn consumed(&self, state: P, num_chars: usize) -> P

Move the stream state forward by the specified number of characters

The characters MUST NOT inclulde newlines

Provided Methods

fn do_while<F: Fn(usize, char) -> bool>( &self, state: P, ch: char, f: &F, ) -> (P, Option<(P, usize)>)

Steps along the stream starting at the provided state (and character) while the provided function returns true; the function is provided with the index and character (starting at 0 / ch), and it returns true if the token continues, otherwise false

If the first invocation of ‘f’ returns false then the token is said to not match, and ‘do_while’ returns the stream state and Ok(None).

If the first N (more than zero) invocations match then the result is the stream state after the matched characters, and Some(initial state, N)

This can be used to match whitespace (where N is probably discarded), or user ‘id’ values in a language. The text can be retrieved with the ‘get_text’ method

Examples found in repository

examples/calc.rs (line 95)

fn parse_value_fn(stream: &TextStream, state: TextPos, ch: char) -> CalcLexResult {
    let is_digit = |_, ch| ('0'..='9').contains(&ch);
    let (state, opt_x) = stream.do_while(state, ch, &is_digit);
    if let Some((start, _n)) = opt_x {
        let s = stream.get_text(start, state);
        let value: f64 = s.parse().unwrap();
        Ok(Some((state, CalcToken::Value(value))))
    } else {
        Ok(None)
    }
}

//fi parse_whitespace_fn
/// Parser function to return a Token if whitespace
fn parse_whitespace_fn(stream: &TextStream, state: TextPos, ch: char) -> CalcLexResult {
    let is_whitespace = |_n, ch| ch == ' ' || ch == '\t' || ch == '\n';
    let (state, opt_x) = stream.do_while(state, ch, &is_whitespace);
    if let Some((_start, _n)) = opt_x {
        Ok(Some((state, CalcToken::Whitespace)))
    } else {
        Ok(None)
    }
}

examples/simple.rs (lines 70-72)

    pub fn parse_comment_line<L>(
        stream: &L,
        state: L::State,
        ch: char,
    ) -> LexerParseResult<P, Self, L::Error>
    where
        L: CharStream<P>,
        L: Lexer<Token = Self, State = P>,
    {
        match stream.do_while(state, ch, &|n, ch| {
            ((n < 2) && (ch == '/')) || ((n >= 2) && ch != '\n')
        }) {
            (state, Some((start, _n))) => {
                let span = StreamCharSpan::new(start, state);
                Ok(Some((state, SimpleToken::CommentLine(span))))
            }
            (_, None) => Ok(None),
        }
    }

    //fp parse_digits
    pub fn parse_digits<L>(
        stream: &L,
        state: L::State,
        ch: char,
    ) -> LexerParseResult<P, Self, L::Error>
    where
        L: CharStream<P>,
        L: Lexer<Token = Self, State = P>,
    {
        match stream.do_while(state, ch, &|_, ch| ch.is_ascii_digit()) {
            (state, Some((start, _n))) => {
                let span = StreamCharSpan::new(start, state);
                Ok(Some((state, SimpleToken::Digits(span))))
            }
            (_, None) => Ok(None),
        }
    }

    //fp parse_whitespace
    pub fn parse_whitespace<L>(
        stream: &L,
        state: L::State,
        ch: char,
    ) -> LexerParseResult<P, Self, L::Error>
    where
        L: CharStream<P>,
        L: Lexer<Token = Self, State = P>,
    {
        match stream.do_while(state, ch, &|_, ch| (ch == ' ' || ch == '\t')) {
            (state, Some((start, _))) => {
                let span = StreamCharSpan::new(start, state);
                Ok(Some((state, SimpleToken::Whitespace(span))))
            }
            (_, None) => Ok(None),
        }
    }

    //fp parse_id
    pub fn parse_id<L, F1, F2>(
        stream: &L,
        state: L::State,
        ch: char,
        is_id_start: F1,
        is_id: F2,
    ) -> LexerParseResult<P, Self, L::Error>
    where
        L: CharStream<P>,
        L: Lexer<Token = Self, State = P>,
        F1: Fn(char) -> bool,
        F2: Fn(char) -> bool,
    {
        match stream.do_while(state, ch, &|n, ch| {
            (n == 0 && is_id_start(ch)) || ((n > 0) && is_id(ch))
        }) {
            (state, Some((start, _))) => {
                let span = StreamCharSpan::new(start, state);
                Ok(Some((state, SimpleToken::Id(span))))
            }
            (_, None) => Ok(None),
        }
    }

fn fold<T, F: Fn(&Self, T, &P, usize, char) -> (T, Option<P>)>( &self, state: P, ch: char, acc: T, f: &F, ) -> (P, Option<(P, usize, T)>)

Steps along the stream starting at the provided state, character and accumulator value while the provided function returns (true, new accumulator); the function is provided with the latest accumulator, index, character (starting at 0 / ch), and it returns true and a new accumulator if the token continues, otherwise false and the final accumulator value

If the first invocation of ‘f’ returns false then the token is said to not match, and ‘fold’ returns the stream state and Ok(None).

If the first N (more than zero) invocations match then the result is the stream state after the matched characters, and Some(initial state, N, final accumulator)

This can be used to accumulate significant state about a token as it is parsed, in excess of the simple number of characters.

fn consumed_char(&self, state: P, ch: char) -> P

where P: PosnInCharStream,

Get a stream state after consuming the specified character at its current state

Examples found in repository

examples/calc.rs (line 85)

fn parse_char_fn(stream: &TextStream, state: TextPos, ch: char) -> CalcLexResult {
    if let Some(t) = {
        match ch {
            '+' => Some(CalcToken::Op(CalcOp::Plus)),
            '-' => Some(CalcToken::Op(CalcOp::Minus)),
            '*' => Some(CalcToken::Op(CalcOp::Times)),
            '/' => Some(CalcToken::Op(CalcOp::Divide)),
            '(' => Some(CalcToken::Open),
            ')' => Some(CalcToken::Close),
            _ => None,
        }
    } {
        Ok(Some((stream.consumed_char(state, ch), t)))
    } else {
        Ok(None)
    }
}

examples/simple.rs (line 56)

    pub fn parse_char<L>(
        stream: &L,
        state: L::State,
        ch: char,
    ) -> LexerParseResult<P, Self, L::Error>
    where
        L: CharStream<P>,
        L: Lexer<Token = Self, State = P>,
    {
        let pos = state;
        match ch {
            '\n' => Ok(Some((stream.consumed(state, 1), Self::Newline(pos)))),
            '(' | '[' | '{' => Ok(Some((stream.consumed(state, 1), Self::OpenBra(pos, ch)))),
            ')' | ']' | '}' => Ok(Some((stream.consumed(state, 1), Self::CloseBra(pos, ch)))),
            ch => Ok(Some((stream.consumed_char(state, ch), Self::Char(pos, ch)))),
        }
    }

unsafe fn consumed_newline(&self, state: P, num_bytes: usize) -> P

where P: PosnInCharStream,

Get a stream state after consuming a newline at its current state

Safety

num_bytes must correspond to the number of bytes that the newline character consists of, and state must point to the bytes offset of that character

fn consumed_ascii_str(&self, state: P, s: &str) -> P

where P: PosnInCharStream,

Get the state after consuming a particular ascii string without newlines

This is safe as there is no unsafe handling of byte offsets within state; however, there is no check that the provided string is ASCII and that it does not contain newlines. If these API rules are broke then the lie and column held by state may be incorrect (which is not unsafe, but potentially a bug)

unsafe fn consumed_chars( &self, state: P, num_bytes: usize, num_chars: usize, ) -> P

where P: PosnInCharStream,

Become the span after consuming a particular string of known character length

Safety

num_bytes must correspond to the number of bytes that ‘num_chars’ indicates start at state. If this constraint is not met then the byte offset indicated by the returned value may not correspond to a UTF8 character boundary within the stream.

fn commit_consumed(&self, _up_to: &P)

Invoked by the Lexer to indicate that the stream has been consumed up to a certain point, and that (for parsing) no state earlier in the stream will be requested in the future

A truly streaming source can drop earlier data in the stream if this fits the application

Dyn Compatibility

This trait is not dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety", so this trait is not object safe.

Implementors

impl<'a, P, T, E> CharStream<P> for LexerOfStr<'a, P, T, E>

where P: PosnInCharStream,