lexington 0.3.0

A very simple library for lexing / parsing
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

use crate::{Token};
use crate::util::{ResetChars};
use crate::scanner::Scanner;

/// =============================================================================
/// Lexer
/// =============================================================================

pub struct Lexer<'a,S:Scanner> {
    iter: ResetChars<'a>,
    rules: S
}

impl<'a,S:Scanner> Lexer<'a,S> {
    pub fn new(input: &'a str, rules: S) -> Self {
        let iter = ResetChars::new(input.chars());
        Self{iter,rules}
    }
}

impl<'a,S:Scanner<Item=char>> Iterator for Lexer<'a,S>
{
    type Item = Token<S::Token>;
    
    fn next(&mut self) -> Option<Self::Item> {
        // Compute start offset
        let start = self.iter.byte_offset();
        // See what we've got
        match self.rules.scan(&mut self.iter) {
            Some(t) => {               
                // Compute end offset
                let end = self.iter.byte_offset();
                // Reset iterator
                self.iter.reset();                
                // Done
                Some(Token::new(t,start..end))
            }
            None => None
        }
    }
}