lexgen 0.16.0

A fully-featured lexer generator implemented as a proc macro
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

//! Stuff related to right contexts
//!
//! A right context is a limited version of lookahead. A rule can have at most one right context.
//! When a rule has right context, after the regex for the rule matches, we run the DFA for the
//! right context with cloned input stream. Only if it matches we consider the the rule as a match.
//! This provides a simple "lookahead" support, which should be good enough when lexing programming
//! languages.

use crate::ast::{Regex, Var};
use crate::collections::Map;
// use crate::dfa::simplify::{simplify, Trans};
use crate::dfa::{StateIdx, DFA};
use crate::nfa::NFA;
use crate::nfa_to_dfa::nfa_to_dfa;

#[derive(Debug)]
pub struct RightCtxDFAs<S> {
    dfas: Vec<DFA<S, ()>>,
}

#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct RightCtxIdx(usize);

impl RightCtxIdx {
    pub fn as_usize(&self) -> usize {
        self.0
    }
}

impl<A> RightCtxDFAs<A> {
    pub fn new() -> Self {
        RightCtxDFAs { dfas: vec![] }
    }

    pub fn iter(&self) -> impl Iterator<Item = (RightCtxIdx, &DFA<A, ()>)> {
        self.dfas
            .iter()
            .enumerate()
            .map(|(i, dfa)| (RightCtxIdx(i), dfa))
    }
}

impl RightCtxDFAs<StateIdx> {
    pub fn new_right_ctx(&mut self, bindings: &Map<Var, Regex>, right_ctx: &Regex) -> RightCtxIdx {
        let idx = self.dfas.len();

        let mut nfa: NFA<()> = NFA::new();
        nfa.add_regex(bindings, right_ctx, None, ());

        let dfa = nfa_to_dfa(&nfa);
        self.dfas.push(dfa);

        RightCtxIdx(idx)
    }

    #[cfg(test)]
    pub fn get(&self, right_ctx: &RightCtxIdx) -> &DFA<StateIdx, ()> {
        &self.dfas[right_ctx.as_usize()]
    }

    /*
    pub fn simplify(self) -> RightCtxDFAs<Trans<()>> {
        RightCtxDFAs {
            dfas: self
                .dfas
                .into_iter()
                .map(|dfa| simplify::<(), ()>(dfa, &mut Default::default()))
                .collect(),
        }
    }
    */
}