lexgen 0.16.0

A fully-featured lexer generator implemented as a proc macro
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

use super::{State, StateIdx, DFA};
use crate::collections::Map;
use crate::nfa::AcceptingState;

#[derive(Debug)]
pub enum Trans<A> {
    Accept(Vec<AcceptingState<A>>),
    Trans(StateIdx),
}

/// Removes accepting states with no transitions, makes the transitions to those states accepting.
pub fn simplify<K, A: Clone>(
    dfa: DFA<StateIdx, A>,
    dfa_state_indices: &mut Map<K, StateIdx>,
) -> DFA<Trans<A>, A> {
    let mut empty_states: Vec<(StateIdx, Vec<AcceptingState<A>>)> = vec![];

    let mut non_empty_states: Vec<(StateIdx, State<StateIdx, A>)> = vec![];

    for (state_idx, state) in dfa.into_state_indices() {
        if state.has_no_transitions() && !state.initial {
            empty_states.push((state_idx, state.accepting));
        } else {
            non_empty_states.push((state_idx, state));
        }
    }

    for (_, t) in dfa_state_indices.iter_mut() {
        let idx = match empty_states.binary_search_by(|(state_idx, _)| state_idx.cmp(t)) {
            Ok(idx) | Err(idx) => idx,
        };
        *t = t.map(|i| i - idx);
    }

    let map_transition = |t: StateIdx| -> Trans<A> {
        match empty_states.binary_search_by(|(state_idx, _action)| state_idx.cmp(&t)) {
            Ok(idx) => Trans::Accept(empty_states[idx].1.clone()),
            Err(idx) => Trans::Trans(t.map(|i| i - idx)),
        }
    };

    let new_states: Vec<State<Trans<A>, A>> = non_empty_states
        .into_iter()
        .map(|(_state_idx, state)| {
            let State {
                initial,
                char_transitions,
                range_transitions,
                any_transition,
                end_of_input_transition,
                accepting,
                predecessors,
                backtrack,
            } = state;

            let char_transitions = char_transitions
                .into_iter()
                .map(|(char, next)| (char, map_transition(next)))
                .collect();

            let range_transitions = range_transitions.map(map_transition);

            let any_transition = any_transition.map(map_transition);

            let end_of_input_transition = end_of_input_transition.map(map_transition);

            let predecessors = predecessors
                .into_iter()
                .map(|pred| match map_transition(pred) {
                    Trans::Trans(pred) => pred,
                    _ => {
                        // This pass should only remove nodes without successors, so it's a bug if
                        // we remove a predecessor
                        panic!("Predecessor of a state is removed in simplification")
                    }
                })
                .collect();

            State {
                initial,
                char_transitions,
                range_transitions,
                any_transition,
                end_of_input_transition,
                accepting,
                predecessors,
                backtrack,
            }
        })
        .collect();

    DFA::from_states(new_states)
}