simple_lexer_bootstrap/
lib.rs

1use std::ops::AddAssign;
2use uuid::Uuid;
3use regular_expression_bootstrap::StateGenerator;
4
5#[macro_use]
6mod util;
7mod lexer;
8
9pub use crate::lexer::{
10    Token,
11    Lexer
12};
13
14#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
15struct TokenState<T> {
16    uuid: u128,
17    sequence_number: u128,
18    token_kind: Option<T>,
19}
20
21impl<T: Clone> TokenState<T> {
22    fn new(token_kind: Option<T>) -> TokenState<T> {
23        TokenState { uuid: Uuid::new_v4().as_u128(), sequence_number: 0, token_kind }
24    }
25
26    fn token_kind(&self) -> &Option<T> {
27        &self.token_kind
28    }
29
30    fn clear_token_kind(&mut self) {
31        self.token_kind = None;
32    }
33}
34
35impl<T> AddAssign<u128> for TokenState<T> {
36    fn add_assign(&mut self, other: u128) {
37        self.sequence_number += other;
38    }
39}
40
41struct TokenStateGenerator<T> {
42    token_state: TokenState<T>,
43    final_enabled: bool,
44}
45
46impl<T: Clone> TokenStateGenerator<T> {
47    pub fn new(token: Option<T>) -> TokenStateGenerator<T> {
48        TokenStateGenerator { token_state: TokenState::new(token), final_enabled: true }
49    }
50
51    fn next_final_enabled(&mut self) -> TokenState<T> {
52        let next = self.token_state.clone();
53        self.token_state += 1;
54        next
55    }
56
57    fn next_final_disabled(&mut self) -> TokenState<T> {
58        let mut next = self.token_state.clone();
59        self.token_state += 1;
60        next.clear_token_kind();
61        next
62    }
63}
64
65impl<T: Clone> StateGenerator for TokenStateGenerator<T> {
66    type State = TokenState<T>;
67
68    fn next_initial(&mut self) -> TokenState<T> {
69        let mut next = self.token_state.clone();
70        self.token_state += 1;
71        next.clear_token_kind();
72        next
73    }
74
75    fn next_final(&mut self) -> TokenState<T> {
76        if self.final_enabled {
77            self.next_final_enabled()
78        } else {
79            self.next_final_disabled()
80        }
81    }
82
83    fn disable_final(&mut self) -> &mut TokenStateGenerator<T> {
84        self.final_enabled = false;
85        self
86    }
87
88    fn enable_final(&mut self) -> &mut TokenStateGenerator<T> {
89        self.final_enabled = true;
90        self
91    }
92}
93
simple_lexer_bootstrap/lib.rs

simple_lexer_bootstrap/
lib.rs
