lang_pt 0.1.2

A parser tool to generate recursive descent top down parser.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144

use super::{LexemeLogger, Mapper, ThunkMapper};
use crate::{
    Code, Log,
    ILexeme, Lex,
};
use once_cell::unsync::OnceCell;
use std::collections::HashMap;

impl<TS: ILexeme> Mapper<TS> {
    /// Create a new [Mapper] utility.
    /// ## Arguments
    /// * 'lexeme' - Another lexer utility which implement [ILexeme] trait.
    /// * 'fields' - A [Vec] of tuples of mappable string values, and their token values.
    pub fn new(lexeme: TS, fields: Vec<(&str, TS::Token)>) -> Result<Self, String> {
        let mut s = Self {
            lexeme,
            fields: HashMap::new(),
            log: OnceCell::new(),
        };

        s.extend_fields(fields).map(|_| s)
    }

    /// Set a log label to debug the lexeme.
    /// Based on the level of the [Log], the lexeme will debug the lexeme result.
    pub fn set_log(&self, log: Log<&'static str>) -> Result<(), String> {
        self.log
            .set(log)
            .map_err(|err| format!("Log label {} is already assigned.", err))
    }

    pub fn extend_fields(&mut self, fields: Vec<(&str, TS::Token)>) -> Result<(), String> {
        for (keyword, token) in fields {
            if let Some(token) = self
                .fields
                .insert(keyword.bytes().collect::<Vec<u8>>(), token)
            {
                return Err(format!(
                    "{:?} is already been used with token {:?}",
                    keyword, token
                ));
            };
        }
        Ok(())
    }
}

impl<TLexer: ILexeme> LexemeLogger for Mapper<TLexer> {
    fn log_cell(&self) -> &OnceCell<crate::Log<&'static str>> {
        &self.log
    }
}
impl<TLexer: ILexeme> ILexeme for Mapper<TLexer> {
    type Token = TLexer::Token;
    type State = TLexer::State;

    fn consume(
        &self,
        code: &Code,
        pointer: usize,
        tokenized_stream: &Vec<Lex<Self::Token>>,
        info: &mut Vec<Self::State>,
    ) -> Option<Lex<Self::Token>> {
        let result = self.lexeme.consume(code, pointer, tokenized_stream, info);
        self.log_result(pointer, code, &result);

        result.map(|mut lex_data| {
            let code_part = &code.value[lex_data.start..lex_data.end];
            if let Some(token) = self.fields.get(code_part) {
                lex_data.token = *token;
            }
            lex_data
        })
    }
    fn get_grammar_field(&self) -> Vec<(<TLexer as ILexeme>::Token, String)> {
        let mut v: Vec<(TLexer::Token, String)> = self
            .fields
            .iter()
            .map(|(s, t)| {
                let s = unsafe { std::str::from_utf8_unchecked(s) };

                (*t, format!("{:?}", s))
            })
            .collect();

        v.extend(self.lexeme.get_grammar_field().into_iter());
        v
    }
}

impl<TL: ILexeme, TF: Fn(&Lex<TL::Token>, &[u8], &Vec<Lex<TL::Token>>) -> Option<TL::Token>>
    ThunkMapper<TL, TF>
{
    pub fn new(lexeme: TL, thunk: TF) -> Self {
        Self {
            lexeme,
            thunk,
            log: OnceCell::new(),
        }
    }
    pub fn set_log(&self, log: Log<&'static str>) -> Result<(), String> {
        self.log
            .set(log)
            .map_err(|err| format!("Log label {} is already assigned.", err))
    }
}

impl<TL: ILexeme, TF: Fn(&Lex<TL::Token>, &[u8], &Vec<Lex<TL::Token>>) -> Option<TL::Token>>
    LexemeLogger for ThunkMapper<TL, TF>
{
    fn log_cell(&self) -> &OnceCell<crate::Log<&'static str>> {
        &self.log
    }
}
impl<TL: ILexeme, TF: Fn(&Lex<TL::Token>, &[u8], &Vec<Lex<TL::Token>>) -> Option<TL::Token>> ILexeme
    for ThunkMapper<TL, TF>
{
    type Token = TL::Token;
    type State = TL::State;

    fn consume(
        &self,
        code: &Code,
        pointer: usize,
        tokenized_stream: &Vec<Lex<Self::Token>>,
        info: &mut Vec<Self::State>,
    ) -> Option<Lex<Self::Token>> {
        let result = self.lexeme.consume(code, pointer, tokenized_stream, info);
        self.log_result(pointer, code, &result);
        result.map(|mut lex| {
            match (self.thunk)(&lex, &code.value, tokenized_stream) {
                Some(token) => {
                    lex.token = token;
                }
                None => {}
            };
            lex
        })
    }

    fn get_grammar_field(&self) -> Vec<(TL::Token, String)> {
        self.lexeme.get_grammar_field()
    }
}