1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
use alloc::string::ToString;
use alloc::vec;
use alloc::vec::Vec;
use regex::Regex;
use lazy_static::*;
mod number;
mod operator;
mod text;
mod whitespace;
mod field;
mod percent;
mod atom;
mod time;
mod money;
mod comment;
mod month;
mod timezone;
use crate::SmartCalcConfig;
pub use self::time::time_regex_parser;
pub use self::number::number_regex_parser;
pub use self::percent::percent_regex_parser;
pub use self::money::money_regex_parser;
pub use self::text::text_regex_parser;
pub use self::field::field_regex_parser;
pub use self::atom::{atom_regex_parser, get_atom};
pub use self::whitespace::whitespace_regex_parser;
pub use self::comment::comment_regex_parser;
pub use self::timezone::timezone_regex_parser;
pub use self::month::month_parser;
pub use self::operator::operator_regex_parser;
use super::Tokinizer;
pub type RegexParser = fn(config: &SmartCalcConfig, tokinizer: &mut Tokinizer, group_item: &[Regex]);
pub type Parser = fn(config: &SmartCalcConfig, tokinizer: &mut Tokinizer, data: &str);
lazy_static! {
pub static ref TOKEN_REGEX_PARSER: Vec<(&'static str, RegexParser)> = {
let m = vec![
("comment", comment_regex_parser as RegexParser),
("field", field_regex_parser as RegexParser),
("money", money_regex_parser as RegexParser),
("atom", atom_regex_parser as RegexParser),
("percent", percent_regex_parser as RegexParser),
("timezone", timezone_regex_parser as RegexParser),
("time", time_regex_parser as RegexParser),
("number", number_regex_parser as RegexParser),
("text", text_regex_parser as RegexParser),
("whitespace", whitespace_regex_parser as RegexParser),
("operator", operator_regex_parser as RegexParser)];
m
};
}
lazy_static! {
pub static ref LANGUAGE_BASED_TOKEN_PARSER: Vec<Parser> = {
let m = vec![month_parser as Parser];
m
};
}
pub fn regex_tokinizer(tokinizer: &mut Tokinizer) {
for (key, func) in TOKEN_REGEX_PARSER.iter() {
if let Some(items) = tokinizer.config.token_parse_regex.get(&key.to_string()) {
func(tokinizer.config, tokinizer, items)
}
}
tokinizer.cleanup_token_infos();
}
pub fn language_tokinizer(tokinizer: &mut Tokinizer) {
let lowercase_data = tokinizer.data.to_lowercase();
for func in LANGUAGE_BASED_TOKEN_PARSER.iter() {
func(tokinizer.config, tokinizer, &lowercase_data);
}
tokinizer.cleanup_token_infos();
}
