zz_validator/
token.rs

1/// -----------------------------
2/// Tokenizer
3/// -----------------------------
4#[derive(Debug, Clone, PartialEq)]
5pub enum Token {
6    Ident(String),
7    Number(String), // 数字统一存为字符串，包括科学计数法
8    Colon,
9    Comma,
10    LParen,
11    RParen,
12    LBracket,
13    RBracket,
14    Question,
15    Lt,
16    Gt,
17    Enum,
18    Equal,
19    Pipe,
20}
21
22pub fn tokenize(input: &str) -> Result<Vec<Token>, String> {
23    let mut tokens = Vec::new();
24    let mut chars = input.chars().peekable();
25
26    while let Some(&ch) = chars.peek() {
27        match ch {
28            '(' => {
29                tokens.push(Token::LParen);
30                chars.next();
31            }
32            ')' => {
33                tokens.push(Token::RParen);
34                chars.next();
35            }
36            '[' => {
37                tokens.push(Token::LBracket);
38                chars.next();
39            }
40            ']' => {
41                tokens.push(Token::RBracket);
42                chars.next();
43            }
44            '<' => {
45                tokens.push(Token::Lt);
46                chars.next();
47            }
48            '>' => {
49                tokens.push(Token::Gt);
50                chars.next();
51            }
52            ',' => {
53                tokens.push(Token::Comma);
54                chars.next();
55            }
56            '?' => {
57                tokens.push(Token::Question);
58                chars.next();
59            }
60            ':' => {
61                tokens.push(Token::Colon);
62                chars.next();
63            }
64            '=' => {
65                tokens.push(Token::Equal);
66                chars.next();
67            }
68            '|' => {
69                tokens.push(Token::Pipe);
70                chars.next();
71            }
72
73            // 新逻辑：支持 + / - 开头
74            '0'..='9' | '.' | '+' | '-' => {
75                let mut num_str = String::new();
76                // 如果开头是 + 或 -，先记录并移动
77                if let Some(&c) = chars.peek() {
78                    if c == '+' || c == '-' {
79                        num_str.push(c);
80                        chars.next();
81                    }
82                }
83
84                while let Some(&c) = chars.peek() {
85                    // 数字主体部分，包括科学计数法 e/E 和可能的 +/-
86                    if
87                        c.is_ascii_digit() ||
88                        c == '.' ||
89                        c == 'e' ||
90                        c == 'E' ||
91                        c == '+' ||
92                        c == '-'
93                    {
94                        num_str.push(c);
95                        chars.next();
96                    } else {
97                        break;
98                    }
99                }
100
101                // 尝试解析为 f64 验证格式是否正确
102                if num_str.parse::<f64>().is_err() {
103                    return Err(format!("Invalid number '{}'", num_str));
104                }
105
106                tokens.push(Token::Number(num_str));
107            }
108            '"' => {
109                chars.next(); // skip opening quote
110                let mut s = String::new();
111                while let Some(&c) = chars.peek() {
112                    if c == '"' {
113                        chars.next(); // skip closing quote
114                        break;
115                    }
116                    // 支持转义字符
117                    if c == '\\' {
118                        chars.next();
119                        if let Some(&esc) = chars.peek() {
120                            let esc_ch = match esc {
121                                'n' => '\n',
122                                'r' => '\r',
123                                't' => '\t',
124                                '"' => '"',
125                                '\\' => '\\',
126                                other => other,
127                            };
128                            s.push(esc_ch);
129                            chars.next();
130                        }
131                    } else {
132                        s.push(c);
133                        chars.next();
134                    }
135                }
136                tokens.push(Token::Ident(s)); // 字符串作为 Ident 保存
137            }
138            c if c.is_alphanumeric() || c == '_' => {
139                let mut ident = String::new();
140                while let Some(&c2) = chars.peek() {
141                    if c2.is_alphanumeric() || c2 == '_' {
142                        ident.push(c2);
143                        chars.next();
144                    } else {
145                        break;
146                    }
147                }
148                tokens.push(Token::Ident(ident));
149            }
150            c if c.is_whitespace() => {
151                chars.next();
152            }
153            _ => {
154                return Err(format!("Unexpected char '{}'", ch));
155            }
156        }
157    }
158
159    Ok(tokens)
160}
zz_validator/token.rs

zz_validator/
token.rs
