promql_parser/parser/
token.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use lazy_static::lazy_static;
16use std::collections::HashMap;
17use std::fmt;
18
19lrlex::lrlex_mod!("token_map");
20pub use token_map::*;
21
22pub type TokenId = u8;
23
24#[derive(Debug, Copy, Clone, PartialEq, Eq)]
25pub struct TokenType(TokenId);
26
27#[cfg(feature = "ser")]
28impl serde::Serialize for TokenType {
29    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
30    where
31        S: serde::Serializer,
32    {
33        serializer.serialize_str(token_display(self.0))
34    }
35}
36
37lazy_static! {
38    static ref KEYWORDS: HashMap<&'static str, TokenId> =
39        [
40            // Operators.
41            ("and", T_LAND),
42            ("or", T_LOR),
43            ("unless", T_LUNLESS),
44            ("atan2", T_ATAN2),
45
46            // Aggregators.
47            ("sum", T_SUM),
48            ("avg", T_AVG),
49            ("count", T_COUNT),
50            ("min", T_MIN),
51            ("max", T_MAX),
52            ("group", T_GROUP),
53            ("stddev", T_STDDEV),
54            ("stdvar", T_STDVAR),
55            ("topk", T_TOPK),
56            ("bottomk", T_BOTTOMK),
57            ("count_values", T_COUNT_VALUES),
58            ("quantile", T_QUANTILE),
59            ("limitk", T_LIMITK),
60            ("limit_ratio", T_LIMIT_RATIO),
61
62            // Keywords.
63            ("offset", T_OFFSET),
64            ("by", T_BY),
65            ("without", T_WITHOUT),
66            ("on", T_ON),
67            ("ignoring", T_IGNORING),
68            ("group_left", T_GROUP_LEFT),
69            ("group_right", T_GROUP_RIGHT),
70            ("bool", T_BOOL),
71            ("smoothed", T_SMOOTHED),
72            ("anchored", T_ANCHORED),
73
74            // Preprocessors.
75            ("start", T_START),
76            ("end", T_END),
77
78            // Special numbers.
79            ("inf", T_NUMBER),
80            ("nan", T_NUMBER),
81        ].into_iter().collect();
82}
83
84/// this is for debug so far, maybe pretty feature in the future.
85#[allow(dead_code)]
86pub(crate) fn token_display(id: TokenId) -> &'static str {
87    match id {
88        // Token.
89        T_EQL => "=",
90        T_BLANK => "_",
91        T_COLON => ":",
92        T_COMMA => ",",
93        T_COMMENT => "#",
94        T_DURATION => "[du]",
95        T_EOF => "<eof>",
96        T_ERROR => "{Err}",
97        T_IDENTIFIER => "{ID}",
98        T_LEFT_BRACE => "{",
99        T_LEFT_BRACKET => "[",
100        T_LEFT_PAREN => "(",
101        T_OPEN_HIST => "{{",
102        T_CLOSE_HIST => "}}",
103        T_METRIC_IDENTIFIER => "{Metric_ID}",
104        T_NUMBER => "{Num}",
105        T_RIGHT_BRACE => "}",
106        T_RIGHT_BRACKET => "]",
107        T_RIGHT_PAREN => ")",
108        T_SEMICOLON => ",",
109        T_SPACE => "<space>",
110        T_STRING => "{Str}",
111        T_TIMES => "x",
112
113        // Operators.
114        T_OPERATORS_START => "operators_start",
115        T_ADD => "+",
116        T_DIV => "/",
117        T_EQLC => "==",
118        T_EQL_REGEX => "=~",
119        T_GTE => ">=",
120        T_GTR => ">",
121        T_LAND => "and",
122        T_LOR => "or",
123        T_LSS => "<",
124        T_LTE => "<=",
125        T_LUNLESS => "unless",
126        T_MOD => "%",
127        T_MUL => "*",
128        T_NEQ => "!=",
129        T_NEQ_REGEX => "!~",
130        T_POW => "^",
131        T_SUB => "-",
132        T_AT => "@",
133        T_ATAN2 => "atan2",
134        T_OPERATORS_END => "operators_end",
135
136        // Aggregators.
137        T_AGGREGATORS_START => "aggregators_start",
138        T_AVG => "avg",
139        T_BOTTOMK => "bottomk",
140        T_COUNT => "count",
141        T_COUNT_VALUES => "count_values",
142        T_GROUP => "group",
143        T_MAX => "max",
144        T_MIN => "min",
145        T_QUANTILE => "quantile",
146        T_STDDEV => "stddev",
147        T_STDVAR => "stdvar",
148        T_SUM => "sum",
149        T_TOPK => "topk",
150        T_LIMITK => "limitk",
151        T_LIMIT_RATIO => "limit_ratio",
152        T_AGGREGATORS_END => "aggregators_end",
153
154        // Keywords.
155        T_KEYWORDS_START => "keywords_start",
156        T_BOOL => "bool",
157        T_BY => "by",
158        T_GROUP_LEFT => "group_left",
159        T_GROUP_RIGHT => "group_right",
160        T_IGNORING => "ignoring",
161        T_OFFSET => "offset",
162        T_SMOOTHED => "smoothed",
163        T_ANCHORED => "anchored",
164        T_ON => "on",
165        T_WITHOUT => "without",
166        T_KEYWORDS_END => "keywords_end",
167
168        // Preprocessors.
169        T_PREPROCESSOR_START => "preprocessor_start",
170        T_START => "start",
171        T_END => "end",
172        T_STEP => "step",
173        T_PREPROCESSOR_END => "preprocessors_end",
174
175        T_STARTSYMBOLS_START
176        | T_START_METRIC
177        | T_START_SERIES_DESCRIPTION
178        | T_START_EXPRESSION
179        | T_START_METRIC_SELECTOR
180        | T_STARTSYMBOLS_END => "not used",
181
182        _ => "unknown token",
183    }
184}
185
186/// This is a list of all keywords in PromQL.
187/// When changing this list, make sure to also change
188/// maybe_label grammar rule in generated parser
189/// to avoid misinterpretation of labels as keywords.
190pub(crate) fn get_keyword_token(s: &str) -> Option<TokenId> {
191    KEYWORDS.get(s).copied()
192}
193
194#[derive(Debug, Clone, PartialEq, Eq)]
195pub struct Token {
196    pub id: TokenType,
197    pub val: String,
198}
199
200impl Token {
201    pub fn new(id: TokenId, val: String) -> Self {
202        Self {
203            id: TokenType(id),
204            val,
205        }
206    }
207
208    pub fn id(&self) -> TokenId {
209        self.id.id()
210    }
211}
212
213impl TokenType {
214    pub fn new(id: TokenId) -> Self {
215        Self(id)
216    }
217
218    pub fn id(&self) -> TokenId {
219        self.0
220    }
221
222    pub fn is_aggregator(&self) -> bool {
223        self.0 > T_AGGREGATORS_START && self.0 < T_AGGREGATORS_END
224    }
225
226    pub fn is_aggregator_with_param(&self) -> bool {
227        matches!(self.0, T_TOPK | T_BOTTOMK | T_COUNT_VALUES | T_QUANTILE)
228    }
229
230    pub fn is_comparison_operator(&self) -> bool {
231        matches!(self.0, T_EQLC | T_NEQ | T_LTE | T_LSS | T_GTE | T_GTR)
232    }
233
234    pub fn is_set_operator(&self) -> bool {
235        matches!(self.0, T_LAND | T_LOR | T_LUNLESS)
236    }
237
238    pub fn is_operator(&self) -> bool {
239        self.0 > T_OPERATORS_START && self.0 < T_OPERATORS_END
240    }
241}
242
243impl fmt::Display for TokenType {
244    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
245        write!(f, "{}", token_display(self.id()))
246    }
247}
248
249#[cfg(test)]
250mod tests {
251    use super::*;
252
253    #[test]
254    fn test_token_display() {
255        assert_eq!(token_display(T_EQL), "=");
256        assert_eq!(token_display(T_BLANK), "_");
257        assert_eq!(token_display(T_COLON), ":");
258        assert_eq!(token_display(T_COMMA), ",");
259        assert_eq!(token_display(T_COMMENT), "#");
260        assert_eq!(token_display(T_DURATION), "[du]");
261        assert_eq!(token_display(T_EOF), "<eof>");
262        assert_eq!(token_display(T_ERROR), "{Err}");
263        assert_eq!(token_display(T_IDENTIFIER), "{ID}");
264        assert_eq!(token_display(T_LEFT_BRACE), "{");
265        assert_eq!(token_display(T_LEFT_BRACKET), "[");
266        assert_eq!(token_display(T_LEFT_PAREN), "(");
267        assert_eq!(token_display(T_OPEN_HIST), "{{");
268        assert_eq!(token_display(T_CLOSE_HIST), "}}");
269        assert_eq!(token_display(T_METRIC_IDENTIFIER), "{Metric_ID}");
270        assert_eq!(token_display(T_NUMBER), "{Num}");
271        assert_eq!(token_display(T_RIGHT_BRACE), "}");
272        assert_eq!(token_display(T_RIGHT_BRACKET), "]");
273        assert_eq!(token_display(T_RIGHT_PAREN), ")");
274        assert_eq!(token_display(T_SEMICOLON), ",");
275        assert_eq!(token_display(T_SPACE), "<space>");
276        assert_eq!(token_display(T_STRING), "{Str}");
277        assert_eq!(token_display(T_TIMES), "x");
278        assert_eq!(token_display(T_OPERATORS_START), "operators_start");
279        assert_eq!(token_display(T_ADD), "+");
280        assert_eq!(token_display(T_DIV), "/");
281        assert_eq!(token_display(T_EQLC), "==");
282        assert_eq!(token_display(T_EQL_REGEX), "=~");
283        assert_eq!(token_display(T_GTE), ">=");
284        assert_eq!(token_display(T_GTR), ">");
285        assert_eq!(token_display(T_LAND), "and");
286        assert_eq!(token_display(T_LOR), "or");
287        assert_eq!(token_display(T_LSS), "<");
288        assert_eq!(token_display(T_LTE), "<=");
289        assert_eq!(token_display(T_LUNLESS), "unless");
290        assert_eq!(token_display(T_MOD), "%");
291        assert_eq!(token_display(T_MUL), "*");
292        assert_eq!(token_display(T_NEQ), "!=");
293        assert_eq!(token_display(T_NEQ_REGEX), "!~");
294        assert_eq!(token_display(T_POW), "^");
295        assert_eq!(token_display(T_SUB), "-");
296        assert_eq!(token_display(T_AT), "@");
297        assert_eq!(token_display(T_ATAN2), "atan2");
298        assert_eq!(token_display(T_OPERATORS_END), "operators_end");
299        assert_eq!(token_display(T_AGGREGATORS_START), "aggregators_start");
300        assert_eq!(token_display(T_AVG), "avg");
301        assert_eq!(token_display(T_BOTTOMK), "bottomk");
302        assert_eq!(token_display(T_COUNT), "count");
303        assert_eq!(token_display(T_COUNT_VALUES), "count_values");
304        assert_eq!(token_display(T_GROUP), "group");
305        assert_eq!(token_display(T_MAX), "max");
306        assert_eq!(token_display(T_MIN), "min");
307        assert_eq!(token_display(T_QUANTILE), "quantile");
308        assert_eq!(token_display(T_STDDEV), "stddev");
309        assert_eq!(token_display(T_STDVAR), "stdvar");
310        assert_eq!(token_display(T_SUM), "sum");
311        assert_eq!(token_display(T_TOPK), "topk");
312        assert_eq!(token_display(T_LIMITK), "limitk");
313        assert_eq!(token_display(T_LIMIT_RATIO), "limit_ratio");
314        assert_eq!(token_display(T_AGGREGATORS_END), "aggregators_end");
315        assert_eq!(token_display(T_KEYWORDS_START), "keywords_start");
316        assert_eq!(token_display(T_BOOL), "bool");
317        assert_eq!(token_display(T_BY), "by");
318        assert_eq!(token_display(T_GROUP_LEFT), "group_left");
319        assert_eq!(token_display(T_GROUP_RIGHT), "group_right");
320        assert_eq!(token_display(T_IGNORING), "ignoring");
321        assert_eq!(token_display(T_OFFSET), "offset");
322        assert_eq!(token_display(T_SMOOTHED), "smoothed");
323        assert_eq!(token_display(T_ANCHORED), "anchored");
324        assert_eq!(token_display(T_ON), "on");
325        assert_eq!(token_display(T_WITHOUT), "without");
326        assert_eq!(token_display(T_KEYWORDS_END), "keywords_end");
327        assert_eq!(token_display(T_PREPROCESSOR_START), "preprocessor_start");
328        assert_eq!(token_display(T_START), "start");
329        assert_eq!(token_display(T_END), "end");
330        assert_eq!(token_display(T_STEP), "step");
331        assert_eq!(token_display(T_PREPROCESSOR_END), "preprocessors_end");
332
333        // if new token added in promql.y, this has to be updated
334        for i in 77..=82 {
335            assert_eq!(token_display(i), "not used");
336        }
337
338        // All tokens are now tested individually above
339
340        for i in 83..=255 {
341            assert_eq!(token_display(i), "unknown token");
342        }
343    }
344
345    #[test]
346    fn test_get_keyword_tokens() {
347        assert!(matches!(get_keyword_token("and"), Some(T_LAND)));
348        assert!(matches!(get_keyword_token("or"), Some(T_LOR)));
349        assert!(matches!(get_keyword_token("unless"), Some(T_LUNLESS)));
350        assert!(matches!(get_keyword_token("atan2"), Some(T_ATAN2)));
351        assert!(matches!(get_keyword_token("sum"), Some(T_SUM)));
352        assert!(matches!(get_keyword_token("avg"), Some(T_AVG)));
353        assert!(matches!(get_keyword_token("count"), Some(T_COUNT)));
354        assert!(matches!(get_keyword_token("min"), Some(T_MIN)));
355        assert!(matches!(get_keyword_token("max"), Some(T_MAX)));
356        assert!(matches!(get_keyword_token("group"), Some(T_GROUP)));
357        assert!(matches!(get_keyword_token("stddev"), Some(T_STDDEV)));
358        assert!(matches!(get_keyword_token("stdvar"), Some(T_STDVAR)));
359        assert!(matches!(get_keyword_token("topk"), Some(T_TOPK)));
360        assert!(matches!(get_keyword_token("bottomk"), Some(T_BOTTOMK)));
361        assert!(matches!(
362            get_keyword_token("count_values"),
363            Some(T_COUNT_VALUES)
364        ));
365        assert!(matches!(get_keyword_token("quantile"), Some(T_QUANTILE)));
366        assert!(matches!(get_keyword_token("offset"), Some(T_OFFSET)));
367        assert!(matches!(get_keyword_token("by"), Some(T_BY)));
368        assert!(matches!(get_keyword_token("without"), Some(T_WITHOUT)));
369        assert!(matches!(get_keyword_token("on"), Some(T_ON)));
370        assert!(matches!(get_keyword_token("ignoring"), Some(T_IGNORING)));
371        assert!(matches!(
372            get_keyword_token("group_left"),
373            Some(T_GROUP_LEFT)
374        ));
375        assert!(matches!(
376            get_keyword_token("group_right"),
377            Some(T_GROUP_RIGHT)
378        ));
379        assert!(matches!(get_keyword_token("bool"), Some(T_BOOL)));
380        assert!(matches!(get_keyword_token("start"), Some(T_START)));
381        assert!(matches!(get_keyword_token("end"), Some(T_END)));
382        assert!(matches!(get_keyword_token("inf"), Some(T_NUMBER)));
383        assert!(matches!(get_keyword_token("nan"), Some(T_NUMBER)));
384
385        // not keywords
386        assert!(get_keyword_token("at").is_none());
387        assert!(get_keyword_token("unknown").is_none());
388    }
389
390    #[test]
391    fn test_with_param() {
392        assert!(TokenType(T_TOPK).is_aggregator_with_param());
393        assert!(TokenType(T_BOTTOMK).is_aggregator_with_param());
394        assert!(TokenType(T_COUNT_VALUES).is_aggregator_with_param());
395        assert!(TokenType(T_QUANTILE).is_aggregator_with_param());
396
397        assert!(!TokenType(T_MAX).is_aggregator_with_param());
398        assert!(!TokenType(T_MIN).is_aggregator_with_param());
399        assert!(!TokenType(T_AVG).is_aggregator_with_param());
400    }
401
402    #[test]
403    fn test_comparison_operator() {
404        assert!(TokenType(T_EQLC).is_comparison_operator());
405        assert!(TokenType(T_NEQ).is_comparison_operator());
406        assert!(TokenType(T_LTE).is_comparison_operator());
407        assert!(TokenType(T_LSS).is_comparison_operator());
408        assert!(TokenType(T_GTE).is_comparison_operator());
409        assert!(TokenType(T_GTR).is_comparison_operator());
410
411        assert!(!TokenType(T_ADD).is_comparison_operator());
412        assert!(!TokenType(T_LAND).is_comparison_operator());
413    }
414
415    #[test]
416    fn test_is_set_operator() {
417        assert!(TokenType(T_LAND).is_set_operator());
418        assert!(TokenType(T_LOR).is_set_operator());
419        assert!(TokenType(T_LUNLESS).is_set_operator());
420
421        assert!(!TokenType(T_ADD).is_set_operator());
422        assert!(!TokenType(T_MAX).is_set_operator());
423        assert!(!TokenType(T_NEQ).is_set_operator());
424    }
425
426    #[test]
427    fn test_is_operator() {
428        assert!(TokenType(T_ADD).is_operator());
429        assert!(TokenType(T_DIV).is_operator());
430        assert!(TokenType(T_EQLC).is_operator());
431        assert!(TokenType(T_EQL_REGEX).is_operator());
432        assert!(TokenType(T_GTE).is_operator());
433        assert!(TokenType(T_GTR).is_operator());
434        assert!(TokenType(T_LAND).is_operator());
435        assert!(TokenType(T_LOR).is_operator());
436        assert!(TokenType(T_LSS).is_operator());
437        assert!(TokenType(T_LTE).is_operator());
438        assert!(TokenType(T_LUNLESS).is_operator());
439        assert!(TokenType(T_MOD).is_operator());
440        assert!(TokenType(T_MUL).is_operator());
441        assert!(TokenType(T_NEQ).is_operator());
442        assert!(TokenType(T_NEQ_REGEX).is_operator());
443        assert!(TokenType(T_POW).is_operator());
444        assert!(TokenType(T_SUB).is_operator());
445        assert!(TokenType(T_AT).is_operator());
446        assert!(TokenType(T_ATAN2).is_operator());
447
448        assert!(!TokenType(T_SUM).is_operator());
449        assert!(!TokenType(T_OPERATORS_START).is_operator());
450        assert!(!TokenType(T_OPERATORS_END).is_operator());
451    }
452
453    #[test]
454    fn test_is_aggregator() {
455        assert!(TokenType(T_AVG).is_aggregator());
456        assert!(TokenType(T_BOTTOMK).is_aggregator());
457        assert!(TokenType(T_COUNT).is_aggregator());
458        assert!(TokenType(T_COUNT_VALUES).is_aggregator());
459        assert!(TokenType(T_GROUP).is_aggregator());
460        assert!(TokenType(T_MAX).is_aggregator());
461        assert!(TokenType(T_MIN).is_aggregator());
462        assert!(TokenType(T_QUANTILE).is_aggregator());
463        assert!(TokenType(T_STDDEV).is_aggregator());
464        assert!(TokenType(T_STDVAR).is_aggregator());
465        assert!(TokenType(T_SUM).is_aggregator());
466        assert!(TokenType(T_TOPK).is_aggregator());
467
468        assert!(!TokenType(T_LOR).is_aggregator());
469        assert!(!TokenType(T_ADD).is_aggregator());
470    }
471}
promql_parser/parser/token.rs

promql_parser/parser/
token.rs
