Skip to main content

logql/
token.rs

1//! # LogQL Tokens
2//!
3//! Token type produced by the [`crate::lexer`]. Function names
4//! (`rate`, `count_over_time`, `sum`, `json`, `line_format`, ...) are
5//! deliberately lexed as [`Token::Ident`] — LogQL treats them as
6//! context-dependent identifiers, so the parser resolves them where the
7//! grammar expects a function or parser stage. Only words that are
8//! reserved in the grammar itself become keyword tokens.
9
10use std::time::Duration;
11
12/// A single LogQL token.
13#[derive(Debug, Clone, PartialEq)]
14#[non_exhaustive]
15pub enum Token {
16    // Literals
17    /// Double-quoted (escaped) or backtick-quoted (raw) string.
18    String(String),
19    /// Numeric literal, including floats and scientific notation.
20    Number(f64),
21    /// Duration literal such as `5m`, `1h30m`, or `1.5h`.
22    Duration(Duration),
23    /// Bytes literal such as `20KB`, `5MB`, or `1GiB`, in bytes.
24    Bytes(u64),
25    /// Identifier: label names, function names, parser stage names.
26    Ident(String),
27    /// A parser-stage flag such as `--strict` or `--keep-empty` (stored
28    /// without the leading dashes).
29    Flag(String),
30
31    // Label matchers / equality
32    /// `=`
33    Eq,
34    /// `!=` (label matcher or line filter, depending on position)
35    Neq,
36    /// `=~`
37    Re,
38    /// `!~` (label matcher or line filter, depending on position)
39    Nre,
40
41    // Pipeline
42    /// `|=`
43    PipeExact,
44    /// `|~`
45    PipeRegex,
46    /// `|`
47    Pipe,
48
49    // Comparison (metric queries and label filter expressions)
50    /// `==`
51    CmpEq,
52    /// `>`
53    Gt,
54    /// `>=`
55    Gte,
56    /// `<`
57    Lt,
58    /// `<=`
59    Lte,
60
61    // Arithmetic
62    /// `+`
63    Add,
64    /// `-`
65    Sub,
66    /// `*`
67    Mul,
68    /// `/`
69    Div,
70    /// `%`
71    Mod,
72    /// `^`
73    Pow,
74
75    // Brackets
76    /// `{`
77    LBrace,
78    /// `}`
79    RBrace,
80    /// `(`
81    LParen,
82    /// `)`
83    RParen,
84    /// `[`
85    LBracket,
86    /// `]`
87    RBracket,
88
89    // Punctuation
90    /// `,`
91    Comma,
92
93    // Keywords (reserved words in the LogQL grammar)
94    /// `by`
95    By,
96    /// `without`
97    Without,
98    /// `on`
99    On,
100    /// `ignoring`
101    Ignoring,
102    /// `group_left`
103    GroupLeft,
104    /// `group_right`
105    GroupRight,
106    /// `unwrap`
107    Unwrap,
108    /// `offset`
109    Offset,
110    /// `bool`
111    Bool,
112    /// `and`
113    And,
114    /// `or`
115    Or,
116    /// `unless`
117    Unless,
118}
119
120impl Token {
121    /// Map a lexed word to its keyword token, if it is one.
122    pub(crate) fn keyword(word: &str) -> Option<Token> {
123        Some(match word {
124            "by" => Token::By,
125            "without" => Token::Without,
126            "on" => Token::On,
127            "ignoring" => Token::Ignoring,
128            "group_left" => Token::GroupLeft,
129            "group_right" => Token::GroupRight,
130            "unwrap" => Token::Unwrap,
131            "offset" => Token::Offset,
132            "bool" => Token::Bool,
133            "and" => Token::And,
134            "or" => Token::Or,
135            "unless" => Token::Unless,
136            _ => return None,
137        })
138    }
139
140    /// The source text of a keyword token, if the token is one. Inverse
141    /// of [`Token::keyword`], used where the grammar allows keywords as
142    /// plain identifiers (e.g. label names).
143    pub(crate) fn keyword_text(token: &Token) -> Option<&'static str> {
144        Some(match token {
145            Token::By => "by",
146            Token::Without => "without",
147            Token::On => "on",
148            Token::Ignoring => "ignoring",
149            Token::GroupLeft => "group_left",
150            Token::GroupRight => "group_right",
151            Token::Unwrap => "unwrap",
152            Token::Offset => "offset",
153            Token::Bool => "bool",
154            Token::And => "and",
155            Token::Or => "or",
156            Token::Unless => "unless",
157            _ => return None,
158        })
159    }
160}
161
162impl std::fmt::Display for Token {
163    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
164        match self {
165            Token::String(s) => write!(f, "{s:?}"),
166            Token::Number(n) => write!(f, "{n}"),
167            Token::Duration(d) => write!(f, "{d:?}"),
168            Token::Bytes(b) => write!(f, "{b}B"),
169            Token::Ident(i) => write!(f, "{i}"),
170            Token::Flag(name) => write!(f, "--{name}"),
171            Token::Eq => f.write_str("="),
172            Token::Neq => f.write_str("!="),
173            Token::Re => f.write_str("=~"),
174            Token::Nre => f.write_str("!~"),
175            Token::PipeExact => f.write_str("|="),
176            Token::PipeRegex => f.write_str("|~"),
177            Token::Pipe => f.write_str("|"),
178            Token::CmpEq => f.write_str("=="),
179            Token::Gt => f.write_str(">"),
180            Token::Gte => f.write_str(">="),
181            Token::Lt => f.write_str("<"),
182            Token::Lte => f.write_str("<="),
183            Token::Add => f.write_str("+"),
184            Token::Sub => f.write_str("-"),
185            Token::Mul => f.write_str("*"),
186            Token::Div => f.write_str("/"),
187            Token::Mod => f.write_str("%"),
188            Token::Pow => f.write_str("^"),
189            Token::LBrace => f.write_str("{"),
190            Token::RBrace => f.write_str("}"),
191            Token::LParen => f.write_str("("),
192            Token::RParen => f.write_str(")"),
193            Token::LBracket => f.write_str("["),
194            Token::RBracket => f.write_str("]"),
195            Token::Comma => f.write_str(","),
196            Token::By => f.write_str("by"),
197            Token::Without => f.write_str("without"),
198            Token::On => f.write_str("on"),
199            Token::Ignoring => f.write_str("ignoring"),
200            Token::GroupLeft => f.write_str("group_left"),
201            Token::GroupRight => f.write_str("group_right"),
202            Token::Unwrap => f.write_str("unwrap"),
203            Token::Offset => f.write_str("offset"),
204            Token::Bool => f.write_str("bool"),
205            Token::And => f.write_str("and"),
206            Token::Or => f.write_str("or"),
207            Token::Unless => f.write_str("unless"),
208        }
209    }
210}
211
212/// A token with its 1-based source position (position of the token's
213/// first character).
214#[derive(Debug, Clone, PartialEq)]
215pub struct SpannedToken {
216    pub token: Token,
217    pub line: u32,
218    pub col: u32,
219}
220
221#[cfg(test)]
222mod tests {
223    use super::*;
224
225    /// Every keyword the lexer recognises, paired with the token it produces.
226    /// Written as one table so a keyword added to `keyword` without a matching
227    /// `keyword_text` arm (or vice versa) fails the round-trip below.
228    const KEYWORDS: &[(&str, Token)] = &[
229        ("by", Token::By),
230        ("without", Token::Without),
231        ("on", Token::On),
232        ("ignoring", Token::Ignoring),
233        ("group_left", Token::GroupLeft),
234        ("group_right", Token::GroupRight),
235        ("unwrap", Token::Unwrap),
236        ("offset", Token::Offset),
237        ("bool", Token::Bool),
238        ("and", Token::And),
239        ("or", Token::Or),
240        ("unless", Token::Unless),
241    ];
242
243    #[test]
244    fn every_keyword_maps_to_its_token() {
245        for (word, expected) in KEYWORDS {
246            assert_eq!(
247                Token::keyword(word).as_ref(),
248                Some(expected),
249                "keyword({word:?})"
250            );
251        }
252    }
253
254    /// `keyword_text` is documented as the inverse of `keyword`, and the
255    /// grammar relies on it where a keyword may appear as a plain label name.
256    /// An arm missing from either direction breaks that.
257    #[test]
258    fn keyword_text_inverts_keyword() {
259        for (word, token) in KEYWORDS {
260            assert_eq!(
261                Token::keyword_text(token),
262                Some(*word),
263                "keyword_text({token:?})"
264            );
265        }
266    }
267
268    #[test]
269    fn non_keywords_are_identifiers() {
270        for word in ["service_name", "rate", "json", "By", "AND", ""] {
271            assert_eq!(Token::keyword(word), None, "{word:?} is not a keyword");
272        }
273    }
274
275    #[test]
276    fn only_keyword_tokens_have_keyword_text() {
277        for token in [
278            Token::Eq,
279            Token::Pipe,
280            Token::LBrace,
281            Token::Ident("service_name".to_string()),
282            Token::Number(1.0),
283        ] {
284            assert_eq!(Token::keyword_text(&token), None, "{token:?}");
285        }
286    }
287
288    /// Display is what error messages show a user, so every variant needs to
289    /// render as the source text it was lexed from — not as a Debug dump.
290    #[test]
291    fn every_token_displays_as_its_source_text() {
292        let cases: &[(Token, &str)] = &[
293            (Token::Number(2.5), "2.5"),
294            (Token::Bytes(1024), "1024B"),
295            (Token::Ident("service_name".to_string()), "service_name"),
296            (Token::Flag("strict".to_string()), "--strict"),
297            (Token::Eq, "="),
298            (Token::Neq, "!="),
299            (Token::Re, "=~"),
300            (Token::Nre, "!~"),
301            (Token::PipeExact, "|="),
302            (Token::PipeRegex, "|~"),
303            (Token::Pipe, "|"),
304            (Token::CmpEq, "=="),
305            (Token::Gt, ">"),
306            (Token::Gte, ">="),
307            (Token::Lt, "<"),
308            (Token::Lte, "<="),
309            (Token::Add, "+"),
310            (Token::Sub, "-"),
311            (Token::Mul, "*"),
312            (Token::Div, "/"),
313            (Token::Mod, "%"),
314            (Token::Pow, "^"),
315            (Token::LBrace, "{"),
316            (Token::RBrace, "}"),
317            (Token::LParen, "("),
318            (Token::RParen, ")"),
319            (Token::LBracket, "["),
320            (Token::RBracket, "]"),
321            (Token::Comma, ","),
322            (Token::By, "by"),
323            (Token::Without, "without"),
324            (Token::On, "on"),
325            (Token::Ignoring, "ignoring"),
326            (Token::GroupLeft, "group_left"),
327            (Token::GroupRight, "group_right"),
328            (Token::Unwrap, "unwrap"),
329            (Token::Offset, "offset"),
330            (Token::Bool, "bool"),
331            (Token::And, "and"),
332            (Token::Or, "or"),
333            (Token::Unless, "unless"),
334        ];
335        for (token, expected) in cases {
336            assert_eq!(&token.to_string(), expected, "Display for {token:?}");
337        }
338    }
339
340    /// A string renders quoted so an error message can show where a literal
341    /// began and ended; a duration renders in its Debug form, which is the
342    /// spelled-out unit rather than a raw nanosecond count.
343    #[test]
344    fn strings_and_durations_render_readably() {
345        assert_eq!(Token::String("api".to_string()).to_string(), "\"api\"");
346        assert_eq!(
347            Token::Duration(std::time::Duration::from_secs(300)).to_string(),
348            "300s"
349        );
350    }
351}