Skip to main content

logql/
token.rs

1//! # LogQL Tokens
2//!
3//! Token type produced by the [`crate::lexer`]. Function names
4//! (`rate`, `count_over_time`, `sum`, `json`, `line_format`, ...) are
5//! deliberately lexed as [`Token::Ident`] — LogQL treats them as
6//! context-dependent identifiers, so the parser resolves them where the
7//! grammar expects a function or parser stage. Only words that are
8//! reserved in the grammar itself become keyword tokens.
9
10use std::time::Duration;
11
12/// A single LogQL token.
13#[derive(Debug, Clone, PartialEq)]
14#[non_exhaustive]
15pub enum Token {
16    // Literals
17    /// Double-quoted (escaped) or backtick-quoted (raw) string.
18    String(String),
19    /// Numeric literal, including floats and scientific notation.
20    Number(f64),
21    /// Duration literal such as `5m`, `1h30m`, or `1.5h`.
22    Duration(Duration),
23    /// Bytes literal such as `20KB`, `5MB`, or `1GiB`, in bytes.
24    Bytes(u64),
25    /// Identifier: label names, function names, parser stage names.
26    Ident(String),
27    /// A parser-stage flag such as `--strict` or `--keep-empty` (stored
28    /// without the leading dashes).
29    Flag(String),
30
31    // Label matchers / equality
32    /// `=`
33    Eq,
34    /// `!=` (label matcher or line filter, depending on position)
35    Neq,
36    /// `=~`
37    Re,
38    /// `!~` (label matcher or line filter, depending on position)
39    Nre,
40
41    // Pipeline
42    /// `|=`
43    PipeExact,
44    /// `|~`
45    PipeRegex,
46    /// `|`
47    Pipe,
48
49    // Comparison (metric queries and label filter expressions)
50    /// `==`
51    CmpEq,
52    /// `>`
53    Gt,
54    /// `>=`
55    Gte,
56    /// `<`
57    Lt,
58    /// `<=`
59    Lte,
60
61    // Arithmetic
62    /// `+`
63    Add,
64    /// `-`
65    Sub,
66    /// `*`
67    Mul,
68    /// `/`
69    Div,
70    /// `%`
71    Mod,
72    /// `^`
73    Pow,
74
75    // Brackets
76    /// `{`
77    LBrace,
78    /// `}`
79    RBrace,
80    /// `(`
81    LParen,
82    /// `)`
83    RParen,
84    /// `[`
85    LBracket,
86    /// `]`
87    RBracket,
88
89    // Punctuation
90    /// `,`
91    Comma,
92
93    // Keywords (reserved words in the LogQL grammar)
94    /// `by`
95    By,
96    /// `without`
97    Without,
98    /// `on`
99    On,
100    /// `ignoring`
101    Ignoring,
102    /// `group_left`
103    GroupLeft,
104    /// `group_right`
105    GroupRight,
106    /// `unwrap`
107    Unwrap,
108    /// `offset`
109    Offset,
110    /// `bool`
111    Bool,
112    /// `and`
113    And,
114    /// `or`
115    Or,
116    /// `unless`
117    Unless,
118}
119
120impl Token {
121    /// Map a lexed word to its keyword token, if it is one.
122    pub(crate) fn keyword(word: &str) -> Option<Token> {
123        Some(match word {
124            "by" => Token::By,
125            "without" => Token::Without,
126            "on" => Token::On,
127            "ignoring" => Token::Ignoring,
128            "group_left" => Token::GroupLeft,
129            "group_right" => Token::GroupRight,
130            "unwrap" => Token::Unwrap,
131            "offset" => Token::Offset,
132            "bool" => Token::Bool,
133            "and" => Token::And,
134            "or" => Token::Or,
135            "unless" => Token::Unless,
136            _ => return None,
137        })
138    }
139
140    /// The source text of a keyword token, if the token is one. Inverse
141    /// of [`Token::keyword`], used where the grammar allows keywords as
142    /// plain identifiers (e.g. label names).
143    pub(crate) fn keyword_text(token: &Token) -> Option<&'static str> {
144        Some(match token {
145            Token::By => "by",
146            Token::Without => "without",
147            Token::On => "on",
148            Token::Ignoring => "ignoring",
149            Token::GroupLeft => "group_left",
150            Token::GroupRight => "group_right",
151            Token::Unwrap => "unwrap",
152            Token::Offset => "offset",
153            Token::Bool => "bool",
154            Token::And => "and",
155            Token::Or => "or",
156            Token::Unless => "unless",
157            _ => return None,
158        })
159    }
160}
161
162impl std::fmt::Display for Token {
163    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
164        match self {
165            Token::String(s) => write!(f, "{s:?}"),
166            Token::Number(n) => write!(f, "{n}"),
167            Token::Duration(d) => write!(f, "{d:?}"),
168            Token::Bytes(b) => write!(f, "{b}B"),
169            Token::Ident(i) => write!(f, "{i}"),
170            Token::Flag(name) => write!(f, "--{name}"),
171            Token::Eq => f.write_str("="),
172            Token::Neq => f.write_str("!="),
173            Token::Re => f.write_str("=~"),
174            Token::Nre => f.write_str("!~"),
175            Token::PipeExact => f.write_str("|="),
176            Token::PipeRegex => f.write_str("|~"),
177            Token::Pipe => f.write_str("|"),
178            Token::CmpEq => f.write_str("=="),
179            Token::Gt => f.write_str(">"),
180            Token::Gte => f.write_str(">="),
181            Token::Lt => f.write_str("<"),
182            Token::Lte => f.write_str("<="),
183            Token::Add => f.write_str("+"),
184            Token::Sub => f.write_str("-"),
185            Token::Mul => f.write_str("*"),
186            Token::Div => f.write_str("/"),
187            Token::Mod => f.write_str("%"),
188            Token::Pow => f.write_str("^"),
189            Token::LBrace => f.write_str("{"),
190            Token::RBrace => f.write_str("}"),
191            Token::LParen => f.write_str("("),
192            Token::RParen => f.write_str(")"),
193            Token::LBracket => f.write_str("["),
194            Token::RBracket => f.write_str("]"),
195            Token::Comma => f.write_str(","),
196            Token::By => f.write_str("by"),
197            Token::Without => f.write_str("without"),
198            Token::On => f.write_str("on"),
199            Token::Ignoring => f.write_str("ignoring"),
200            Token::GroupLeft => f.write_str("group_left"),
201            Token::GroupRight => f.write_str("group_right"),
202            Token::Unwrap => f.write_str("unwrap"),
203            Token::Offset => f.write_str("offset"),
204            Token::Bool => f.write_str("bool"),
205            Token::And => f.write_str("and"),
206            Token::Or => f.write_str("or"),
207            Token::Unless => f.write_str("unless"),
208        }
209    }
210}
211
212/// A token with its 1-based source position (position of the token's
213/// first character).
214#[derive(Debug, Clone, PartialEq)]
215pub struct SpannedToken {
216    pub token: Token,
217    pub line: u32,
218    pub col: u32,
219}