apif_ast/
tokenizer.rs

1//! Public tokenizer for GCTF assertion expressions.
2//!
3//! Pipeline: `text → tokenize_assertion() → Vec<Token>` where each `Token`
4//! has a `Span { start, end }` with exact byte positions in the source.
5//!
6//! This is used by:
7//! - **LSP semantic tokens** — highlight operators, keywords, plugins, regexes
8//! - **Optimizer** — safe string-literal-aware rule matching (4.3)
9//! - **Semantics** — type-checking operators against TypeInfo
10
11use serde::{Deserialize, Serialize};
12
13/// Byte range in source text.
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
15pub struct Span {
16    pub start: usize,
17    pub end: usize,
18}
19
20impl Span {
21    pub fn len(&self) -> usize {
22        self.end - self.start
23    }
24    #[must_use]
25    pub fn is_empty(&self) -> bool {
26        self.start == self.end
27    }
28}
29
30/// Token kinds produced by the assertion tokenizer.
31#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
32pub enum TokenKind {
33    Ident(String),
34    StringLit(String),
35    NumberLit(String),
36    Op(String),
37    RegExpLit { pattern: String, flags: String },
38    At,
39    LParen,
40    RParen,
41    LBracket,
42    RBracket,
43    LBrace,
44    RBrace,
45    Dot,
46    Comma,
47    Bang,
48    Pipe,
49    Slash,
50    Colon,
51    VarDelim,
52}
53
54/// A token with its source position.
55#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
56pub struct Token {
57    pub kind: TokenKind,
58    pub span: Span,
59}
60
61impl Token {
62    pub fn new(kind: TokenKind, span: Span) -> Self {
63        Self { kind, span }
64    }
65}
66
67/// Tokenize an assertion expression string into a list of tokens with
68/// exact byte positions.
69pub fn tokenize_assertion(source: &str) -> Vec<Token> {
70    let mut out = Vec::with_capacity(source.len() / 2);
71    let cs: Vec<char> = source.chars().collect();
72    let mut i = 0;
73
74    while i < cs.len() {
75        match cs[i] {
76            ' ' | '\t' | '\n' | '\r' => {
77                i += 1;
78            }
79            '@' => {
80                let s = i;
81                i += 1;
82                out.push(Token::new(TokenKind::At, Span { start: s, end: i }));
83            }
84            '(' => {
85                let s = i;
86                i += 1;
87                out.push(Token::new(TokenKind::LParen, Span { start: s, end: i }));
88            }
89            ')' => {
90                let s = i;
91                i += 1;
92                out.push(Token::new(TokenKind::RParen, Span { start: s, end: i }));
93            }
94            '[' => {
95                let s = i;
96                i += 1;
97                out.push(Token::new(TokenKind::LBracket, Span { start: s, end: i }));
98            }
99            ']' => {
100                let s = i;
101                i += 1;
102                out.push(Token::new(TokenKind::RBracket, Span { start: s, end: i }));
103            }
104            '{' => {
105                if i + 1 < cs.len() && cs[i + 1] == '{' {
106                    let s = i;
107                    i += 2;
108                    out.push(Token::new(TokenKind::VarDelim, Span { start: s, end: i }));
109                } else {
110                    let s = i;
111                    i += 1;
112                    out.push(Token::new(TokenKind::LBrace, Span { start: s, end: i }));
113                }
114            }
115            '}' => {
116                if i + 1 < cs.len() && cs[i + 1] == '}' {
117                    let s = i;
118                    i += 2;
119                    out.push(Token::new(TokenKind::VarDelim, Span { start: s, end: i }));
120                } else {
121                    let s = i;
122                    i += 1;
123                    out.push(Token::new(TokenKind::RBrace, Span { start: s, end: i }));
124                }
125            }
126            '.' => {
127                let s = i;
128                i += 1;
129                out.push(Token::new(TokenKind::Dot, Span { start: s, end: i }));
130            }
131            ',' => {
132                let s = i;
133                i += 1;
134                out.push(Token::new(TokenKind::Comma, Span { start: s, end: i }));
135            }
136            '|' => {
137                let s = i;
138                i += 1;
139                out.push(Token::new(TokenKind::Pipe, Span { start: s, end: i }));
140            }
141            '!' => {
142                if i + 1 < cs.len() && cs[i + 1] == '=' {
143                    let s = i;
144                    i += 2;
145                    out.push(Token::new(
146                        TokenKind::Op("!=".into()),
147                        Span { start: s, end: i },
148                    ));
149                } else {
150                    let s = i;
151                    i += 1;
152                    out.push(Token::new(TokenKind::Bang, Span { start: s, end: i }));
153                }
154            }
155            '=' if i + 1 < cs.len() && cs[i + 1] == '=' => {
156                let s = i;
157                i += 2;
158                out.push(Token::new(
159                    TokenKind::Op("==".into()),
160                    Span { start: s, end: i },
161                ));
162            }
163            '=' => i += 1,
164            '>' => {
165                if i + 1 < cs.len() && cs[i + 1] == '=' {
166                    let s = i;
167                    i += 2;
168                    out.push(Token::new(
169                        TokenKind::Op(">=".into()),
170                        Span { start: s, end: i },
171                    ));
172                } else {
173                    let s = i;
174                    i += 1;
175                    out.push(Token::new(
176                        TokenKind::Op(">".into()),
177                        Span { start: s, end: i },
178                    ));
179                }
180            }
181            '<' => {
182                if i + 1 < cs.len() && cs[i + 1] == '=' {
183                    let s = i;
184                    i += 2;
185                    out.push(Token::new(
186                        TokenKind::Op("<=".into()),
187                        Span { start: s, end: i },
188                    ));
189                } else {
190                    let s = i;
191                    i += 1;
192                    out.push(Token::new(
193                        TokenKind::Op("<".into()),
194                        Span { start: s, end: i },
195                    ));
196                }
197            }
198            '-' => {
199                let s = i;
200                i += 1;
201                out.push(Token::new(
202                    TokenKind::Op("-".into()),
203                    Span { start: s, end: i },
204                ));
205            }
206            '"' => {
207                let s = i;
208                i += 1;
209                let mut v = String::new();
210                while i < cs.len() && cs[i] != '"' {
211                    if cs[i] == '\\' && i + 1 < cs.len() {
212                        i += 1;
213                        v.push(cs[i]);
214                    } else {
215                        v.push(cs[i]);
216                    }
217                    i += 1;
218                }
219                if i < cs.len() {
220                    i += 1;
221                }
222                out.push(Token::new(
223                    TokenKind::StringLit(v),
224                    Span { start: s, end: i },
225                ));
226            }
227            c if c.is_ascii_digit() => {
228                let s = i;
229                let mut v = String::new();
230                while i < cs.len() && (cs[i].is_ascii_digit() || cs[i] == '.') {
231                    v.push(cs[i]);
232                    i += 1;
233                }
234                out.push(Token::new(
235                    TokenKind::NumberLit(v),
236                    Span { start: s, end: i },
237                ));
238            }
239            c if c.is_alphabetic() || c == '_' => {
240                let s = i;
241                let mut v = String::new();
242                while i < cs.len() && (cs[i].is_alphanumeric() || cs[i] == '_') {
243                    v.push(cs[i]);
244                    i += 1;
245                }
246                let kind = match v.as_str() {
247                    "contains" | "matches" | "startsWith" | "endsWith" | "startswith"
248                    | "endswith" => TokenKind::Op(v),
249                    _ => TokenKind::Ident(v),
250                };
251                out.push(Token::new(kind, Span { start: s, end: i }));
252            }
253            ':' => {
254                let s = i;
255                i += 1;
256                out.push(Token::new(TokenKind::Colon, Span { start: s, end: i }));
257            }
258            '$' => {
259                let s = i;
260                i += 1;
261                let mut v = String::from('$');
262                while i < cs.len() && (cs[i].is_alphanumeric() || cs[i] == '_') {
263                    v.push(cs[i]);
264                    i += 1;
265                }
266                out.push(Token::new(TokenKind::Ident(v), Span { start: s, end: i }));
267            }
268            '/' => {
269                let s = i;
270                let mut j = i + 1;
271                let mut is_regex = true;
272                let mut pattern = String::new();
273                let mut escaped = false;
274                let mut in_cc = false;
275                while j < cs.len() {
276                    if escaped {
277                        pattern.push(cs[j]);
278                        escaped = false;
279                        j += 1;
280                    } else if cs[j] == '\\' {
281                        pattern.push(cs[j]);
282                        escaped = true;
283                        j += 1;
284                    } else if cs[j] == '[' {
285                        in_cc = true;
286                        pattern.push(cs[j]);
287                        j += 1;
288                    } else if cs[j] == ']' {
289                        in_cc = false;
290                        pattern.push(cs[j]);
291                        j += 1;
292                    } else if cs[j] == '/' && !in_cc {
293                        j += 1;
294                        let mut flags = String::new();
295                        while j < cs.len()
296                            && cs[j].is_ascii_alphabetic()
297                            && "gimsuy".contains(cs[j])
298                        {
299                            flags.push(cs[j]);
300                            j += 1;
301                        }
302                        if j < cs.len()
303                            && (cs[j].is_alphanumeric() || cs[j] == '_')
304                            && !flags.is_empty()
305                        {
306                            is_regex = false;
307                        }
308                        break;
309                    } else if cs[j] == '\n' || cs[j] == '\r' || cs[j] == ' ' || cs[j] == '\t' {
310                        is_regex = false;
311                        break;
312                    } else {
313                        pattern.push(cs[j]);
314                        j += 1;
315                    }
316                }
317                if is_regex && !pattern.is_empty() && j > i + 1 {
318                    out.push(Token::new(
319                        TokenKind::RegExpLit {
320                            pattern,
321                            flags: String::new(),
322                        },
323                        Span { start: s, end: j },
324                    ));
325                    i = j;
326                } else {
327                    i += 1;
328                    out.push(Token::new(TokenKind::Slash, Span { start: s, end: i }));
329                }
330            }
331            _ => {
332                i += 1;
333            }
334        }
335    }
336    out
337}
338
339/// Collect all identifier tokens.
340pub fn collect_identifiers(tokens: &[Token]) -> Vec<&str> {
341    tokens
342        .iter()
343        .filter_map(|t| match &t.kind {
344            TokenKind::Ident(s) => Some(s.as_str()),
345            _ => None,
346        })
347        .collect()
348}
349
350/// Collect all operator tokens.
351pub fn collect_operators(tokens: &[Token]) -> Vec<&str> {
352    tokens
353        .iter()
354        .filter_map(|t| match &t.kind {
355            TokenKind::Op(s) => Some(s.as_str()),
356            _ => None,
357        })
358        .collect()
359}
360
361/// Collect plugin call names with spans.
362pub fn collect_plugin_calls(tokens: &[Token]) -> Vec<(&str, Span)> {
363    let mut result = Vec::with_capacity(tokens.len() / 4);
364    for [at, ident] in tokens.array_windows::<2>() {
365        if let TokenKind::At = at.kind
366            && let TokenKind::Ident(name) = &ident.kind
367        {
368            result.push((name.as_str(), at.span));
369        }
370    }
371    result
372}
373
374#[cfg(test)]
375mod tests {
376    use super::*;
377
378    #[test]
379    fn test_tokenize_simple() {
380        let tokens = tokenize_assertion(".id == 123");
381        assert!(!tokens.is_empty());
382        assert!(
383            tokens
384                .iter()
385                .any(|t| matches!(&t.kind, TokenKind::Op(s) if s == "=="))
386        );
387        assert!(
388            tokens
389                .iter()
390                .any(|t| matches!(&t.kind, TokenKind::NumberLit(s) if s == "123"))
391        );
392    }
393
394    #[test]
395    fn test_tokenize_plugin_call() {
396        let tokens = tokenize_assertion("@len(.items) == 0");
397        assert!(tokens.iter().any(|t| matches!(t.kind, TokenKind::At)));
398        assert!(
399            tokens
400                .iter()
401                .any(|t| matches!(&t.kind, TokenKind::Ident(s) if s == "len"))
402        );
403    }
404
405    #[test]
406    fn test_tokenize_regex() {
407        let tokens = tokenize_assertion("@regex(.x, /hello/i) == true");
408        assert!(tokens.iter().any(
409            |t| matches!(&t.kind, TokenKind::RegExpLit { pattern, .. } if pattern == "hello")
410        ));
411    }
412
413    #[test]
414    fn test_tokenize_string_literal() {
415        let tokens = tokenize_assertion(".name == \"hello world\"");
416        assert!(
417            tokens
418                .iter()
419                .any(|t| matches!(&t.kind, TokenKind::StringLit(s) if s == "hello world"))
420        );
421    }
422
423    #[test]
424    fn test_spans_correct() {
425        let tokens = tokenize_assertion(".x == 0");
426        // "==" should be at position 3
427        if let Some(t) = tokens
428            .iter()
429            .find(|t| matches!(&t.kind, TokenKind::Op(s) if s == "=="))
430        {
431            assert_eq!(t.span.start, 3);
432            assert_eq!(t.span.end, 5);
433        } else {
434            panic!("No == token found");
435        }
436    }
437}
apif_ast/tokenizer.rs

apif_ast/
tokenizer.rs
