ripbi_core/dax/
lexer.rs

1//! DAX tokenizer: a Rust port of SQLBI Whiteboard's `DaxLexer` (MIT), reduced to
2//! what reference extraction needs. See [`crate::dax`] for the full provenance
3//! and for what was deliberately dropped.
4//!
5//! Like the original, the scanner is tolerance-first: unterminated strings,
6//! quotes, and brackets never fail — they simply run to the end of the input.
7//! Lexing is a total function; there is nothing to report as an error.
8
9/// What a [`Token`] is.
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
11pub enum TokenKind {
12    /// End of input. Always the last token; `text` is empty.
13    Eof,
14    /// `Sales`, `MyVar`, `NORM.DIST` — internal dots are absorbed.
15    Identifier,
16    /// `'Sales Header'` — a single-quoted table name, `''` as the escape.
17    QuotedTable,
18    /// `[Net Price]` — a bracketed column, measure, or hierarchy-level name.
19    /// The lexer cannot tell the three apart; resolution does.
20    BracketName,
21    /// `"text"` — a string literal, `""` as the escape.
22    String,
23    /// `dt"2024-01-01"` — a date-time literal.
24    DateTime,
25    /// `1`, `1.5`, `1.5E+10`.
26    Number,
27    /// `@Risk` — a query parameter (DAX queries, not model expressions).
28    QueryParameter,
29    /// `// …`, `-- …`, or `/* … */` — never produces a reference.
30    Comment,
31    /// `+ - * / ^ & = == <> < > <= >= && || ! => :=`
32    Operator,
33    /// `(`
34    OpenParen,
35    /// `)`
36    CloseParen,
37    /// `{`
38    OpenBrace,
39    /// `}`
40    CloseBrace,
41    /// `,`
42    Comma,
43    /// `;`
44    Semicolon,
45    /// `:`
46    Colon,
47    /// `.`
48    Dot,
49    /// Any other character. Never fails the scan.
50    Unknown,
51}
52
53/// One lexical token: [`kind`](Token::kind), the source [`slice`](Token::text)
54/// it covers, and the byte offset it starts at.
55///
56/// The text is borrowed from the expression — lexing allocates nothing but the
57/// returned `Vec`.
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
59pub struct Token<'a> {
60    /// What the token is.
61    pub kind: TokenKind,
62    /// The exact source text, delimiters and escapes included.
63    pub text: &'a str,
64    /// Byte offset of the token in the expression.
65    pub start: usize,
66}
67
68impl Token<'_> {
69    /// The byte offset one past the token's last byte.
70    #[must_use]
71    pub fn end(&self) -> usize {
72        self.start + self.text.len()
73    }
74}
75
76/// Two-character operators, checked before single characters.
77const TWO_CHAR_OPERATORS: [&str; 8] = ["=>", ":=", "==", "<>", ">=", "<=", "&&", "||"];
78
79/// Converts DAX source into tokens, ending with one [`TokenKind::Eof`].
80///
81/// Whitespace produces no tokens, so the token after any given token is exactly
82/// its next significant neighbour — the property the reference extractor's
83/// adjacency rules rely on.
84///
85/// ```
86/// use ripbi_core::dax::{TokenKind, tokenize};
87///
88/// let tokens = tokenize("SUM(Sales[Amount])");
89/// let kinds: Vec<TokenKind> = tokens.iter().map(|t| t.kind).collect();
90/// assert_eq!(
91///     kinds,
92///     [
93///         TokenKind::Identifier,
94///         TokenKind::OpenParen,
95///         TokenKind::Identifier,
96///         TokenKind::BracketName,
97///         TokenKind::CloseParen,
98///         TokenKind::Eof,
99///     ]
100/// );
101/// ```
102#[must_use]
103pub fn tokenize(source: &str) -> Vec<Token<'_>> {
104    let bytes = source.as_bytes();
105    let mut tokens = Vec::new();
106    let mut index = 0usize;
107
108    while index < source.len() {
109        let start = index;
110        // `index` is always on a char boundary: every advance below moves past a
111        // whole character (or an ASCII-only run).
112        let ch = source[index..].chars().next().expect("index < len");
113        let ch_len = ch.len_utf8();
114
115        if ch.is_whitespace() {
116            index += ch_len;
117            continue;
118        }
119
120        // Line comments: both -- and // are legal DAX.
121        let rest = &source[index..];
122        if rest.starts_with("--") || rest.starts_with("//") {
123            let end = rest.find(['\r', '\n']).map_or(source.len(), |i| index + i);
124            tokens.push(Token {
125                kind: TokenKind::Comment,
126                text: &source[start..end],
127                start,
128            });
129            index = end;
130            continue;
131        }
132
133        // Block comments: unterminated runs to the end of the input.
134        if let Some(after_open) = rest.strip_prefix("/*") {
135            let end = after_open
136                .find("*/")
137                .map_or(source.len(), |i| index + 2 + i + 2);
138            tokens.push(Token {
139                kind: TokenKind::Comment,
140                text: &source[start..end],
141                start,
142            });
143            index = end;
144            continue;
145        }
146
147        let kind;
148
149        // dt"2024-01-01" — the prefix belongs to the literal.
150        if matches!(ch, 'd' | 'D')
151            && bytes
152                .get(index + 1)
153                .is_some_and(|b| b.eq_ignore_ascii_case(&b't'))
154            && bytes.get(index + 2) == Some(&b'"')
155        {
156            index = scan_delimited(source, index + 2, b'"');
157            kind = TokenKind::DateTime;
158        } else if ch == '"' {
159            index = scan_delimited(source, index, b'"');
160            kind = TokenKind::String;
161        } else if ch == '\'' {
162            index = scan_delimited(source, index, b'\'');
163            kind = TokenKind::QuotedTable;
164        } else if ch == '[' {
165            index += 1;
166            while index < bytes.len() && bytes[index] != b']' {
167                index += 1;
168            }
169            if index < bytes.len() {
170                index += 1;
171            }
172            kind = TokenKind::BracketName;
173        } else if ch == '@' {
174            index = advance_while(source, index + 1, is_identifier_part);
175            kind = TokenKind::QueryParameter;
176        } else if ch.is_ascii_digit()
177            || (ch == '.' && bytes.get(index + 1).is_some_and(|b| b.is_ascii_digit()))
178        {
179            index = scan_number(source, index);
180            kind = TokenKind::Number;
181        } else if is_identifier_start(ch) {
182            index = scan_identifier(source, index);
183            kind = TokenKind::Identifier;
184        } else if TWO_CHAR_OPERATORS.iter().any(|op| rest.starts_with(op)) {
185            index += 2;
186            kind = TokenKind::Operator;
187        } else {
188            index += ch_len;
189            kind = match ch {
190                '(' => TokenKind::OpenParen,
191                ')' => TokenKind::CloseParen,
192                '{' => TokenKind::OpenBrace,
193                '}' => TokenKind::CloseBrace,
194                ',' => TokenKind::Comma,
195                ';' => TokenKind::Semicolon,
196                ':' => TokenKind::Colon,
197                '.' => TokenKind::Dot,
198                '+' | '-' | '*' | '/' | '^' | '&' | '=' | '<' | '>' | '!' => TokenKind::Operator,
199                _ => TokenKind::Unknown,
200            };
201        }
202
203        tokens.push(Token {
204            kind,
205            text: &source[start..index],
206            start,
207        });
208    }
209
210    tokens.push(Token {
211        kind: TokenKind::Eof,
212        text: "",
213        start: source.len(),
214    });
215    tokens
216}
217
218/// Scans a quoted run, treating a doubled delimiter as an escaped delimiter.
219/// Unterminated input consumes the rest of the source.
220fn scan_delimited(source: &str, mut index: usize, delimiter: u8) -> usize {
221    let bytes = source.as_bytes();
222    index += 1; // opening delimiter
223    while index < bytes.len() {
224        if bytes[index] != delimiter {
225            index += 1;
226            continue;
227        }
228        if bytes.get(index + 1) == Some(&delimiter) {
229            index += 2;
230            continue;
231        }
232        return index + 1;
233    }
234    index
235}
236
237fn scan_number(source: &str, mut index: usize) -> usize {
238    let bytes = source.as_bytes();
239    while index < bytes.len() && (bytes[index].is_ascii_digit() || bytes[index] == b'.') {
240        index += 1;
241    }
242
243    // Exponent: 1.5E+10, 2e-3. Only consume it when digits actually follow, so
244    // that a column reference such as Sales[E] cannot be mistaken for an exponent.
245    if bytes
246        .get(index)
247        .is_some_and(|b| b.eq_ignore_ascii_case(&b'e'))
248    {
249        let mut lookahead = index + 1;
250        if matches!(bytes.get(lookahead), Some(&b'+') | Some(&b'-')) {
251            lookahead += 1;
252        }
253        if bytes.get(lookahead).is_some_and(|b| b.is_ascii_digit()) {
254            index = lookahead;
255            while index < bytes.len() && bytes[index].is_ascii_digit() {
256                index += 1;
257            }
258        }
259    }
260    index
261}
262
263/// Scans an identifier, absorbing the dots inside names such as `NORM.DIST` or
264/// `CHISQ.INV.RT`. A dot that is not followed by an identifier character is left
265/// as its own token, so `'Date'.[Date]` still lexes as three tokens.
266fn scan_identifier(source: &str, mut index: usize) -> usize {
267    index = advance_while(source, index, is_identifier_part);
268    while bytes_at(source, index) == Some(b'.')
269        && source[index + 1..]
270            .chars()
271            .next()
272            .is_some_and(is_identifier_part)
273    {
274        index = advance_while(source, index + 1, is_identifier_part);
275    }
276    index
277}
278
279/// The byte at `index`, when it is inside the source. Multi-byte characters are
280/// never equal to an ASCII byte, so this is safe to branch on.
281fn bytes_at(source: &str, index: usize) -> Option<u8> {
282    source.as_bytes().get(index).copied()
283}
284
285/// Advances `index` past every character satisfying `predicate`.
286fn advance_while(source: &str, mut index: usize, predicate: impl Fn(char) -> bool) -> usize {
287    while let Some(ch) = source[index..].chars().next() {
288        if !predicate(ch) {
289            break;
290        }
291        index += ch.len_utf8();
292    }
293    index
294}
295
296fn is_identifier_start(ch: char) -> bool {
297    ch.is_alphabetic() || ch == '_'
298}
299
300fn is_identifier_part(ch: char) -> bool {
301    ch.is_alphanumeric() || ch == '_'
302}
303
304#[cfg(test)]
305mod tests {
306    use super::*;
307    use rstest::rstest;
308
309    /// The kinds of every token, including the trailing `Eof` — the shape-level
310    /// assertions below all compare against this compact form.
311    fn kinds(source: &str) -> Vec<TokenKind> {
312        tokenize(source).into_iter().map(|t| t.kind).collect()
313    }
314
315    /// The text of every token except the trailing `Eof`.
316    fn texts(source: &str) -> Vec<&str> {
317        let all = tokenize(source);
318        let end = all.len() - 1; // drop Eof, whose text is empty
319        all[..end].iter().map(|t| t.text).collect()
320    }
321
322    #[test]
323    fn tokenizes_a_simple_measure_expression() {
324        assert_eq!(
325            kinds("SUM(Sales[Amount]) + 1"),
326            [
327                TokenKind::Identifier,
328                TokenKind::OpenParen,
329                TokenKind::Identifier,
330                TokenKind::BracketName,
331                TokenKind::CloseParen,
332                TokenKind::Operator,
333                TokenKind::Number,
334                TokenKind::Eof,
335            ]
336        );
337    }
338
339    #[test]
340    fn empty_input_produces_only_eof() {
341        assert_eq!(kinds(""), [TokenKind::Eof]);
342        assert_eq!(kinds("   \r\n\t"), [TokenKind::Eof]);
343    }
344
345    #[test]
346    fn eof_marks_the_end_of_input() {
347        let tokens = tokenize("[X]");
348        let eof = tokens.last().expect("eof token");
349        assert_eq!(eof.kind, TokenKind::Eof);
350        assert_eq!(eof.start, 3);
351        assert_eq!(eof.text, "");
352    }
353
354    #[test]
355    fn tokens_carry_their_exact_source_slices_and_offsets() {
356        let source = "SUM('Sales Header'[Net Price])";
357        let tokens = tokenize(source);
358        for token in &tokens {
359            assert_eq!(&source[token.start..token.end()], token.text);
360        }
361        assert_eq!(
362            texts(source),
363            ["SUM", "(", "'Sales Header'", "[Net Price]", ")"]
364        );
365    }
366
367    #[test]
368    fn quoted_tables_keep_doubled_quotes_verbatim() {
369        // The lexer preserves escapes; logical unescaping happens at
370        // materialization (see refs::unescape_name).
371        assert_eq!(texts("'It''s'[X]"), ["'It''s'", "[X]"]);
372        assert_eq!(
373            texts("'Sales''s Data'[Amount]"),
374            ["'Sales''s Data'", "[Amount]"]
375        );
376    }
377
378    #[test]
379    fn strings_keep_doubled_quotes_verbatim() {
380        assert_eq!(texts(r#""say ""hi"" ok""#), [r#""say ""hi"" ok""#]);
381    }
382
383    #[test]
384    fn identifiers_absorb_internal_dots() {
385        assert_eq!(
386            texts("NORM.DIST(1, 2, TRUE)"),
387            ["NORM.DIST", "(", "1", ",", "2", ",", "TRUE", ")"]
388        );
389        assert_eq!(texts("CHISQ.INV.RT(x)"), ["CHISQ.INV.RT", "(", "x", ")"]);
390    }
391
392    #[test]
393    fn a_trailing_dot_stays_its_own_token() {
394        assert_eq!(
395            kinds("'Date'.[Date]"),
396            [
397                TokenKind::QuotedTable,
398                TokenKind::Dot,
399                TokenKind::BracketName,
400                TokenKind::Eof,
401            ]
402        );
403    }
404
405    #[rstest]
406    #[case("Sales[E]", &["Sales", "[E]"])]
407    #[case("1.5E+10", &["1.5E+10"])]
408    #[case("2e-3", &["2e-3"])]
409    #[case("1.5E", &["1.5", "E"])]
410    #[case("1E", &["1", "E"])]
411    #[case(".5", &[".5"])]
412    #[case("1.2.3", &["1.2.3"])]
413    fn numbers_follow_the_exponent_lookahead_rule(#[case] source: &str, #[case] expected: &[&str]) {
414        assert_eq!(texts(source), expected);
415    }
416
417    #[test]
418    fn comment_markers_inside_a_string_are_not_a_comment() {
419        // Ported from the SQLBI smoke tests: the "--" inside the string literal
420        // must remain part of the string.
421        assert_eq!(
422            kinds(r#"VAR Note = "-- not a comment""#),
423            [
424                TokenKind::Identifier,
425                TokenKind::Identifier,
426                TokenKind::Operator,
427                TokenKind::String,
428                TokenKind::Eof,
429            ]
430        );
431        assert_eq!(
432            texts(r#"VAR Note = "-- not a comment""#),
433            ["VAR", "Note", "=", r#""-- not a comment""#,]
434        );
435    }
436
437    #[rstest]
438    #[case("-- line comment")]
439    #[case("// line comment")]
440    #[case("/* block comment */")]
441    fn comments_become_single_comment_tokens(#[case] source: &str) {
442        assert_eq!(kinds(source), [TokenKind::Comment, TokenKind::Eof]);
443    }
444
445    #[test]
446    fn line_comments_end_at_the_line_break() {
447        let source = "[A] -- [B] not a ref\n[C]";
448        let tokens = tokenize(source);
449        assert_eq!(texts(source), ["[A]", "-- [B] not a ref", "[C]"]);
450        let comment = &tokens[1];
451        assert_eq!(comment.kind, TokenKind::Comment);
452        assert!(!comment.text.contains('\n'));
453    }
454
455    #[test]
456    fn unterminated_block_comment_runs_to_the_end() {
457        assert_eq!(kinds("/* [hidden"), [TokenKind::Comment, TokenKind::Eof]);
458        assert_eq!(
459            tokenize("/* [hidden").first().expect("comment").text,
460            "/* [hidden"
461        );
462    }
463
464    #[test]
465    fn datetime_literals_swallow_the_whole_prefix() {
466        assert_eq!(
467            kinds("dt\"2024-01-01\""),
468            [TokenKind::DateTime, TokenKind::Eof]
469        );
470        assert_eq!(
471            kinds("DT\"2024-01-01\""),
472            [TokenKind::DateTime, TokenKind::Eof]
473        );
474        assert_eq!(texts("dt\"2024-01-01\""), ["dt\"2024-01-01\""]);
475    }
476
477    #[rstest]
478    #[case("==")]
479    #[case("<>")]
480    #[case(">=")]
481    #[case("<=")]
482    #[case("&&")]
483    #[case("||")]
484    #[case("=>")]
485    #[case(":=")]
486    fn two_character_operators_lex_as_one_token(#[case] source: &str) {
487        assert_eq!(kinds(source), [TokenKind::Operator, TokenKind::Eof]);
488        assert_eq!(tokenize(source).first().expect("op").text, source);
489    }
490
491    #[test]
492    fn operators_prefer_the_two_character_form() {
493        // "a<=b" is a single <= operator, not < then =.
494        assert_eq!(texts("a<=b"), ["a", "<=", "b"]);
495        assert_eq!(texts("a<b"), ["a", "<", "b"]);
496    }
497
498    #[test]
499    fn query_parameters_take_the_identifier_tail() {
500        assert_eq!(kinds("@Risk"), [TokenKind::QueryParameter, TokenKind::Eof]);
501        assert_eq!(texts("@Risk"), ["@Risk"]);
502    }
503
504    #[test]
505    fn unknown_characters_never_fail_the_scan() {
506        assert_eq!(
507            kinds("[A] # [B]"),
508            [
509                TokenKind::BracketName,
510                TokenKind::Unknown,
511                TokenKind::BracketName,
512                TokenKind::Eof,
513            ]
514        );
515    }
516
517    #[rstest]
518    #[case("'Table")]
519    #[case("\"string")]
520    #[case("[Col")]
521    fn unterminated_delimiters_run_to_the_end_without_panicking(#[case] source: &str) {
522        let tokens = tokenize(source);
523        assert_eq!(tokens.len(), 2); // one token + Eof
524        assert_eq!(&source[tokens[0].start..tokens[0].end()], source);
525    }
526
527    #[test]
528    fn unicode_names_lex_as_single_identifiers() {
529        assert_eq!(texts("MÅNED.Æble"), ["MÅNED.Æble"]);
530        assert_eq!(texts("'Salg'[Årsag]"), ["'Salg'", "[Årsag]"]);
531    }
532
533    #[test]
534    fn unicode_whitespace_is_skipped_at_char_boundaries() {
535        // U+00A0 no-break space between the two identifiers.
536        let source = "A\u{00A0}B";
537        assert_eq!(texts(source), ["A", "B"]);
538    }
539
540    #[test]
541    fn multibyte_characters_do_not_distort_offsets() {
542        let source = "'Ærø'[Ø]";
543        let tokens = tokenize(source);
544        for token in &tokens {
545            assert_eq!(&source[token.start..token.end()], token.text);
546        }
547        assert_eq!(tokens.last().expect("eof").start, source.len());
548    }
549}
ripbi_core/dax/lexer.rs

ripbi_core/dax/
lexer.rs
