lemma/parsing/
lexer.rs

1use crate::error::Error;
2use crate::parsing::ast::{BooleanValue, ConversionTypeKind, PrimitiveKind, Span};
3use crate::parsing::source::Source;
4use std::sync::Arc;
5
6#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7pub enum TokenKind {
8    // Keywords
9    Spec,
10    Repo,
11    Data,
12    Rule,
13    Unless,
14    Then,
15    That,
16    Not,
17    And,
18    Or,
19    In,
20    As,
21    Uses,
22    With,
23    Effective,
24    Meta,
25    Veto,
26    Now,
27    Past,
28    Future,
29
30    // Boolean keywords
31    True,
32    False,
33    Yes,
34    No,
35
36    // Type keywords
37    MeasureKw,
38    NumberKw,
39    TextKw,
40    DateKw,
41    TimeKw,
42    BooleanKw,
43    RatioKw,
44
45    // Math function keywords
46    Sqrt,
47    Sin,
48    Cos,
49    Tan,
50    Asin,
51    Acos,
52    Atan,
53    Log,
54    Exp,
55    Abs,
56    Floor,
57    Ceil,
58    Round,
59
60    // List keywords
61    List,
62    Key,
63    Find,
64    First,
65    Last,
66    Sum,
67    Avg,
68    Product,
69    Count,
70    Min,
71    Max,
72    Any,
73    All,
74    Where,
75    Sort,
76    Top,
77    Asc,
78    Desc,
79
80    // Range endpoint operators
81    Lower,
82    Upper,
83
84    Permille,
85
86    // Comparison keyword operators
87    Is,
88
89    // Operators
90    Plus,
91    Minus,
92    Star,
93    Slash,
94    Comma,
95    Percent,
96    PercentPercent,
97    Mod,
98    Caret,
99    Gt,
100    Lt,
101    Gte,
102    Lte,
103
104    // Punctuation
105    Colon,
106    Arrow,
107    Ellipsis,
108    Dot,
109    LParen,
110    RParen,
111
112    // Literals
113    NumberLit,
114    StringLit,
115
116    // Commentary (raw text between """ delimiters)
117    Commentary,
118
119    // Identifiers
120    Identifier,
121
122    /// Opaque spec / repo / registry name: optional `@` + `[A-Za-z0-9_./-]+`.
123    LogicalName,
124
125    // End of file
126    Eof,
127}
128
129impl std::fmt::Display for TokenKind {
130    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
131        match self {
132            TokenKind::Spec => write!(f, "'spec'"),
133            TokenKind::Repo => write!(f, "'repo'"),
134            TokenKind::Data => write!(f, "'data'"),
135            TokenKind::Rule => write!(f, "'rule'"),
136            TokenKind::Unless => write!(f, "'unless'"),
137            TokenKind::Then => write!(f, "'then'"),
138            TokenKind::That => write!(f, "'that'"),
139            TokenKind::Not => write!(f, "'not'"),
140            TokenKind::And => write!(f, "'and'"),
141            TokenKind::Or => write!(f, "'or'"),
142            TokenKind::In => write!(f, "'in'"),
143            TokenKind::As => write!(f, "'as'"),
144            TokenKind::Uses => write!(f, "'uses'"),
145            TokenKind::With => write!(f, "'with'"),
146            TokenKind::Effective => write!(f, "'effective'"),
147            TokenKind::Meta => write!(f, "'meta'"),
148            TokenKind::Veto => write!(f, "'veto'"),
149            TokenKind::Now => write!(f, "'now'"),
150            TokenKind::Past => write!(f, "'past'"),
151            TokenKind::Future => write!(f, "'future'"),
152            TokenKind::True => write!(f, "'true'"),
153            TokenKind::False => write!(f, "'false'"),
154            TokenKind::Yes => write!(f, "'yes'"),
155            TokenKind::No => write!(f, "'no'"),
156            TokenKind::MeasureKw => write!(f, "'measure'"),
157            TokenKind::NumberKw => write!(f, "'number'"),
158            TokenKind::TextKw => write!(f, "'text'"),
159            TokenKind::DateKw => write!(f, "'date'"),
160            TokenKind::TimeKw => write!(f, "'time'"),
161            TokenKind::BooleanKw => write!(f, "'boolean'"),
162            TokenKind::RatioKw => write!(f, "'ratio'"),
163            TokenKind::Sqrt => write!(f, "'sqrt'"),
164            TokenKind::Sin => write!(f, "'sin'"),
165            TokenKind::Cos => write!(f, "'cos'"),
166            TokenKind::Tan => write!(f, "'tan'"),
167            TokenKind::Asin => write!(f, "'asin'"),
168            TokenKind::Acos => write!(f, "'acos'"),
169            TokenKind::Atan => write!(f, "'atan'"),
170            TokenKind::Log => write!(f, "'log'"),
171            TokenKind::Exp => write!(f, "'exp'"),
172            TokenKind::Abs => write!(f, "'abs'"),
173            TokenKind::Floor => write!(f, "'floor'"),
174            TokenKind::Ceil => write!(f, "'ceil'"),
175            TokenKind::Round => write!(f, "'round'"),
176            TokenKind::List => write!(f, "'list'"),
177            TokenKind::Key => write!(f, "'key'"),
178            TokenKind::Find => write!(f, "'find'"),
179            TokenKind::First => write!(f, "'first'"),
180            TokenKind::Last => write!(f, "'last'"),
181            TokenKind::Sum => write!(f, "'sum'"),
182            TokenKind::Avg => write!(f, "'avg'"),
183            TokenKind::Product => write!(f, "'product'"),
184            TokenKind::Count => write!(f, "'count'"),
185            TokenKind::Min => write!(f, "'min'"),
186            TokenKind::Max => write!(f, "'max'"),
187            TokenKind::Any => write!(f, "'any'"),
188            TokenKind::All => write!(f, "'all'"),
189            TokenKind::Where => write!(f, "'where'"),
190            TokenKind::Sort => write!(f, "'sort'"),
191            TokenKind::Top => write!(f, "'top'"),
192            TokenKind::Asc => write!(f, "'asc'"),
193            TokenKind::Desc => write!(f, "'desc'"),
194            TokenKind::Lower => write!(f, "'lower'"),
195            TokenKind::Upper => write!(f, "'upper'"),
196            TokenKind::Permille => write!(f, "'permille'"),
197            TokenKind::Is => write!(f, "'is'"),
198            TokenKind::Plus => write!(f, "'+'"),
199            TokenKind::Minus => write!(f, "'-'"),
200            TokenKind::Star => write!(f, "'*'"),
201            TokenKind::Slash => write!(f, "'/'"),
202            TokenKind::Comma => write!(f, "','"),
203            TokenKind::Percent => write!(f, "'%'"),
204            TokenKind::PercentPercent => write!(f, "'%%'"),
205            TokenKind::Mod => write!(f, "'mod'"),
206            TokenKind::Caret => write!(f, "'^'"),
207            TokenKind::Gt => write!(f, "'>'"),
208            TokenKind::Lt => write!(f, "'<'"),
209            TokenKind::Gte => write!(f, "'>='"),
210            TokenKind::Lte => write!(f, "'<='"),
211            TokenKind::Colon => write!(f, "':'"),
212            TokenKind::Arrow => write!(f, "'->'"),
213            TokenKind::Ellipsis => write!(f, "'...'"),
214            TokenKind::Dot => write!(f, "'.'"),
215            TokenKind::LParen => write!(f, "'('"),
216            TokenKind::RParen => write!(f, "')'"),
217            TokenKind::NumberLit => write!(f, "a number"),
218            TokenKind::StringLit => write!(f, "a string"),
219            TokenKind::Commentary => write!(f, "commentary block"),
220            TokenKind::Identifier => write!(f, "an identifier"),
221            TokenKind::LogicalName => write!(f, "a name"),
222            TokenKind::Eof => write!(f, "end of file"),
223        }
224    }
225}
226
227/// Byte range of a token's content in the source. For `StringLit` this excludes
228/// the quotes (the span covers them); for `Commentary` it is the content between
229/// the triple quotes; for every other kind it equals the span.
230#[derive(Debug, Clone, Copy)]
231pub struct TokenText {
232    pub start: usize,
233    pub end: usize,
234}
235
236impl TokenText {
237    /// The content of this range in `source`.
238    pub fn of(self, source: &str) -> &str {
239        &source[self.start..self.end]
240    }
241
242    pub fn len(&self) -> usize {
243        self.end - self.start
244    }
245
246    pub fn is_empty(&self) -> bool {
247        self.end == self.start
248    }
249}
250
251#[derive(Debug, Clone, Copy)]
252pub struct Token {
253    pub kind: TokenKind,
254    pub span: Span,
255    pub text: TokenText,
256}
257
258impl Token {
259    pub fn eof(offset: usize, line: usize, col: usize) -> Self {
260        Token {
261            kind: TokenKind::Eof,
262            span: Span {
263                start: offset,
264                end: offset,
265                line,
266                col,
267            },
268            text: TokenText {
269                start: offset,
270                end: offset,
271            },
272        }
273    }
274}
275
276#[derive(Clone, Copy)]
277pub struct LexerCheckpoint {
278    line: usize,
279    col: usize,
280    byte_offset: usize,
281    peeked: Option<Token>,
282    peeked2: Option<Token>,
283}
284
285pub struct Lexer {
286    line: usize,
287    col: usize,
288    byte_offset: usize,
289    source_type: crate::parsing::source::SourceType,
290    source_text: Arc<str>,
291    peeked: Option<Token>,
292    peeked2: Option<Token>,
293}
294
295impl Lexer {
296    pub fn new(
297        input: impl Into<Arc<str>>,
298        source_type: &crate::parsing::source::SourceType,
299    ) -> Self {
300        Lexer {
301            line: 1,
302            col: 1,
303            byte_offset: 0,
304            source_type: source_type.clone(),
305            source_text: input.into(),
306            peeked: None,
307            peeked2: None,
308        }
309    }
310
311    pub fn peek(&mut self) -> Result<&Token, Error> {
312        if self.peeked.is_none() {
313            let token = self.lex_token()?;
314            self.peeked = Some(token);
315        }
316        Ok(self.peeked.as_ref().expect("just assigned"))
317    }
318
319    pub fn peek_second(&mut self) -> Result<&Token, Error> {
320        self.peek()?;
321        if self.peeked2.is_none() {
322            let token = self.lex_token()?;
323            self.peeked2 = Some(token);
324        }
325        Ok(self.peeked2.as_ref().expect("just assigned"))
326    }
327
328    pub fn next_token(&mut self) -> Result<Token, Error> {
329        if let Some(token) = self.peeked.take() {
330            self.peeked = self.peeked2.take();
331            return Ok(token);
332        }
333        self.lex_token()
334    }
335
336    /// Opaque spec / repo / registry name for name positions only.
337    ///
338    /// If a normal peek already materialized a prefix token, rewind and rescan
339    /// so connectors (`/`, `.`, `-`) are absorbed into one [`TokenKind::LogicalName`].
340    pub fn next_logical_name(&mut self) -> Result<Token, Error> {
341        if let Some(tok) = &self.peeked {
342            if tok.kind == TokenKind::LogicalName {
343                return self.next_token();
344            }
345            let start_byte = tok.span.start;
346            let start_line = tok.span.line;
347            let start_col = tok.span.col;
348            self.peeked = None;
349            self.peeked2 = None;
350            self.rewind_to(start_byte, start_line, start_col);
351            return self.scan_integral_logical_name(start_byte, start_line, start_col);
352        }
353        self.skip_whitespace();
354        let start_byte = self.byte_offset;
355        let start_line = self.line;
356        let start_col = self.col;
357        self.scan_integral_logical_name(start_byte, start_line, start_col)
358    }
359
360    fn rewind_to(&mut self, byte_offset: usize, line: usize, col: usize) {
361        self.byte_offset = byte_offset;
362        self.line = line;
363        self.col = col;
364    }
365
366    /// Saved lexer position for speculative parsing.
367    pub fn checkpoint(&self) -> LexerCheckpoint {
368        LexerCheckpoint {
369            line: self.line,
370            col: self.col,
371            byte_offset: self.byte_offset,
372            peeked: self.peeked,
373            peeked2: self.peeked2,
374        }
375    }
376
377    pub fn restore(&mut self, checkpoint: LexerCheckpoint) {
378        self.line = checkpoint.line;
379        self.col = checkpoint.col;
380        self.byte_offset = checkpoint.byte_offset;
381        self.peeked = checkpoint.peeked;
382        self.peeked2 = checkpoint.peeked2;
383    }
384
385    fn chars_from(&self, byte_offset: usize) -> std::str::Chars<'_> {
386        self.source_text[byte_offset..].chars()
387    }
388
389    fn current_char(&self) -> Option<char> {
390        self.chars_from(self.byte_offset).next()
391    }
392
393    fn peek_char(&self) -> Option<char> {
394        let mut chars = self.chars_from(self.byte_offset);
395        chars.next();
396        chars.next()
397    }
398
399    fn peek_char_at(&self, offset: usize) -> Option<char> {
400        self.chars_from(self.byte_offset).nth(offset)
401    }
402
403    fn advance(&mut self) {
404        if let Some(ch) = self.current_char() {
405            self.byte_offset += ch.len_utf8();
406            if ch == '\n' {
407                self.line += 1;
408                self.col = 1;
409            } else {
410                self.col += 1;
411            }
412        }
413    }
414
415    fn skip_whitespace(&mut self) {
416        while let Some(ch) = self.current_char() {
417            if ch.is_whitespace() {
418                self.advance();
419            } else {
420                break;
421            }
422        }
423    }
424
425    fn make_span(&self, start_byte: usize, start_line: usize, start_col: usize) -> Span {
426        Span {
427            start: start_byte,
428            end: self.byte_offset,
429            line: start_line,
430            col: start_col,
431        }
432    }
433
434    fn make_error(&self, message: impl Into<String>, span: Span) -> Error {
435        Error::parsing(
436            message,
437            Source::new(self.source_type.clone(), span),
438            None::<String>,
439        )
440    }
441
442    fn lex_token(&mut self) -> Result<Token, Error> {
443        self.skip_whitespace();
444
445        let start_byte = self.byte_offset;
446        let start_line = self.line;
447        let start_col = self.col;
448
449        let Some(ch) = self.current_char() else {
450            return Ok(Token::eof(start_byte, start_line, start_col));
451        };
452
453        // Triple-quote commentary
454        if ch == '"' && self.peek_char() == Some('"') && self.peek_char_at(2) == Some('"') {
455            return self.scan_triple_quote(start_byte, start_line, start_col);
456        }
457
458        // String literal
459        if ch == '"' {
460            return self.scan_string(start_byte, start_line, start_col);
461        }
462
463        // Number literal (sign handled by parser, not lexer)
464        if ch.is_ascii_digit() {
465            return self.scan_number(start_byte, start_line, start_col);
466        }
467
468        // Two-character operators (check before single-char)
469        if let Some(token) = self.try_two_char_operator(start_byte, start_line, start_col) {
470            return Ok(token);
471        }
472
473        // Three-character ellipsis
474        if ch == '.' && self.peek_char() == Some('.') && self.peek_char_at(2) == Some('.') {
475            self.advance();
476            self.advance();
477            self.advance();
478            let span = self.make_span(start_byte, start_line, start_col);
479            return Ok(Token {
480                kind: TokenKind::Ellipsis,
481                text: TokenText {
482                    start: span.start,
483                    end: span.end,
484                },
485                span,
486            });
487        }
488
489        // Single-character operators/punctuation
490        if let Some(kind) = self.single_char_token(ch) {
491            self.advance();
492            let span = self.make_span(start_byte, start_line, start_col);
493            return Ok(Token {
494                kind,
495                text: TokenText {
496                    start: span.start,
497                    end: span.end,
498                },
499                span,
500            });
501        }
502
503        // Identifier or keyword
504        if ch.is_ascii_alphabetic() || ch == '_' {
505            return Ok(self.scan_identifier(start_byte, start_line, start_col));
506        }
507
508        // Registry qualifier: `@` + opaque name characters
509        if ch == '@' {
510            return self.scan_integral_logical_name(start_byte, start_line, start_col);
511        }
512
513        // Unknown character
514        self.advance();
515        let span = self.make_span(start_byte, start_line, start_col);
516        Err(self.make_error(format!("Unexpected character '{}'", ch), span))
517    }
518
519    fn is_logical_name_char(ch: char) -> bool {
520        ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '/' | '-')
521    }
522
523    fn scan_integral_logical_name(
524        &mut self,
525        start_byte: usize,
526        start_line: usize,
527        start_col: usize,
528    ) -> Result<Token, Error> {
529        let had_at = self.current_char() == Some('@');
530        if had_at {
531            self.advance();
532        }
533        let Some(first) = self.current_char() else {
534            let span = self.make_span(start_byte, start_line, start_col);
535            return Err(self.make_error(
536                if had_at {
537                    "Expected a name after '@'"
538                } else {
539                    "Expected a name"
540                },
541                span,
542            ));
543        };
544        if !Self::is_logical_name_char(first) {
545            let span = self.make_span(start_byte, start_line, start_col);
546            return Err(self.make_error(
547                if had_at {
548                    format!("Expected a name after '@', found '{}'", first)
549                } else {
550                    format!("Expected a name, found '{}'", first)
551                },
552                span,
553            ));
554        }
555        while let Some(ch) = self.current_char() {
556            if Self::is_logical_name_char(ch) {
557                self.advance();
558            } else {
559                break;
560            }
561        }
562        let span = self.make_span(start_byte, start_line, start_col);
563        Ok(Token {
564            kind: TokenKind::LogicalName,
565            text: TokenText {
566                start: span.start,
567                end: span.end,
568            },
569            span,
570        })
571    }
572
573    fn scan_triple_quote(
574        &mut self,
575        start_byte: usize,
576        start_line: usize,
577        start_col: usize,
578    ) -> Result<Token, Error> {
579        self.advance(); // "
580        self.advance(); // "
581        self.advance(); // "
582
583        let content_start = self.byte_offset;
584        loop {
585            match self.current_char() {
586                None => {
587                    let span = self.make_span(start_byte, start_line, start_col);
588                    return Err(self.make_error(
589                        "Unterminated commentary block: expected closing \"\"\"",
590                        span,
591                    ));
592                }
593                Some('"') if self.peek_char() == Some('"') && self.peek_char_at(2) == Some('"') => {
594                    let content_end = self.byte_offset;
595                    self.advance(); // "
596                    self.advance(); // "
597                    self.advance(); // "
598                    let span = self.make_span(start_byte, start_line, start_col);
599                    return Ok(Token {
600                        kind: TokenKind::Commentary,
601                        text: TokenText {
602                            start: content_start,
603                            end: content_end,
604                        },
605                        span,
606                    });
607                }
608                Some(_) => {
609                    self.advance();
610                }
611            }
612        }
613    }
614
615    fn scan_string(
616        &mut self,
617        start_byte: usize,
618        start_line: usize,
619        start_col: usize,
620    ) -> Result<Token, Error> {
621        self.advance(); // consume opening "
622        let content_start = self.byte_offset;
623        let content_end = loop {
624            match self.current_char() {
625                None => {
626                    let span = self.make_span(start_byte, start_line, start_col);
627                    return Err(self.make_error("String starting here was never closed", span));
628                }
629                Some('"') => {
630                    break self.byte_offset;
631                }
632                Some(_) => {
633                    self.advance();
634                }
635            }
636        };
637        let content_length = self.source_text[content_start..content_end].len();
638        self.advance(); // consume closing "
639        let span = self.make_span(start_byte, start_line, start_col);
640        if content_length > crate::limits::MAX_TEXT_VALUE_LENGTH {
641            return Err(self.make_error(
642                format!(
643                    "Text literal exceeds maximum length of {} characters (found {})",
644                    crate::limits::MAX_TEXT_VALUE_LENGTH,
645                    content_length
646                ),
647                span,
648            ));
649        }
650        // Token text is the unquoted content. The span still covers the quotes so
651        // diagnostics point at the full literal; the parser must not strip again.
652        Ok(Token {
653            kind: TokenKind::StringLit,
654            text: TokenText {
655                start: content_start,
656                end: content_end,
657            },
658            span,
659        })
660    }
661
662    fn scan_number(
663        &mut self,
664        start_byte: usize,
665        start_line: usize,
666        start_col: usize,
667    ) -> Result<Token, Error> {
668        // Integer part: digits with optional `_` separators
669        while let Some(ch) = self.current_char() {
670            if ch.is_ascii_digit() || ch == '_' {
671                self.advance();
672            } else {
673                break;
674            }
675        }
676
677        // Decimal part
678        if self.current_char() == Some('.') {
679            // Check if next char after dot is a digit (not a method call or dotted reference)
680            if let Some(next) = self.peek_char() {
681                if next.is_ascii_digit() {
682                    self.advance(); // consume .
683                    while let Some(ch) = self.current_char() {
684                        if ch.is_ascii_digit() {
685                            self.advance();
686                        } else {
687                            break;
688                        }
689                    }
690                }
691            }
692        }
693
694        // Scientific notation: e or E followed by optional +/- and digits
695        if let Some(ch) = self.current_char() {
696            if ch == 'e' || ch == 'E' {
697                let save_byte = self.byte_offset;
698                let save_line = self.line;
699                let save_col = self.col;
700                self.advance(); // consume e/E
701
702                if let Some(sign) = self.current_char() {
703                    if sign == '+' || sign == '-' {
704                        self.advance();
705                    }
706                }
707
708                if let Some(d) = self.current_char() {
709                    if d.is_ascii_digit() {
710                        while let Some(ch) = self.current_char() {
711                            if ch.is_ascii_digit() {
712                                self.advance();
713                            } else {
714                                break;
715                            }
716                        }
717                    } else {
718                        // Not actually scientific notation, backtrack
719                        self.byte_offset = save_byte;
720                        self.line = save_line;
721                        self.col = save_col;
722                    }
723                } else {
724                    self.byte_offset = save_byte;
725                    self.line = save_line;
726                    self.col = save_col;
727                }
728            }
729        }
730
731        let span = self.make_span(start_byte, start_line, start_col);
732        Ok(Token {
733            kind: TokenKind::NumberLit,
734            text: TokenText {
735                start: span.start,
736                end: span.end,
737            },
738            span,
739        })
740    }
741
742    fn try_two_char_operator(
743        &mut self,
744        start_byte: usize,
745        start_line: usize,
746        start_col: usize,
747    ) -> Option<Token> {
748        let ch = self.current_char()?;
749        let next = self.peek_char();
750
751        let kind = match (ch, next) {
752            ('-', Some('>')) => TokenKind::Arrow,
753            ('>', Some('=')) => TokenKind::Gte,
754            ('<', Some('=')) => TokenKind::Lte,
755            ('%', Some('%')) => {
756                // Check that it's not followed by a digit (invalid permille like 10%%5)
757                TokenKind::PercentPercent
758            }
759            _ => return None,
760        };
761
762        self.advance();
763        self.advance();
764        let span = self.make_span(start_byte, start_line, start_col);
765        Some(Token {
766            kind,
767            text: TokenText {
768                start: span.start,
769                end: span.end,
770            },
771            span,
772        })
773    }
774
775    fn single_char_token(&self, ch: char) -> Option<TokenKind> {
776        match ch {
777            '+' => Some(TokenKind::Plus),
778            '*' => Some(TokenKind::Star),
779            '/' => Some(TokenKind::Slash),
780            ',' => Some(TokenKind::Comma),
781            '^' => Some(TokenKind::Caret),
782            ':' => Some(TokenKind::Colon),
783            '.' => Some(TokenKind::Dot),
784            '(' => Some(TokenKind::LParen),
785            ')' => Some(TokenKind::RParen),
786            '>' => Some(TokenKind::Gt),
787            '<' => Some(TokenKind::Lt),
788            '%' => Some(TokenKind::Percent),
789            '-' => Some(TokenKind::Minus),
790            _ => None,
791        }
792    }
793
794    fn scan_identifier(&mut self, start_byte: usize, start_line: usize, start_col: usize) -> Token {
795        while let Some(ch) = self.current_char() {
796            if ch.is_ascii_alphanumeric() || ch == '_' {
797                self.advance();
798            } else {
799                break;
800            }
801        }
802
803        let span = self.make_span(start_byte, start_line, start_col);
804        let kind = keyword_from_identifier(&self.source_text[start_byte..self.byte_offset]);
805        Token {
806            kind,
807            text: TokenText {
808                start: span.start,
809                end: span.end,
810            },
811            span,
812        }
813    }
814}
815
816macro_rules! lemma_keywords {
817    ($(($spell:literal, $kind:ident)),+ $(,)?) => {
818        /// Spellings shared by [`keyword_from_identifier`] and LSP completion.
819        pub const COMPLETION_KEYWORD_LABELS: &[&str] = &[$($spell),+];
820
821        fn keyword_from_identifier(text: &str) -> TokenKind {
822            match text {
823                $(s if s.eq_ignore_ascii_case($spell) => TokenKind::$kind,)+
824                _ => TokenKind::Identifier,
825            }
826        }
827    };
828}
829
830lemma_keywords! {
831    ("spec", Spec),
832    ("repo", Repo),
833    ("data", Data),
834    ("rule", Rule),
835    ("unless", Unless),
836    ("then", Then),
837    ("that", That),
838    ("not", Not),
839    ("and", And),
840    ("or", Or),
841    ("in", In),
842    ("as", As),
843    ("uses", Uses),
844    ("with", With),
845    ("effective", Effective),
846    ("meta", Meta),
847    ("veto", Veto),
848    ("now", Now),
849    ("past", Past),
850    ("future", Future),
851    ("true", True),
852    ("false", False),
853    ("yes", Yes),
854    ("no", No),
855    ("measure", MeasureKw),
856    ("number", NumberKw),
857    ("text", TextKw),
858    ("date", DateKw),
859    ("time", TimeKw),
860    ("boolean", BooleanKw),
861    ("ratio", RatioKw),
862    ("sqrt", Sqrt),
863    ("sin", Sin),
864    ("cos", Cos),
865    ("tan", Tan),
866    ("asin", Asin),
867    ("acos", Acos),
868    ("atan", Atan),
869    ("log", Log),
870    ("exp", Exp),
871    ("abs", Abs),
872    ("floor", Floor),
873    ("ceil", Ceil),
874    ("round", Round),
875    ("list", List),
876    ("key", Key),
877    ("find", Find),
878    ("first", First),
879    ("last", Last),
880    ("sum", Sum),
881    ("avg", Avg),
882    ("product", Product),
883    ("count", Count),
884    ("min", Min),
885    ("max", Max),
886    ("any", Any),
887    ("all", All),
888    ("where", Where),
889    ("sort", Sort),
890    ("top", Top),
891    ("asc", Asc),
892    ("desc", Desc),
893    ("lower", Lower),
894    ("upper", Upper),
895    ("is", Is),
896    ("permille", Permille),
897    ("mod", Mod),
898}
899
900/// Structural keywords can never be used as identifiers (data/rule names).
901/// Type keywords (measure, number, text, date, time, boolean, ratio)
902/// are reserved and cannot be used as names.
903pub fn is_keyword(kind: &TokenKind) -> bool {
904    matches!(
905        kind,
906        TokenKind::Spec
907            | TokenKind::Repo
908            | TokenKind::Data
909            | TokenKind::Rule
910            | TokenKind::Unless
911            | TokenKind::Then
912            | TokenKind::That
913            | TokenKind::Not
914            | TokenKind::And
915            | TokenKind::Or
916            | TokenKind::In
917            | TokenKind::As
918            | TokenKind::Uses
919            | TokenKind::With
920            | TokenKind::Effective
921            | TokenKind::Meta
922            | TokenKind::Veto
923            | TokenKind::Now
924            | TokenKind::Is
925            | TokenKind::Past
926            | TokenKind::Future
927            | TokenKind::Permille
928            | TokenKind::Mod
929            | TokenKind::Sqrt
930            | TokenKind::Sin
931            | TokenKind::Cos
932            | TokenKind::Tan
933            | TokenKind::Asin
934            | TokenKind::Acos
935            | TokenKind::Atan
936            | TokenKind::Log
937            | TokenKind::Exp
938            | TokenKind::Abs
939            | TokenKind::Floor
940            | TokenKind::Ceil
941            | TokenKind::Round
942            | TokenKind::List
943            | TokenKind::Key
944            | TokenKind::Find
945            | TokenKind::First
946            | TokenKind::Last
947            | TokenKind::Sum
948            | TokenKind::Avg
949            | TokenKind::Product
950            | TokenKind::Count
951            | TokenKind::Min
952            | TokenKind::Max
953            | TokenKind::Any
954            | TokenKind::All
955            | TokenKind::Where
956            | TokenKind::Sort
957            | TokenKind::Top
958            | TokenKind::Asc
959            | TokenKind::Desc
960            | TokenKind::Lower
961            | TokenKind::Upper
962            | TokenKind::True
963            | TokenKind::False
964            | TokenKind::Yes
965            | TokenKind::No
966            | TokenKind::MeasureKw
967            | TokenKind::NumberKw
968            | TokenKind::TextKw
969            | TokenKind::DateKw
970            | TokenKind::TimeKw
971            | TokenKind::BooleanKw
972            | TokenKind::RatioKw
973    )
974}
975
976/// Map type keyword token to PrimitiveKind. Single source of truth for type keywords.
977#[must_use]
978pub fn token_kind_to_primitive(kind: &TokenKind) -> Option<PrimitiveKind> {
979    match kind {
980        TokenKind::BooleanKw => Some(PrimitiveKind::Boolean),
981        TokenKind::MeasureKw => Some(PrimitiveKind::Measure),
982        TokenKind::NumberKw => Some(PrimitiveKind::Number),
983        TokenKind::RatioKw => Some(PrimitiveKind::Ratio),
984        TokenKind::TextKw => Some(PrimitiveKind::Text),
985        TokenKind::DateKw => Some(PrimitiveKind::Date),
986        TokenKind::TimeKw => Some(PrimitiveKind::Time),
987        _ => None,
988    }
989}
990
991/// Map type keyword token to [`ConversionTypeKind`] for `as` targets. Returns
992/// `None` for `text` (display formatting is not a conversion) and for the
993/// range keywords (not accepted by the parser at this position anyway).
994#[must_use]
995pub fn token_kind_to_conversion_type_kind(kind: &TokenKind) -> Option<ConversionTypeKind> {
996    match kind {
997        TokenKind::BooleanKw => Some(ConversionTypeKind::Boolean),
998        TokenKind::MeasureKw => Some(ConversionTypeKind::Measure),
999        TokenKind::NumberKw => Some(ConversionTypeKind::Number),
1000        TokenKind::RatioKw => Some(ConversionTypeKind::Ratio),
1001        TokenKind::DateKw => Some(ConversionTypeKind::Date),
1002        TokenKind::TimeKw => Some(ConversionTypeKind::Time),
1003        _ => None,
1004    }
1005}
1006
1007/// Returns true if the token kind represents a boolean literal keyword.
1008pub fn is_boolean_keyword(kind: &TokenKind) -> bool {
1009    matches!(
1010        kind,
1011        TokenKind::True | TokenKind::False | TokenKind::Yes | TokenKind::No
1012    )
1013}
1014
1015/// Maps a boolean-keyword token kind to BooleanValue. Call only when `is_boolean_keyword(kind)`.
1016#[must_use]
1017pub fn token_kind_to_boolean_value(kind: &TokenKind) -> BooleanValue {
1018    match kind {
1019        TokenKind::True => BooleanValue::True,
1020        TokenKind::False => BooleanValue::False,
1021        TokenKind::Yes => BooleanValue::Yes,
1022        TokenKind::No => BooleanValue::No,
1023        _ => unreachable!(
1024            "BUG: token_kind_to_boolean_value called with non-boolean token {:?}",
1025            kind
1026        ),
1027    }
1028}
1029
1030/// Returns true if the token kind represents a math function keyword.
1031pub fn is_math_function(kind: &TokenKind) -> bool {
1032    matches!(
1033        kind,
1034        TokenKind::Sqrt
1035            | TokenKind::Sin
1036            | TokenKind::Cos
1037            | TokenKind::Tan
1038            | TokenKind::Asin
1039            | TokenKind::Acos
1040            | TokenKind::Atan
1041            | TokenKind::Log
1042            | TokenKind::Exp
1043            | TokenKind::Abs
1044    )
1045}
1046
1047/// Returns true if the token is a postfix rounding command written as a prefix by mistake.
1048pub fn is_prefix_rounding(kind: &TokenKind) -> bool {
1049    matches!(kind, TokenKind::Floor | TokenKind::Ceil | TokenKind::Round)
1050}
1051
1052/// Returns true if the token kind can start the body of a spec
1053/// (`data`, `rule`, `meta`, or `uses`).
1054pub fn is_spec_body_keyword(kind: &TokenKind) -> bool {
1055    matches!(
1056        kind,
1057        TokenKind::Data | TokenKind::Rule | TokenKind::Meta | TokenKind::Uses
1058    )
1059}
1060
1061/// Returns true if the token kind can be used as a label or reference segment
1062/// (identifier, or one of the keywords `past` / `future` / `permille` / `is` in label position).
1063pub fn can_be_label(kind: &TokenKind) -> bool {
1064    matches!(
1065        kind,
1066        TokenKind::Identifier
1067            | TokenKind::Past
1068            | TokenKind::Future
1069            | TokenKind::Permille
1070            | TokenKind::Is
1071    )
1072}
1073
1074/// Returns true if the token kind can name a unit (`5 eur`, `-> unit eur: 1`).
1075/// `is`, `past`, `future`, `permille` are reserved and never units: after a number they start a
1076/// comparison, a date window, or a literal suffix.
1077#[must_use]
1078pub fn can_be_unit_name(kind: &TokenKind) -> bool {
1079    matches!(kind, TokenKind::Identifier)
1080}
1081
1082/// True when a bare (non-`@`) repository name spelling is a reserved keyword.
1083#[must_use]
1084pub fn is_reserved_repository_name(name: &str) -> bool {
1085    is_keyword(&keyword_from_identifier(name))
1086}
1087
1088/// True when the opaque spelling is allowed as a `spec` name: a plain identifier, not a keyword.
1089#[must_use]
1090pub fn can_be_spec_name_spelling(name: &str) -> bool {
1091    matches!(keyword_from_identifier(name), TokenKind::Identifier)
1092}
1093
1094/// `calendar` in `in calendar month` / `past calendar year` — not a type keyword.
1095#[must_use]
1096pub fn token_is_calendar_period_marker(source: &str, tok: &Token) -> bool {
1097    tok.kind == TokenKind::Identifier && tok.text.of(source) == "calendar"
1098}
1099
1100#[cfg(test)]
1101mod tests {
1102    use super::*;
1103
1104    fn lex_all(input: &str) -> Result<Vec<Token>, Error> {
1105        let mut lexer = Lexer::new(input, &crate::parsing::source::SourceType::Volatile);
1106        let mut tokens = Vec::new();
1107        loop {
1108            let token = lexer.next_token()?;
1109            if token.kind == TokenKind::Eof {
1110                tokens.push(token);
1111                break;
1112            }
1113            tokens.push(token);
1114        }
1115        Ok(tokens)
1116    }
1117
1118    fn lex_kinds(input: &str) -> Result<Vec<TokenKind>, Error> {
1119        Ok(lex_all(input)?.into_iter().map(|t| t.kind).collect())
1120    }
1121
1122    #[test]
1123    fn lex_empty_input() {
1124        let tokens = lex_all("").unwrap();
1125        assert_eq!(tokens.len(), 1);
1126        assert_eq!(tokens[0].kind, TokenKind::Eof);
1127    }
1128
1129    #[test]
1130    fn completion_keyword_labels_match_identifier_lex() {
1131        for label in COMPLETION_KEYWORD_LABELS {
1132            let kinds = lex_kinds(label).unwrap();
1133            assert_eq!(
1134                kinds.last(),
1135                Some(&TokenKind::Eof),
1136                "keyword {label} must be a single token"
1137            );
1138            assert_eq!(kinds.len(), 2, "keyword {label} lexed as {kinds:?}");
1139            assert_ne!(
1140                kinds[0],
1141                TokenKind::Identifier,
1142                "catalog entry {label} must be a keyword token"
1143            );
1144        }
1145    }
1146
1147    #[test]
1148    fn string_literal_at_max_length_is_accepted() {
1149        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH);
1150        let tokens = lex_all(&format!("\"{content}\"")).unwrap();
1151        assert_eq!(tokens[0].kind, TokenKind::StringLit);
1152    }
1153
1154    #[test]
1155    fn string_literal_over_max_length_is_parse_error() {
1156        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH + 1);
1157        let err = lex_all(&format!("\"{content}\"")).unwrap_err();
1158        assert!(
1159            err.message().contains("maximum length"),
1160            "expected length error, got: {err}"
1161        );
1162        assert!(err.location().is_some(), "parse error must carry a source");
1163    }
1164
1165    #[test]
1166    fn number_literal_with_separators_lexes() {
1167        let tokens = lex_all("9_999_999_999_999_999_999_999_999_999").unwrap();
1168        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1169    }
1170
1171    #[test]
1172    fn lex_spec_declaration() {
1173        let kinds = lex_kinds("spec person").unwrap();
1174        assert_eq!(
1175            kinds,
1176            vec![TokenKind::Spec, TokenKind::Identifier, TokenKind::Eof]
1177        );
1178    }
1179
1180    #[test]
1181    fn lex_data_definition() {
1182        let kinds = lex_kinds("data age: 25").unwrap();
1183        assert_eq!(
1184            kinds,
1185            vec![
1186                TokenKind::Data,
1187                TokenKind::Identifier,
1188                TokenKind::Colon,
1189                TokenKind::NumberLit,
1190                TokenKind::Eof,
1191            ]
1192        );
1193    }
1194
1195    #[test]
1196    fn lex_rule_with_comparison() {
1197        let kinds = lex_kinds("rule is_adult: age >= 18").unwrap();
1198        assert_eq!(
1199            kinds,
1200            vec![
1201                TokenKind::Rule,
1202                TokenKind::Identifier,
1203                TokenKind::Colon,
1204                TokenKind::Identifier,
1205                TokenKind::Gte,
1206                TokenKind::NumberLit,
1207                TokenKind::Eof,
1208            ]
1209        );
1210    }
1211
1212    #[test]
1213    fn lex_string_literal() {
1214        let tokens = lex_all(r#""hello world""#).unwrap();
1215        assert_eq!(tokens[0].kind, TokenKind::StringLit);
1216        assert_eq!(tokens[0].text.of(r#""hello world""#), "hello world");
1217    }
1218
1219    #[test]
1220    fn lex_unterminated_string() {
1221        let result = lex_all(r#""hello"#);
1222        assert!(result.is_err());
1223    }
1224
1225    #[test]
1226    fn lex_number_with_decimal() {
1227        let tokens = lex_all("3.14").unwrap();
1228        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1229        assert_eq!(tokens[0].text.of("3.14"), "3.14");
1230    }
1231
1232    #[test]
1233    fn lex_number_with_underscores() {
1234        let tokens = lex_all("1_000_000").unwrap();
1235        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1236        assert_eq!(tokens[0].text.of("1_000_000"), "1_000_000");
1237    }
1238
1239    #[test]
1240    fn lex_scientific_notation() {
1241        let tokens = lex_all("1.5e+10").unwrap();
1242        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1243        assert_eq!(tokens[0].text.of("1.5e+10"), "1.5e+10");
1244    }
1245
1246    #[test]
1247    fn lex_all_operators() {
1248        let kinds = lex_kinds("+ - * / % ^ > < >= <= -> %%").unwrap();
1249        assert_eq!(
1250            &kinds[..12],
1251            &[
1252                TokenKind::Plus,
1253                TokenKind::Minus,
1254                TokenKind::Star,
1255                TokenKind::Slash,
1256                TokenKind::Percent,
1257                TokenKind::Caret,
1258                TokenKind::Gt,
1259                TokenKind::Lt,
1260                TokenKind::Gte,
1261                TokenKind::Lte,
1262                TokenKind::Arrow,
1263                TokenKind::PercentPercent,
1264            ]
1265        );
1266    }
1267
1268    #[test]
1269    fn lex_keywords() {
1270        let kinds =
1271            lex_kinds("spec data rule unless then not and or in as uses meta veto now").unwrap();
1272        assert_eq!(
1273            &kinds[..14],
1274            &[
1275                TokenKind::Spec,
1276                TokenKind::Data,
1277                TokenKind::Rule,
1278                TokenKind::Unless,
1279                TokenKind::Then,
1280                TokenKind::Not,
1281                TokenKind::And,
1282                TokenKind::Or,
1283                TokenKind::In,
1284                TokenKind::As,
1285                TokenKind::Uses,
1286                TokenKind::Meta,
1287                TokenKind::Veto,
1288                TokenKind::Now,
1289            ]
1290        );
1291    }
1292
1293    #[test]
1294    fn lex_boolean_keywords() {
1295        let kinds = lex_kinds("true false yes no").unwrap();
1296        assert_eq!(
1297            &kinds[..4],
1298            &[
1299                TokenKind::True,
1300                TokenKind::False,
1301                TokenKind::Yes,
1302                TokenKind::No,
1303            ]
1304        );
1305    }
1306
1307    #[test]
1308    fn lex_duration_keywords() {
1309        let kinds = lex_kinds("year month week day hour minute second").unwrap();
1310        assert_eq!(
1311            &kinds[..7],
1312            &[
1313                TokenKind::Identifier,
1314                TokenKind::Identifier,
1315                TokenKind::Identifier,
1316                TokenKind::Identifier,
1317                TokenKind::Identifier,
1318                TokenKind::Identifier,
1319                TokenKind::Identifier,
1320            ]
1321        );
1322    }
1323
1324    #[test]
1325    fn lex_commentary() {
1326        let tokens = lex_all(r#""""hello world""""#).unwrap();
1327        assert_eq!(tokens[0].kind, TokenKind::Commentary);
1328        assert_eq!(tokens[0].text.of(r#""""hello world""""#), "hello world");
1329    }
1330
1331    #[test]
1332    fn lex_registry_logical_name() {
1333        let tokens = lex_all("@iso/countries").unwrap();
1334        assert_eq!(tokens[0].kind, TokenKind::LogicalName);
1335        assert_eq!(tokens[0].text.of("@iso/countries"), "@iso/countries");
1336        assert_eq!(tokens[1].kind, TokenKind::Eof);
1337    }
1338
1339    #[test]
1340    fn lex_bare_at_is_error() {
1341        assert!(lex_all("@").is_err());
1342    }
1343
1344    #[test]
1345    fn next_logical_name_absorbs_slashes() {
1346        let mut lexer = Lexer::new(
1347            "contracts/employment/jack",
1348            &crate::parsing::source::SourceType::Volatile,
1349        );
1350        let tok = lexer.next_logical_name().unwrap();
1351        assert_eq!(tok.kind, TokenKind::LogicalName);
1352        assert_eq!(
1353            tok.text.of("contracts/employment/jack"),
1354            "contracts/employment/jack"
1355        );
1356    }
1357
1358    #[test]
1359    fn next_logical_name_after_peeked_identifier_rewinds() {
1360        let mut lexer = Lexer::new(
1361            "finance/units",
1362            &crate::parsing::source::SourceType::Volatile,
1363        );
1364        assert_eq!(lexer.peek().unwrap().kind, TokenKind::Identifier);
1365        let tok = lexer.next_logical_name().unwrap();
1366        assert_eq!(tok.kind, TokenKind::LogicalName);
1367        assert_eq!(tok.text.of("finance/units"), "finance/units");
1368    }
1369
1370    #[test]
1371    fn division_and_units_keep_slash_token() {
1372        let kinds = lex_kinds("a / b").unwrap();
1373        assert_eq!(
1374            &kinds[..3],
1375            &[
1376                TokenKind::Identifier,
1377                TokenKind::Slash,
1378                TokenKind::Identifier
1379            ]
1380        );
1381        let kinds = lex_kinds("meter/second").unwrap();
1382        assert_eq!(
1383            &kinds[..3],
1384            &[
1385                TokenKind::Identifier,
1386                TokenKind::Slash,
1387                TokenKind::Identifier
1388            ]
1389        );
1390    }
1391
1392    #[test]
1393    fn lex_parentheses() {
1394        let kinds = lex_kinds("(x + 1)").unwrap();
1395        assert_eq!(
1396            &kinds[..5],
1397            &[
1398                TokenKind::LParen,
1399                TokenKind::Identifier,
1400                TokenKind::Plus,
1401                TokenKind::NumberLit,
1402                TokenKind::RParen,
1403            ]
1404        );
1405    }
1406
1407    #[test]
1408    fn lex_dot_for_references() {
1409        let kinds = lex_kinds("employee.salary").unwrap();
1410        assert_eq!(
1411            &kinds[..3],
1412            &[TokenKind::Identifier, TokenKind::Dot, TokenKind::Identifier]
1413        );
1414    }
1415
1416    #[test]
1417    fn lex_spec_keyword_then_slashed_name_via_next_logical_name() {
1418        let mut lexer = Lexer::new(
1419            "spec contracts/employment/jack",
1420            &crate::parsing::source::SourceType::Volatile,
1421        );
1422        assert_eq!(lexer.next_token().unwrap().kind, TokenKind::Spec);
1423        let name = lexer.next_logical_name().unwrap();
1424        assert_eq!(name.kind, TokenKind::LogicalName);
1425        assert_eq!(
1426            name.text.of("spec contracts/employment/jack"),
1427            "contracts/employment/jack"
1428        );
1429    }
1430
1431    #[test]
1432    fn lex_number_not_followed_by_e_identifier() {
1433        // "42 eur" should be number then identifier, not scientific notation
1434        let tokens = lex_all("42 eur").unwrap();
1435        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1436        assert_eq!(tokens[0].text.of("42 eur"), "42");
1437        assert_eq!(tokens[1].kind, TokenKind::Identifier);
1438        assert_eq!(tokens[1].text.of("42 eur"), "eur");
1439    }
1440
1441    #[test]
1442    fn lex_unknown_character() {
1443        let result = lex_all("§");
1444        assert!(result.is_err());
1445    }
1446
1447    #[test]
1448    fn lex_peek_does_not_consume() {
1449        let mut lexer = Lexer::new("spec test", &crate::parsing::source::SourceType::Volatile);
1450        let peeked_kind = lexer.peek().unwrap().kind;
1451        assert_eq!(peeked_kind, TokenKind::Spec);
1452        let next = lexer.next_token().unwrap();
1453        assert_eq!(next.kind, TokenKind::Spec);
1454    }
1455
1456    #[test]
1457    fn lex_span_byte_offsets() {
1458        let tokens = lex_all("spec test").unwrap();
1459        assert_eq!(tokens[0].span.start, 0);
1460        assert_eq!(tokens[0].span.end, 4);
1461        assert_eq!(tokens[0].span.line, 1);
1462        assert_eq!(tokens[0].span.col, 1);
1463
1464        assert_eq!(tokens[1].span.start, 5);
1465        assert_eq!(tokens[1].span.end, 9);
1466        assert_eq!(tokens[1].span.line, 1);
1467        assert_eq!(tokens[1].span.col, 6);
1468    }
1469
1470    #[test]
1471    fn lex_multiline_span_tracking() {
1472        let tokens = lex_all("spec test\ndata x: 1").unwrap();
1473        // "data" should be on line 2
1474        let data_token = &tokens[2]; // spec, test, data
1475        assert_eq!(data_token.kind, TokenKind::Data);
1476        assert_eq!(data_token.span.line, 2);
1477        assert_eq!(data_token.span.col, 1);
1478    }
1479
1480    #[test]
1481    fn lex_case_insensitive_keywords() {
1482        // Lemma keywords are case-insensitive
1483        let kinds = lex_kinds("SPEC Data RULE").unwrap();
1484        assert_eq!(kinds[0], TokenKind::Spec);
1485        assert_eq!(kinds[1], TokenKind::Data);
1486        assert_eq!(kinds[2], TokenKind::Rule);
1487    }
1488
1489    #[test]
1490    fn lex_math_function_keywords() {
1491        let kinds =
1492            lex_kinds("sqrt sin cos tan asin acos atan log exp abs floor ceil round").unwrap();
1493        assert_eq!(
1494            &kinds[..13],
1495            &[
1496                TokenKind::Sqrt,
1497                TokenKind::Sin,
1498                TokenKind::Cos,
1499                TokenKind::Tan,
1500                TokenKind::Asin,
1501                TokenKind::Acos,
1502                TokenKind::Atan,
1503                TokenKind::Log,
1504                TokenKind::Exp,
1505                TokenKind::Abs,
1506                TokenKind::Floor,
1507                TokenKind::Ceil,
1508                TokenKind::Round,
1509            ]
1510        );
1511    }
1512
1513    #[test]
1514    fn lex_range_bound_keywords() {
1515        let kinds = lex_kinds("lower upper LOWER").unwrap();
1516        assert_eq!(
1517            &kinds[..3],
1518            &[TokenKind::Lower, TokenKind::Upper, TokenKind::Lower,]
1519        );
1520    }
1521
1522    #[test]
1523    fn lex_is_keyword() {
1524        let kinds = lex_kinds("status is \"active\"").unwrap();
1525        assert_eq!(kinds[0], TokenKind::Identifier);
1526        assert_eq!(kinds[1], TokenKind::Is);
1527        assert_eq!(kinds[2], TokenKind::StringLit);
1528    }
1529
1530    #[test]
1531    fn lex_percent_not_followed_by_digit() {
1532        // "50%" should be number then percent
1533        let kinds = lex_kinds("50%").unwrap();
1534        assert_eq!(kinds[0], TokenKind::NumberLit);
1535        assert_eq!(kinds[1], TokenKind::Percent);
1536    }
1537
1538    #[test]
1539    fn lex_number_with_commas() {
1540        let kinds = lex_kinds("1,000,000").unwrap();
1541        assert_eq!(
1542            kinds,
1543            vec![
1544                TokenKind::NumberLit,
1545                TokenKind::Comma,
1546                TokenKind::NumberLit,
1547                TokenKind::Comma,
1548                TokenKind::NumberLit,
1549                TokenKind::Eof,
1550            ]
1551        );
1552    }
1553
1554    #[test]
1555    fn lex_arrow_chain() {
1556        let kinds = lex_kinds("-> unit eur: 1.00 -> decimals 2").unwrap();
1557        assert_eq!(kinds[0], TokenKind::Arrow);
1558        assert_eq!(kinds[1], TokenKind::Identifier);
1559        assert_eq!(kinds[2], TokenKind::Identifier);
1560        assert_eq!(kinds[3], TokenKind::Colon);
1561        assert_eq!(kinds[4], TokenKind::NumberLit);
1562        assert_eq!(kinds[5], TokenKind::Arrow);
1563    }
1564}
lemma/parsing/lexer.rs

lemma/parsing/
lexer.rs
