lemma/parsing/
lexer.rs

1use crate::error::Error;
2use crate::parsing::ast::{BooleanValue, PrimitiveKind, Span};
3use crate::parsing::source::Source;
4use std::sync::Arc;
5
6#[derive(Debug, Clone, PartialEq, Eq)]
7pub enum TokenKind {
8    // Keywords
9    Spec,
10    Repo,
11    Data,
12    Rule,
13    Unless,
14    Then,
15    Not,
16    And,
17    In,
18    As,
19    Uses,
20    With,
21    Meta,
22    Veto,
23    Now,
24    Past,
25    Future,
26
27    // Boolean keywords
28    True,
29    False,
30    Yes,
31    No,
32
33    // Type keywords
34    MeasureKw,
35    NumberKw,
36    TextKw,
37    DateKw,
38    TimeKw,
39    BooleanKw,
40    RatioKw,
41
42    // Math function keywords
43    Sqrt,
44    Sin,
45    Cos,
46    Tan,
47    Asin,
48    Acos,
49    Atan,
50    Log,
51    Exp,
52    Abs,
53    Floor,
54    Ceil,
55    Round,
56
57    // List keywords
58    List,
59    Key,
60    Find,
61    First,
62    Last,
63    Sum,
64    Avg,
65    Product,
66    Count,
67    Min,
68    Max,
69    Any,
70    All,
71    Where,
72    Sort,
73    Top,
74    Asc,
75    Desc,
76
77    // Range endpoint operators
78    Lower,
79    Upper,
80
81    Permille,
82
83    // Comparison keyword operators
84    Is,
85
86    // Operators
87    Plus,
88    Minus,
89    Star,
90    Slash,
91    Comma,
92    Percent,
93    PercentPercent,
94    Mod,
95    Caret,
96    Gt,
97    Lt,
98    Gte,
99    Lte,
100
101    // Punctuation
102    Colon,
103    Arrow,
104    Ellipsis,
105    Dot,
106    LParen,
107    RParen,
108
109    // Literals
110    NumberLit,
111    StringLit,
112
113    // Commentary (raw text between """ delimiters)
114    Commentary,
115
116    // Identifiers
117    Identifier,
118
119    /// Opaque spec / repo / registry name: optional `@` + `[A-Za-z0-9_./-]+`.
120    LogicalName,
121
122    // End of file
123    Eof,
124}
125
126impl std::fmt::Display for TokenKind {
127    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
128        match self {
129            TokenKind::Spec => write!(f, "'spec'"),
130            TokenKind::Repo => write!(f, "'repo'"),
131            TokenKind::Data => write!(f, "'data'"),
132            TokenKind::Rule => write!(f, "'rule'"),
133            TokenKind::Unless => write!(f, "'unless'"),
134            TokenKind::Then => write!(f, "'then'"),
135            TokenKind::Not => write!(f, "'not'"),
136            TokenKind::And => write!(f, "'and'"),
137            TokenKind::In => write!(f, "'in'"),
138            TokenKind::As => write!(f, "'as'"),
139            TokenKind::Uses => write!(f, "'uses'"),
140            TokenKind::With => write!(f, "'with'"),
141            TokenKind::Meta => write!(f, "'meta'"),
142            TokenKind::Veto => write!(f, "'veto'"),
143            TokenKind::Now => write!(f, "'now'"),
144            TokenKind::Past => write!(f, "'past'"),
145            TokenKind::Future => write!(f, "'future'"),
146            TokenKind::True => write!(f, "'true'"),
147            TokenKind::False => write!(f, "'false'"),
148            TokenKind::Yes => write!(f, "'yes'"),
149            TokenKind::No => write!(f, "'no'"),
150            TokenKind::MeasureKw => write!(f, "'measure'"),
151            TokenKind::NumberKw => write!(f, "'number'"),
152            TokenKind::TextKw => write!(f, "'text'"),
153            TokenKind::DateKw => write!(f, "'date'"),
154            TokenKind::TimeKw => write!(f, "'time'"),
155            TokenKind::BooleanKw => write!(f, "'boolean'"),
156            TokenKind::RatioKw => write!(f, "'ratio'"),
157            TokenKind::Sqrt => write!(f, "'sqrt'"),
158            TokenKind::Sin => write!(f, "'sin'"),
159            TokenKind::Cos => write!(f, "'cos'"),
160            TokenKind::Tan => write!(f, "'tan'"),
161            TokenKind::Asin => write!(f, "'asin'"),
162            TokenKind::Acos => write!(f, "'acos'"),
163            TokenKind::Atan => write!(f, "'atan'"),
164            TokenKind::Log => write!(f, "'log'"),
165            TokenKind::Exp => write!(f, "'exp'"),
166            TokenKind::Abs => write!(f, "'abs'"),
167            TokenKind::Floor => write!(f, "'floor'"),
168            TokenKind::Ceil => write!(f, "'ceil'"),
169            TokenKind::Round => write!(f, "'round'"),
170            TokenKind::List => write!(f, "'list'"),
171            TokenKind::Key => write!(f, "'key'"),
172            TokenKind::Find => write!(f, "'find'"),
173            TokenKind::First => write!(f, "'first'"),
174            TokenKind::Last => write!(f, "'last'"),
175            TokenKind::Sum => write!(f, "'sum'"),
176            TokenKind::Avg => write!(f, "'avg'"),
177            TokenKind::Product => write!(f, "'product'"),
178            TokenKind::Count => write!(f, "'count'"),
179            TokenKind::Min => write!(f, "'min'"),
180            TokenKind::Max => write!(f, "'max'"),
181            TokenKind::Any => write!(f, "'any'"),
182            TokenKind::All => write!(f, "'all'"),
183            TokenKind::Where => write!(f, "'where'"),
184            TokenKind::Sort => write!(f, "'sort'"),
185            TokenKind::Top => write!(f, "'top'"),
186            TokenKind::Asc => write!(f, "'asc'"),
187            TokenKind::Desc => write!(f, "'desc'"),
188            TokenKind::Lower => write!(f, "'lower'"),
189            TokenKind::Upper => write!(f, "'upper'"),
190            TokenKind::Permille => write!(f, "'permille'"),
191            TokenKind::Is => write!(f, "'is'"),
192            TokenKind::Plus => write!(f, "'+'"),
193            TokenKind::Minus => write!(f, "'-'"),
194            TokenKind::Star => write!(f, "'*'"),
195            TokenKind::Slash => write!(f, "'/'"),
196            TokenKind::Comma => write!(f, "','"),
197            TokenKind::Percent => write!(f, "'%'"),
198            TokenKind::PercentPercent => write!(f, "'%%'"),
199            TokenKind::Mod => write!(f, "'mod'"),
200            TokenKind::Caret => write!(f, "'^'"),
201            TokenKind::Gt => write!(f, "'>'"),
202            TokenKind::Lt => write!(f, "'<'"),
203            TokenKind::Gte => write!(f, "'>='"),
204            TokenKind::Lte => write!(f, "'<='"),
205            TokenKind::Colon => write!(f, "':'"),
206            TokenKind::Arrow => write!(f, "'->'"),
207            TokenKind::Ellipsis => write!(f, "'...'"),
208            TokenKind::Dot => write!(f, "'.'"),
209            TokenKind::LParen => write!(f, "'('"),
210            TokenKind::RParen => write!(f, "')'"),
211            TokenKind::NumberLit => write!(f, "a number"),
212            TokenKind::StringLit => write!(f, "a string"),
213            TokenKind::Commentary => write!(f, "commentary block"),
214            TokenKind::Identifier => write!(f, "an identifier"),
215            TokenKind::LogicalName => write!(f, "a name"),
216            TokenKind::Eof => write!(f, "end of file"),
217        }
218    }
219}
220
221#[derive(Debug, Clone)]
222pub struct Token {
223    pub kind: TokenKind,
224    pub span: Span,
225    pub text: String,
226}
227
228impl Token {
229    pub fn eof(offset: usize, line: usize, col: usize) -> Self {
230        Token {
231            kind: TokenKind::Eof,
232            span: Span {
233                start: offset,
234                end: offset,
235                line,
236                col,
237            },
238            text: String::new(),
239        }
240    }
241}
242
243#[derive(Clone)]
244pub struct LexerCheckpoint {
245    pos: usize,
246    line: usize,
247    col: usize,
248    byte_offset: usize,
249    peeked: Option<Token>,
250    peeked2: Option<Token>,
251}
252
253// todo: find out why derive Clone is necessary
254#[derive(Clone)]
255pub struct Lexer {
256    source: Vec<char>,
257    pos: usize,
258    line: usize,
259    col: usize,
260    byte_offset: usize,
261    source_type: crate::parsing::source::SourceType,
262    source_text: Arc<str>,
263    peeked: Option<Token>,
264    peeked2: Option<Token>,
265}
266
267impl Lexer {
268    pub fn new(input: &str, source_type: &crate::parsing::source::SourceType) -> Self {
269        let source_text: Arc<str> = Arc::from(input);
270        Lexer {
271            source: input.chars().collect(),
272            pos: 0,
273            line: 1,
274            col: 1,
275            byte_offset: 0,
276            source_type: source_type.clone(),
277            source_text,
278            peeked: None,
279            peeked2: None,
280        }
281    }
282
283    pub fn peek(&mut self) -> Result<&Token, Error> {
284        if self.peeked.is_none() {
285            let token = self.lex_token()?;
286            self.peeked = Some(token);
287        }
288        Ok(self.peeked.as_ref().expect("just assigned"))
289    }
290
291    pub fn peek_second(&mut self) -> Result<&Token, Error> {
292        self.peek()?;
293        if self.peeked2.is_none() {
294            let token = self.lex_token()?;
295            self.peeked2 = Some(token);
296        }
297        Ok(self.peeked2.as_ref().expect("just assigned"))
298    }
299
300    pub fn next_token(&mut self) -> Result<Token, Error> {
301        if let Some(token) = self.peeked.take() {
302            self.peeked = self.peeked2.take();
303            return Ok(token);
304        }
305        self.lex_token()
306    }
307
308    /// Opaque spec / repo / registry name for name positions only.
309    ///
310    /// If a normal peek already materialized a prefix token, rewind and rescan
311    /// so connectors (`/`, `.`, `-`) are absorbed into one [`TokenKind::LogicalName`].
312    pub fn next_logical_name(&mut self) -> Result<Token, Error> {
313        if let Some(tok) = &self.peeked {
314            if tok.kind == TokenKind::LogicalName {
315                return self.next_token();
316            }
317            let start_byte = tok.span.start;
318            let start_line = tok.span.line;
319            let start_col = tok.span.col;
320            self.peeked = None;
321            self.peeked2 = None;
322            self.rewind_to(start_byte, start_line, start_col);
323            return self.scan_integral_logical_name(start_byte, start_line, start_col);
324        }
325        self.skip_whitespace();
326        let start_byte = self.byte_offset;
327        let start_line = self.line;
328        let start_col = self.col;
329        self.scan_integral_logical_name(start_byte, start_line, start_col)
330    }
331
332    fn rewind_to(&mut self, byte_offset: usize, line: usize, col: usize) {
333        let mut pos = 0;
334        let mut bytes = 0;
335        while bytes < byte_offset && pos < self.source.len() {
336            bytes += self.source[pos].len_utf8();
337            pos += 1;
338        }
339        self.pos = pos;
340        self.byte_offset = byte_offset;
341        self.line = line;
342        self.col = col;
343    }
344
345    /// Saved lexer position for speculative parsing.
346    pub fn checkpoint(&self) -> LexerCheckpoint {
347        LexerCheckpoint {
348            pos: self.pos,
349            line: self.line,
350            col: self.col,
351            byte_offset: self.byte_offset,
352            peeked: self.peeked.clone(),
353            peeked2: self.peeked2.clone(),
354        }
355    }
356
357    pub fn restore(&mut self, checkpoint: LexerCheckpoint) {
358        self.pos = checkpoint.pos;
359        self.line = checkpoint.line;
360        self.col = checkpoint.col;
361        self.byte_offset = checkpoint.byte_offset;
362        self.peeked = checkpoint.peeked;
363        self.peeked2 = checkpoint.peeked2;
364    }
365
366    fn current_char(&self) -> Option<char> {
367        self.source.get(self.pos).copied()
368    }
369
370    fn peek_char(&self) -> Option<char> {
371        self.source.get(self.pos + 1).copied()
372    }
373
374    fn peek_char_at(&self, offset: usize) -> Option<char> {
375        self.source.get(self.pos + offset).copied()
376    }
377
378    fn advance(&mut self) {
379        if let Some(ch) = self.current_char() {
380            self.byte_offset += ch.len_utf8();
381            if ch == '\n' {
382                self.line += 1;
383                self.col = 1;
384            } else {
385                self.col += 1;
386            }
387            self.pos += 1;
388        }
389    }
390
391    fn skip_whitespace(&mut self) {
392        while let Some(ch) = self.current_char() {
393            if ch.is_whitespace() {
394                self.advance();
395            } else {
396                break;
397            }
398        }
399    }
400
401    fn make_span(&self, start_byte: usize, start_line: usize, start_col: usize) -> Span {
402        Span {
403            start: start_byte,
404            end: self.byte_offset,
405            line: start_line,
406            col: start_col,
407        }
408    }
409
410    fn make_error(&self, message: impl Into<String>, span: Span) -> Error {
411        Error::parsing(
412            message,
413            Source::new(self.source_type.clone(), span),
414            None::<String>,
415        )
416    }
417
418    fn lex_token(&mut self) -> Result<Token, Error> {
419        self.skip_whitespace();
420
421        let start_byte = self.byte_offset;
422        let start_line = self.line;
423        let start_col = self.col;
424
425        let Some(ch) = self.current_char() else {
426            return Ok(Token::eof(start_byte, start_line, start_col));
427        };
428
429        // Triple-quote commentary
430        if ch == '"' && self.peek_char() == Some('"') && self.peek_char_at(2) == Some('"') {
431            return self.scan_triple_quote(start_byte, start_line, start_col);
432        }
433
434        // String literal
435        if ch == '"' {
436            return self.scan_string(start_byte, start_line, start_col);
437        }
438
439        // Number literal (sign handled by parser, not lexer)
440        if ch.is_ascii_digit() {
441            return self.scan_number(start_byte, start_line, start_col);
442        }
443
444        // Two-character operators (check before single-char)
445        if let Some(token) = self.try_two_char_operator(start_byte, start_line, start_col) {
446            return Ok(token);
447        }
448
449        // Three-character ellipsis
450        if ch == '.' && self.peek_char() == Some('.') && self.peek_char_at(2) == Some('.') {
451            self.advance();
452            self.advance();
453            self.advance();
454            let span = self.make_span(start_byte, start_line, start_col);
455            return Ok(Token {
456                kind: TokenKind::Ellipsis,
457                span,
458                text: "...".to_string(),
459            });
460        }
461
462        // Single-character operators/punctuation
463        if let Some(kind) = self.single_char_token(ch) {
464            self.advance();
465            let span = self.make_span(start_byte, start_line, start_col);
466            let text = ch.to_string();
467            return Ok(Token { kind, span, text });
468        }
469
470        // Identifier or keyword
471        if ch.is_ascii_alphabetic() || ch == '_' {
472            return Ok(self.scan_identifier(start_byte, start_line, start_col));
473        }
474
475        // Registry qualifier: `@` + opaque name characters
476        if ch == '@' {
477            return self.scan_integral_logical_name(start_byte, start_line, start_col);
478        }
479
480        // Unknown character
481        self.advance();
482        let span = self.make_span(start_byte, start_line, start_col);
483        Err(self.make_error(format!("Unexpected character '{}'", ch), span))
484    }
485
486    fn is_logical_name_char(ch: char) -> bool {
487        ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '/' | '-')
488    }
489
490    fn scan_integral_logical_name(
491        &mut self,
492        start_byte: usize,
493        start_line: usize,
494        start_col: usize,
495    ) -> Result<Token, Error> {
496        let mut text = String::new();
497        let had_at = self.current_char() == Some('@');
498        if had_at {
499            text.push('@');
500            self.advance();
501        }
502        let Some(first) = self.current_char() else {
503            let span = self.make_span(start_byte, start_line, start_col);
504            return Err(self.make_error(
505                if had_at {
506                    "Expected a name after '@'"
507                } else {
508                    "Expected a name"
509                },
510                span,
511            ));
512        };
513        if !Self::is_logical_name_char(first) {
514            let span = self.make_span(start_byte, start_line, start_col);
515            return Err(self.make_error(
516                if had_at {
517                    format!("Expected a name after '@', found '{}'", first)
518                } else {
519                    format!("Expected a name, found '{}'", first)
520                },
521                span,
522            ));
523        }
524        while let Some(ch) = self.current_char() {
525            if Self::is_logical_name_char(ch) {
526                text.push(ch);
527                self.advance();
528            } else {
529                break;
530            }
531        }
532        let span = self.make_span(start_byte, start_line, start_col);
533        Ok(Token {
534            kind: TokenKind::LogicalName,
535            span,
536            text,
537        })
538    }
539
540    fn scan_triple_quote(
541        &mut self,
542        start_byte: usize,
543        start_line: usize,
544        start_col: usize,
545    ) -> Result<Token, Error> {
546        self.advance(); // "
547        self.advance(); // "
548        self.advance(); // "
549
550        let content_start = self.byte_offset;
551        loop {
552            match self.current_char() {
553                None => {
554                    let span = self.make_span(start_byte, start_line, start_col);
555                    return Err(self.make_error(
556                        "Unterminated commentary block: expected closing \"\"\"",
557                        span,
558                    ));
559                }
560                Some('"')
561                    if self.source.get(self.pos + 1) == Some(&'"')
562                        && self.source.get(self.pos + 2) == Some(&'"') =>
563                {
564                    let content_end = self.byte_offset;
565                    self.advance(); // "
566                    self.advance(); // "
567                    self.advance(); // "
568                    let raw: String = self.source_text[content_start..content_end].to_string();
569                    let span = self.make_span(start_byte, start_line, start_col);
570                    return Ok(Token {
571                        kind: TokenKind::Commentary,
572                        span,
573                        text: raw,
574                    });
575                }
576                Some(_) => {
577                    self.advance();
578                }
579            }
580        }
581    }
582
583    fn scan_string(
584        &mut self,
585        start_byte: usize,
586        start_line: usize,
587        start_col: usize,
588    ) -> Result<Token, Error> {
589        self.advance(); // consume opening "
590        let mut content = String::new();
591        loop {
592            match self.current_char() {
593                None => {
594                    let span = self.make_span(start_byte, start_line, start_col);
595                    return Err(self.make_error("String starting here was never closed", span));
596                }
597                Some('"') => {
598                    self.advance(); // consume closing "
599                    break;
600                }
601                Some(ch) => {
602                    content.push(ch);
603                    self.advance();
604                }
605            }
606        }
607        let span = self.make_span(start_byte, start_line, start_col);
608        if content.len() > crate::limits::MAX_TEXT_VALUE_LENGTH {
609            return Err(self.make_error(
610                format!(
611                    "Text literal exceeds maximum length of {} characters (found {})",
612                    crate::limits::MAX_TEXT_VALUE_LENGTH,
613                    content.len()
614                ),
615                span,
616            ));
617        }
618        // Token text is the unquoted content. The span still covers the quotes so
619        // diagnostics point at the full literal; the parser must not strip again.
620        Ok(Token {
621            kind: TokenKind::StringLit,
622            span,
623            text: content,
624        })
625    }
626
627    fn scan_number(
628        &mut self,
629        start_byte: usize,
630        start_line: usize,
631        start_col: usize,
632    ) -> Result<Token, Error> {
633        let mut text = String::new();
634
635        // Integer part: digits with optional `_` separators
636        while let Some(ch) = self.current_char() {
637            if ch.is_ascii_digit() || ch == '_' {
638                text.push(ch);
639                self.advance();
640            } else {
641                break;
642            }
643        }
644
645        // Decimal part
646        if self.current_char() == Some('.') {
647            // Check if next char after dot is a digit (not a method call or dotted reference)
648            if let Some(next) = self.peek_char() {
649                if next.is_ascii_digit() {
650                    text.push('.');
651                    self.advance(); // consume .
652                    while let Some(ch) = self.current_char() {
653                        if ch.is_ascii_digit() {
654                            text.push(ch);
655                            self.advance();
656                        } else {
657                            break;
658                        }
659                    }
660                }
661            }
662        }
663
664        // Scientific notation: e or E followed by optional +/- and digits
665        if let Some(ch) = self.current_char() {
666            if ch == 'e' || ch == 'E' {
667                let mut sci_text = String::new();
668                sci_text.push(ch);
669                let save_pos = self.pos;
670                let save_byte = self.byte_offset;
671                let save_line = self.line;
672                let save_col = self.col;
673                self.advance(); // consume e/E
674
675                if let Some(sign) = self.current_char() {
676                    if sign == '+' || sign == '-' {
677                        sci_text.push(sign);
678                        self.advance();
679                    }
680                }
681
682                if let Some(d) = self.current_char() {
683                    if d.is_ascii_digit() {
684                        while let Some(ch) = self.current_char() {
685                            if ch.is_ascii_digit() {
686                                sci_text.push(ch);
687                                self.advance();
688                            } else {
689                                break;
690                            }
691                        }
692                        text.push_str(&sci_text);
693                    } else {
694                        // Not actually scientific notation, backtrack
695                        self.pos = save_pos;
696                        self.byte_offset = save_byte;
697                        self.line = save_line;
698                        self.col = save_col;
699                    }
700                } else {
701                    self.pos = save_pos;
702                    self.byte_offset = save_byte;
703                    self.line = save_line;
704                    self.col = save_col;
705                }
706            }
707        }
708
709        let span = self.make_span(start_byte, start_line, start_col);
710        Ok(Token {
711            kind: TokenKind::NumberLit,
712            span,
713            text,
714        })
715    }
716
717    fn try_two_char_operator(
718        &mut self,
719        start_byte: usize,
720        start_line: usize,
721        start_col: usize,
722    ) -> Option<Token> {
723        let ch = self.current_char()?;
724        let next = self.peek_char();
725
726        let kind = match (ch, next) {
727            ('-', Some('>')) => TokenKind::Arrow,
728            ('>', Some('=')) => TokenKind::Gte,
729            ('<', Some('=')) => TokenKind::Lte,
730            ('%', Some('%')) => {
731                // Check that it's not followed by a digit (invalid permille like 10%%5)
732                TokenKind::PercentPercent
733            }
734            _ => return None,
735        };
736
737        self.advance();
738        self.advance();
739        let span = self.make_span(start_byte, start_line, start_col);
740        let text: String = self.source_text[span.start..span.end].to_string();
741        Some(Token { kind, span, text })
742    }
743
744    fn single_char_token(&self, ch: char) -> Option<TokenKind> {
745        match ch {
746            '+' => Some(TokenKind::Plus),
747            '*' => Some(TokenKind::Star),
748            '/' => Some(TokenKind::Slash),
749            ',' => Some(TokenKind::Comma),
750            '^' => Some(TokenKind::Caret),
751            ':' => Some(TokenKind::Colon),
752            '.' => Some(TokenKind::Dot),
753            '(' => Some(TokenKind::LParen),
754            ')' => Some(TokenKind::RParen),
755            '>' => Some(TokenKind::Gt),
756            '<' => Some(TokenKind::Lt),
757            '%' => Some(TokenKind::Percent),
758            '-' => Some(TokenKind::Minus),
759            _ => None,
760        }
761    }
762
763    fn scan_identifier(&mut self, start_byte: usize, start_line: usize, start_col: usize) -> Token {
764        let mut text = String::new();
765        while let Some(ch) = self.current_char() {
766            if ch.is_ascii_alphanumeric() || ch == '_' {
767                text.push(ch);
768                self.advance();
769            } else {
770                break;
771            }
772        }
773
774        let kind = keyword_from_identifier(&text);
775        let span = self.make_span(start_byte, start_line, start_col);
776        Token { kind, span, text }
777    }
778}
779
780macro_rules! lemma_keywords {
781    ($(($spell:literal, $kind:ident)),+ $(,)?) => {
782        /// Spellings shared by [`keyword_from_identifier`] and LSP completion.
783        pub const COMPLETION_KEYWORD_LABELS: &[&str] = &[$($spell),+];
784
785        fn keyword_from_identifier(text: &str) -> TokenKind {
786            match text {
787                $(s if s.eq_ignore_ascii_case($spell) => TokenKind::$kind,)+
788                _ => TokenKind::Identifier,
789            }
790        }
791    };
792}
793
794lemma_keywords! {
795    ("spec", Spec),
796    ("repo", Repo),
797    ("data", Data),
798    ("rule", Rule),
799    ("unless", Unless),
800    ("then", Then),
801    ("not", Not),
802    ("and", And),
803    ("in", In),
804    ("as", As),
805    ("uses", Uses),
806    ("with", With),
807    ("meta", Meta),
808    ("veto", Veto),
809    ("now", Now),
810    ("past", Past),
811    ("future", Future),
812    ("true", True),
813    ("false", False),
814    ("yes", Yes),
815    ("no", No),
816    ("measure", MeasureKw),
817    ("number", NumberKw),
818    ("text", TextKw),
819    ("date", DateKw),
820    ("time", TimeKw),
821    ("boolean", BooleanKw),
822    ("ratio", RatioKw),
823    ("sqrt", Sqrt),
824    ("sin", Sin),
825    ("cos", Cos),
826    ("tan", Tan),
827    ("asin", Asin),
828    ("acos", Acos),
829    ("atan", Atan),
830    ("log", Log),
831    ("exp", Exp),
832    ("abs", Abs),
833    ("floor", Floor),
834    ("ceil", Ceil),
835    ("round", Round),
836    ("list", List),
837    ("key", Key),
838    ("find", Find),
839    ("first", First),
840    ("last", Last),
841    ("sum", Sum),
842    ("avg", Avg),
843    ("product", Product),
844    ("count", Count),
845    ("min", Min),
846    ("max", Max),
847    ("any", Any),
848    ("all", All),
849    ("where", Where),
850    ("sort", Sort),
851    ("top", Top),
852    ("asc", Asc),
853    ("desc", Desc),
854    ("lower", Lower),
855    ("upper", Upper),
856    ("is", Is),
857    ("permille", Permille),
858    ("mod", Mod),
859}
860
861/// Structural keywords can never be used as identifiers (data/rule names).
862/// Type keywords (measure, number, text, date, time, boolean, ratio)
863/// are reserved and cannot be used as names.
864pub fn is_keyword(kind: &TokenKind) -> bool {
865    matches!(
866        kind,
867        TokenKind::Spec
868            | TokenKind::Repo
869            | TokenKind::Data
870            | TokenKind::Rule
871            | TokenKind::Unless
872            | TokenKind::Then
873            | TokenKind::Not
874            | TokenKind::And
875            | TokenKind::In
876            | TokenKind::As
877            | TokenKind::Uses
878            | TokenKind::With
879            | TokenKind::Meta
880            | TokenKind::Veto
881            | TokenKind::Now
882            | TokenKind::Is
883            | TokenKind::Past
884            | TokenKind::Future
885            | TokenKind::Permille
886            | TokenKind::Mod
887            | TokenKind::Sqrt
888            | TokenKind::Sin
889            | TokenKind::Cos
890            | TokenKind::Tan
891            | TokenKind::Asin
892            | TokenKind::Acos
893            | TokenKind::Atan
894            | TokenKind::Log
895            | TokenKind::Exp
896            | TokenKind::Abs
897            | TokenKind::Floor
898            | TokenKind::Ceil
899            | TokenKind::Round
900            | TokenKind::List
901            | TokenKind::Key
902            | TokenKind::Find
903            | TokenKind::First
904            | TokenKind::Last
905            | TokenKind::Sum
906            | TokenKind::Avg
907            | TokenKind::Product
908            | TokenKind::Count
909            | TokenKind::Min
910            | TokenKind::Max
911            | TokenKind::Any
912            | TokenKind::All
913            | TokenKind::Where
914            | TokenKind::Sort
915            | TokenKind::Top
916            | TokenKind::Asc
917            | TokenKind::Desc
918            | TokenKind::Lower
919            | TokenKind::Upper
920            | TokenKind::True
921            | TokenKind::False
922            | TokenKind::Yes
923            | TokenKind::No
924            | TokenKind::MeasureKw
925            | TokenKind::NumberKw
926            | TokenKind::TextKw
927            | TokenKind::DateKw
928            | TokenKind::TimeKw
929            | TokenKind::BooleanKw
930            | TokenKind::RatioKw
931    )
932}
933
934/// Map type keyword token to PrimitiveKind. Single source of truth for type keywords.
935#[must_use]
936pub fn token_kind_to_primitive(kind: &TokenKind) -> Option<PrimitiveKind> {
937    match kind {
938        TokenKind::BooleanKw => Some(PrimitiveKind::Boolean),
939        TokenKind::MeasureKw => Some(PrimitiveKind::Measure),
940        TokenKind::NumberKw => Some(PrimitiveKind::Number),
941        TokenKind::RatioKw => Some(PrimitiveKind::Ratio),
942        TokenKind::TextKw => Some(PrimitiveKind::Text),
943        TokenKind::DateKw => Some(PrimitiveKind::Date),
944        TokenKind::TimeKw => Some(PrimitiveKind::Time),
945        _ => None,
946    }
947}
948
949/// Returns true if the token kind represents a boolean literal keyword.
950pub fn is_boolean_keyword(kind: &TokenKind) -> bool {
951    matches!(
952        kind,
953        TokenKind::True | TokenKind::False | TokenKind::Yes | TokenKind::No
954    )
955}
956
957/// Maps a boolean-keyword token kind to BooleanValue. Call only when `is_boolean_keyword(kind)`.
958#[must_use]
959pub fn token_kind_to_boolean_value(kind: &TokenKind) -> BooleanValue {
960    match kind {
961        TokenKind::True => BooleanValue::True,
962        TokenKind::False => BooleanValue::False,
963        TokenKind::Yes => BooleanValue::Yes,
964        TokenKind::No => BooleanValue::No,
965        _ => unreachable!(
966            "BUG: token_kind_to_boolean_value called with non-boolean token {:?}",
967            kind
968        ),
969    }
970}
971
972/// Returns true if the token kind represents a math function keyword.
973pub fn is_math_function(kind: &TokenKind) -> bool {
974    matches!(
975        kind,
976        TokenKind::Sqrt
977            | TokenKind::Sin
978            | TokenKind::Cos
979            | TokenKind::Tan
980            | TokenKind::Asin
981            | TokenKind::Acos
982            | TokenKind::Atan
983            | TokenKind::Log
984            | TokenKind::Exp
985            | TokenKind::Abs
986            | TokenKind::Floor
987            | TokenKind::Ceil
988            | TokenKind::Round
989    )
990}
991
992/// Returns true if the token kind can start the body of a spec
993/// (`data`, `rule`, `meta`, or `uses`).
994pub fn is_spec_body_keyword(kind: &TokenKind) -> bool {
995    matches!(
996        kind,
997        TokenKind::Data | TokenKind::Rule | TokenKind::Meta | TokenKind::Uses
998    )
999}
1000
1001/// Returns true if the token kind can be used as a label or reference segment
1002/// (identifier, or one of the keywords `past` / `future` / `permille` / `is` in label position).
1003pub fn can_be_label(kind: &TokenKind) -> bool {
1004    matches!(
1005        kind,
1006        TokenKind::Identifier
1007            | TokenKind::Past
1008            | TokenKind::Future
1009            | TokenKind::Permille
1010            | TokenKind::Is
1011    )
1012}
1013
1014/// Returns true if the token kind can name a unit (`5 eur`, `-> unit eur: 1`).
1015/// `is`, `past`, `future`, `permille` are reserved and never units: after a number they start a
1016/// comparison, a date window, or a literal suffix.
1017#[must_use]
1018pub fn can_be_unit_name(kind: &TokenKind) -> bool {
1019    matches!(kind, TokenKind::Identifier)
1020}
1021
1022/// True when a bare (non-`@`) repository name spelling is a reserved keyword.
1023#[must_use]
1024pub fn is_reserved_repository_name(name: &str) -> bool {
1025    is_keyword(&keyword_from_identifier(name))
1026}
1027
1028/// True when the opaque spelling is allowed as a `spec` name: a plain identifier, not a keyword.
1029#[must_use]
1030pub fn can_be_spec_name_spelling(name: &str) -> bool {
1031    matches!(keyword_from_identifier(name), TokenKind::Identifier)
1032}
1033
1034/// `calendar` in `in calendar month` / `past calendar year` — not a type keyword.
1035#[must_use]
1036pub fn token_is_calendar_period_marker(tok: &Token) -> bool {
1037    tok.kind == TokenKind::Identifier && tok.text == "calendar"
1038}
1039
1040#[cfg(test)]
1041mod tests {
1042    use super::*;
1043
1044    fn lex_all(input: &str) -> Result<Vec<Token>, Error> {
1045        let mut lexer = Lexer::new(input, &crate::parsing::source::SourceType::Volatile);
1046        let mut tokens = Vec::new();
1047        loop {
1048            let token = lexer.next_token()?;
1049            if token.kind == TokenKind::Eof {
1050                tokens.push(token);
1051                break;
1052            }
1053            tokens.push(token);
1054        }
1055        Ok(tokens)
1056    }
1057
1058    fn lex_kinds(input: &str) -> Result<Vec<TokenKind>, Error> {
1059        Ok(lex_all(input)?.into_iter().map(|t| t.kind).collect())
1060    }
1061
1062    #[test]
1063    fn lex_empty_input() {
1064        let tokens = lex_all("").unwrap();
1065        assert_eq!(tokens.len(), 1);
1066        assert_eq!(tokens[0].kind, TokenKind::Eof);
1067    }
1068
1069    #[test]
1070    fn completion_keyword_labels_match_identifier_lex() {
1071        for label in COMPLETION_KEYWORD_LABELS {
1072            let kinds = lex_kinds(label).unwrap();
1073            assert_eq!(
1074                kinds.last(),
1075                Some(&TokenKind::Eof),
1076                "keyword {label} must be a single token"
1077            );
1078            assert_eq!(kinds.len(), 2, "keyword {label} lexed as {kinds:?}");
1079            assert_ne!(
1080                kinds[0],
1081                TokenKind::Identifier,
1082                "catalog entry {label} must be a keyword token"
1083            );
1084        }
1085    }
1086
1087    #[test]
1088    fn string_literal_at_max_length_is_accepted() {
1089        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH);
1090        let tokens = lex_all(&format!("\"{content}\"")).unwrap();
1091        assert_eq!(tokens[0].kind, TokenKind::StringLit);
1092    }
1093
1094    #[test]
1095    fn string_literal_over_max_length_is_parse_error() {
1096        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH + 1);
1097        let err = lex_all(&format!("\"{content}\"")).unwrap_err();
1098        assert!(
1099            err.message().contains("maximum length"),
1100            "expected length error, got: {err}"
1101        );
1102        assert!(err.location().is_some(), "parse error must carry a source");
1103    }
1104
1105    #[test]
1106    fn number_literal_with_separators_lexes() {
1107        let tokens = lex_all("9_999_999_999_999_999_999_999_999_999").unwrap();
1108        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1109    }
1110
1111    #[test]
1112    fn lex_spec_declaration() {
1113        let kinds = lex_kinds("spec person").unwrap();
1114        assert_eq!(
1115            kinds,
1116            vec![TokenKind::Spec, TokenKind::Identifier, TokenKind::Eof]
1117        );
1118    }
1119
1120    #[test]
1121    fn lex_data_definition() {
1122        let kinds = lex_kinds("data age: 25").unwrap();
1123        assert_eq!(
1124            kinds,
1125            vec![
1126                TokenKind::Data,
1127                TokenKind::Identifier,
1128                TokenKind::Colon,
1129                TokenKind::NumberLit,
1130                TokenKind::Eof,
1131            ]
1132        );
1133    }
1134
1135    #[test]
1136    fn lex_rule_with_comparison() {
1137        let kinds = lex_kinds("rule is_adult: age >= 18").unwrap();
1138        assert_eq!(
1139            kinds,
1140            vec![
1141                TokenKind::Rule,
1142                TokenKind::Identifier,
1143                TokenKind::Colon,
1144                TokenKind::Identifier,
1145                TokenKind::Gte,
1146                TokenKind::NumberLit,
1147                TokenKind::Eof,
1148            ]
1149        );
1150    }
1151
1152    #[test]
1153    fn lex_string_literal() {
1154        let tokens = lex_all(r#""hello world""#).unwrap();
1155        assert_eq!(tokens[0].kind, TokenKind::StringLit);
1156        assert_eq!(tokens[0].text, "hello world");
1157    }
1158
1159    #[test]
1160    fn lex_unterminated_string() {
1161        let result = lex_all(r#""hello"#);
1162        assert!(result.is_err());
1163    }
1164
1165    #[test]
1166    fn lex_number_with_decimal() {
1167        let tokens = lex_all("3.14").unwrap();
1168        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1169        assert_eq!(tokens[0].text, "3.14");
1170    }
1171
1172    #[test]
1173    fn lex_number_with_underscores() {
1174        let tokens = lex_all("1_000_000").unwrap();
1175        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1176        assert_eq!(tokens[0].text, "1_000_000");
1177    }
1178
1179    #[test]
1180    fn lex_scientific_notation() {
1181        let tokens = lex_all("1.5e+10").unwrap();
1182        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1183        assert_eq!(tokens[0].text, "1.5e+10");
1184    }
1185
1186    #[test]
1187    fn lex_all_operators() {
1188        let kinds = lex_kinds("+ - * / % ^ > < >= <= -> %%").unwrap();
1189        assert_eq!(
1190            &kinds[..12],
1191            &[
1192                TokenKind::Plus,
1193                TokenKind::Minus,
1194                TokenKind::Star,
1195                TokenKind::Slash,
1196                TokenKind::Percent,
1197                TokenKind::Caret,
1198                TokenKind::Gt,
1199                TokenKind::Lt,
1200                TokenKind::Gte,
1201                TokenKind::Lte,
1202                TokenKind::Arrow,
1203                TokenKind::PercentPercent,
1204            ]
1205        );
1206    }
1207
1208    #[test]
1209    fn lex_keywords() {
1210        let kinds =
1211            lex_kinds("spec data rule unless then not and in as uses meta veto now").unwrap();
1212        assert_eq!(
1213            &kinds[..13],
1214            &[
1215                TokenKind::Spec,
1216                TokenKind::Data,
1217                TokenKind::Rule,
1218                TokenKind::Unless,
1219                TokenKind::Then,
1220                TokenKind::Not,
1221                TokenKind::And,
1222                TokenKind::In,
1223                TokenKind::As,
1224                TokenKind::Uses,
1225                TokenKind::Meta,
1226                TokenKind::Veto,
1227                TokenKind::Now,
1228            ]
1229        );
1230    }
1231
1232    #[test]
1233    fn lex_boolean_keywords() {
1234        let kinds = lex_kinds("true false yes no").unwrap();
1235        assert_eq!(
1236            &kinds[..4],
1237            &[
1238                TokenKind::True,
1239                TokenKind::False,
1240                TokenKind::Yes,
1241                TokenKind::No,
1242            ]
1243        );
1244    }
1245
1246    #[test]
1247    fn lex_duration_keywords() {
1248        let kinds = lex_kinds("year month week day hour minute second").unwrap();
1249        assert_eq!(
1250            &kinds[..7],
1251            &[
1252                TokenKind::Identifier,
1253                TokenKind::Identifier,
1254                TokenKind::Identifier,
1255                TokenKind::Identifier,
1256                TokenKind::Identifier,
1257                TokenKind::Identifier,
1258                TokenKind::Identifier,
1259            ]
1260        );
1261    }
1262
1263    #[test]
1264    fn lex_commentary() {
1265        let tokens = lex_all(r#""""hello world""""#).unwrap();
1266        assert_eq!(tokens[0].kind, TokenKind::Commentary);
1267        assert_eq!(tokens[0].text, "hello world");
1268    }
1269
1270    #[test]
1271    fn lex_registry_logical_name() {
1272        let tokens = lex_all("@iso/countries").unwrap();
1273        assert_eq!(tokens[0].kind, TokenKind::LogicalName);
1274        assert_eq!(tokens[0].text, "@iso/countries");
1275        assert_eq!(tokens[1].kind, TokenKind::Eof);
1276    }
1277
1278    #[test]
1279    fn lex_bare_at_is_error() {
1280        assert!(lex_all("@").is_err());
1281    }
1282
1283    #[test]
1284    fn next_logical_name_absorbs_slashes() {
1285        let mut lexer = Lexer::new(
1286            "contracts/employment/jack",
1287            &crate::parsing::source::SourceType::Volatile,
1288        );
1289        let tok = lexer.next_logical_name().unwrap();
1290        assert_eq!(tok.kind, TokenKind::LogicalName);
1291        assert_eq!(tok.text, "contracts/employment/jack");
1292    }
1293
1294    #[test]
1295    fn next_logical_name_after_peeked_identifier_rewinds() {
1296        let mut lexer = Lexer::new(
1297            "finance/units",
1298            &crate::parsing::source::SourceType::Volatile,
1299        );
1300        assert_eq!(lexer.peek().unwrap().kind, TokenKind::Identifier);
1301        let tok = lexer.next_logical_name().unwrap();
1302        assert_eq!(tok.kind, TokenKind::LogicalName);
1303        assert_eq!(tok.text, "finance/units");
1304    }
1305
1306    #[test]
1307    fn division_and_units_keep_slash_token() {
1308        let kinds = lex_kinds("a / b").unwrap();
1309        assert_eq!(
1310            &kinds[..3],
1311            &[
1312                TokenKind::Identifier,
1313                TokenKind::Slash,
1314                TokenKind::Identifier
1315            ]
1316        );
1317        let kinds = lex_kinds("meter/second").unwrap();
1318        assert_eq!(
1319            &kinds[..3],
1320            &[
1321                TokenKind::Identifier,
1322                TokenKind::Slash,
1323                TokenKind::Identifier
1324            ]
1325        );
1326    }
1327
1328    #[test]
1329    fn lex_parentheses() {
1330        let kinds = lex_kinds("(x + 1)").unwrap();
1331        assert_eq!(
1332            &kinds[..5],
1333            &[
1334                TokenKind::LParen,
1335                TokenKind::Identifier,
1336                TokenKind::Plus,
1337                TokenKind::NumberLit,
1338                TokenKind::RParen,
1339            ]
1340        );
1341    }
1342
1343    #[test]
1344    fn lex_dot_for_references() {
1345        let kinds = lex_kinds("employee.salary").unwrap();
1346        assert_eq!(
1347            &kinds[..3],
1348            &[TokenKind::Identifier, TokenKind::Dot, TokenKind::Identifier]
1349        );
1350    }
1351
1352    #[test]
1353    fn lex_spec_keyword_then_slashed_name_via_next_logical_name() {
1354        let mut lexer = Lexer::new(
1355            "spec contracts/employment/jack",
1356            &crate::parsing::source::SourceType::Volatile,
1357        );
1358        assert_eq!(lexer.next_token().unwrap().kind, TokenKind::Spec);
1359        let name = lexer.next_logical_name().unwrap();
1360        assert_eq!(name.kind, TokenKind::LogicalName);
1361        assert_eq!(name.text, "contracts/employment/jack");
1362    }
1363
1364    #[test]
1365    fn lex_number_not_followed_by_e_identifier() {
1366        // "42 eur" should be number then identifier, not scientific notation
1367        let tokens = lex_all("42 eur").unwrap();
1368        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1369        assert_eq!(tokens[0].text, "42");
1370        assert_eq!(tokens[1].kind, TokenKind::Identifier);
1371        assert_eq!(tokens[1].text, "eur");
1372    }
1373
1374    #[test]
1375    fn lex_unknown_character() {
1376        let result = lex_all("§");
1377        assert!(result.is_err());
1378    }
1379
1380    #[test]
1381    fn lex_peek_does_not_consume() {
1382        let mut lexer = Lexer::new("spec test", &crate::parsing::source::SourceType::Volatile);
1383        let peeked_kind = lexer.peek().unwrap().kind.clone();
1384        assert_eq!(peeked_kind, TokenKind::Spec);
1385        let next = lexer.next_token().unwrap();
1386        assert_eq!(next.kind, TokenKind::Spec);
1387    }
1388
1389    #[test]
1390    fn lex_span_byte_offsets() {
1391        let tokens = lex_all("spec test").unwrap();
1392        assert_eq!(tokens[0].span.start, 0);
1393        assert_eq!(tokens[0].span.end, 4);
1394        assert_eq!(tokens[0].span.line, 1);
1395        assert_eq!(tokens[0].span.col, 1);
1396
1397        assert_eq!(tokens[1].span.start, 5);
1398        assert_eq!(tokens[1].span.end, 9);
1399        assert_eq!(tokens[1].span.line, 1);
1400        assert_eq!(tokens[1].span.col, 6);
1401    }
1402
1403    #[test]
1404    fn lex_multiline_span_tracking() {
1405        let tokens = lex_all("spec test\ndata x: 1").unwrap();
1406        // "data" should be on line 2
1407        let data_token = &tokens[2]; // spec, test, data
1408        assert_eq!(data_token.kind, TokenKind::Data);
1409        assert_eq!(data_token.span.line, 2);
1410        assert_eq!(data_token.span.col, 1);
1411    }
1412
1413    #[test]
1414    fn lex_case_insensitive_keywords() {
1415        // Lemma keywords are case-insensitive
1416        let kinds = lex_kinds("SPEC Data RULE").unwrap();
1417        assert_eq!(kinds[0], TokenKind::Spec);
1418        assert_eq!(kinds[1], TokenKind::Data);
1419        assert_eq!(kinds[2], TokenKind::Rule);
1420    }
1421
1422    #[test]
1423    fn lex_math_function_keywords() {
1424        let kinds =
1425            lex_kinds("sqrt sin cos tan asin acos atan log exp abs floor ceil round").unwrap();
1426        assert_eq!(
1427            &kinds[..13],
1428            &[
1429                TokenKind::Sqrt,
1430                TokenKind::Sin,
1431                TokenKind::Cos,
1432                TokenKind::Tan,
1433                TokenKind::Asin,
1434                TokenKind::Acos,
1435                TokenKind::Atan,
1436                TokenKind::Log,
1437                TokenKind::Exp,
1438                TokenKind::Abs,
1439                TokenKind::Floor,
1440                TokenKind::Ceil,
1441                TokenKind::Round,
1442            ]
1443        );
1444    }
1445
1446    #[test]
1447    fn lex_range_bound_keywords() {
1448        let kinds = lex_kinds("lower upper LOWER").unwrap();
1449        assert_eq!(
1450            &kinds[..3],
1451            &[TokenKind::Lower, TokenKind::Upper, TokenKind::Lower,]
1452        );
1453    }
1454
1455    #[test]
1456    fn lex_is_keyword() {
1457        let kinds = lex_kinds("status is \"active\"").unwrap();
1458        assert_eq!(kinds[0], TokenKind::Identifier);
1459        assert_eq!(kinds[1], TokenKind::Is);
1460        assert_eq!(kinds[2], TokenKind::StringLit);
1461    }
1462
1463    #[test]
1464    fn lex_percent_not_followed_by_digit() {
1465        // "50%" should be number then percent
1466        let kinds = lex_kinds("50%").unwrap();
1467        assert_eq!(kinds[0], TokenKind::NumberLit);
1468        assert_eq!(kinds[1], TokenKind::Percent);
1469    }
1470
1471    #[test]
1472    fn lex_number_with_commas() {
1473        let kinds = lex_kinds("1,000,000").unwrap();
1474        assert_eq!(
1475            kinds,
1476            vec![
1477                TokenKind::NumberLit,
1478                TokenKind::Comma,
1479                TokenKind::NumberLit,
1480                TokenKind::Comma,
1481                TokenKind::NumberLit,
1482                TokenKind::Eof,
1483            ]
1484        );
1485    }
1486
1487    #[test]
1488    fn lex_arrow_chain() {
1489        let kinds = lex_kinds("-> unit eur: 1.00 -> decimals 2").unwrap();
1490        assert_eq!(kinds[0], TokenKind::Arrow);
1491        assert_eq!(kinds[1], TokenKind::Identifier);
1492        assert_eq!(kinds[2], TokenKind::Identifier);
1493        assert_eq!(kinds[3], TokenKind::Colon);
1494        assert_eq!(kinds[4], TokenKind::NumberLit);
1495        assert_eq!(kinds[5], TokenKind::Arrow);
1496    }
1497}
lemma/parsing/lexer.rs

lemma/parsing/
lexer.rs
