lemma/parsing/
lexer.rs

1use crate::error::Error;
2use crate::parsing::ast::{BooleanValue, PrimitiveKind, Span};
3use crate::parsing::source::Source;
4use std::sync::Arc;
5
6#[derive(Debug, Clone, PartialEq, Eq)]
7pub enum TokenKind {
8    // Keywords
9    Spec,
10    Repo,
11    Data,
12    Rule,
13    Unless,
14    Then,
15    Not,
16    And,
17    In,
18    As,
19    Uses,
20    With,
21    Meta,
22    Veto,
23    Now,
24    Past,
25    Future,
26
27    // Boolean keywords
28    True,
29    False,
30    Yes,
31    No,
32
33    // Type keywords
34    MeasureKw,
35    NumberKw,
36    TextKw,
37    DateKw,
38    TimeKw,
39    BooleanKw,
40    RatioKw,
41
42    // Math function keywords
43    Sqrt,
44    Sin,
45    Cos,
46    Tan,
47    Asin,
48    Acos,
49    Atan,
50    Log,
51    Exp,
52    Abs,
53    Floor,
54    Ceil,
55    Round,
56
57    // Range endpoint operators
58    Lower,
59    Upper,
60
61    Permille,
62
63    // Comparison keyword operators
64    Is,
65
66    // Operators
67    Plus,
68    Minus,
69    Star,
70    Slash,
71    Comma,
72    Percent,
73    PercentPercent,
74    Caret,
75    Gt,
76    Lt,
77    Gte,
78    Lte,
79
80    // Punctuation
81    Colon,
82    Arrow,
83    Ellipsis,
84    Dot,
85    At,
86    LParen,
87    RParen,
88
89    // Literals
90    NumberLit,
91    StringLit,
92
93    // Commentary (raw text between """ delimiters)
94    Commentary,
95
96    // Identifiers
97    Identifier,
98
99    // End of file
100    Eof,
101}
102
103impl std::fmt::Display for TokenKind {
104    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
105        match self {
106            TokenKind::Spec => write!(f, "'spec'"),
107            TokenKind::Repo => write!(f, "'repo'"),
108            TokenKind::Data => write!(f, "'data'"),
109            TokenKind::Rule => write!(f, "'rule'"),
110            TokenKind::Unless => write!(f, "'unless'"),
111            TokenKind::Then => write!(f, "'then'"),
112            TokenKind::Not => write!(f, "'not'"),
113            TokenKind::And => write!(f, "'and'"),
114            TokenKind::In => write!(f, "'in'"),
115            TokenKind::As => write!(f, "'as'"),
116            TokenKind::Uses => write!(f, "'uses'"),
117            TokenKind::With => write!(f, "'with'"),
118            TokenKind::Meta => write!(f, "'meta'"),
119            TokenKind::Veto => write!(f, "'veto'"),
120            TokenKind::Now => write!(f, "'now'"),
121            TokenKind::Past => write!(f, "'past'"),
122            TokenKind::Future => write!(f, "'future'"),
123            TokenKind::True => write!(f, "'true'"),
124            TokenKind::False => write!(f, "'false'"),
125            TokenKind::Yes => write!(f, "'yes'"),
126            TokenKind::No => write!(f, "'no'"),
127            TokenKind::MeasureKw => write!(f, "'measure'"),
128            TokenKind::NumberKw => write!(f, "'number'"),
129            TokenKind::TextKw => write!(f, "'text'"),
130            TokenKind::DateKw => write!(f, "'date'"),
131            TokenKind::TimeKw => write!(f, "'time'"),
132            TokenKind::BooleanKw => write!(f, "'boolean'"),
133            TokenKind::RatioKw => write!(f, "'ratio'"),
134            TokenKind::Sqrt => write!(f, "'sqrt'"),
135            TokenKind::Sin => write!(f, "'sin'"),
136            TokenKind::Cos => write!(f, "'cos'"),
137            TokenKind::Tan => write!(f, "'tan'"),
138            TokenKind::Asin => write!(f, "'asin'"),
139            TokenKind::Acos => write!(f, "'acos'"),
140            TokenKind::Atan => write!(f, "'atan'"),
141            TokenKind::Log => write!(f, "'log'"),
142            TokenKind::Exp => write!(f, "'exp'"),
143            TokenKind::Abs => write!(f, "'abs'"),
144            TokenKind::Floor => write!(f, "'floor'"),
145            TokenKind::Ceil => write!(f, "'ceil'"),
146            TokenKind::Round => write!(f, "'round'"),
147            TokenKind::Lower => write!(f, "'lower'"),
148            TokenKind::Upper => write!(f, "'upper'"),
149            TokenKind::Permille => write!(f, "'permille'"),
150            TokenKind::Is => write!(f, "'is'"),
151            TokenKind::Plus => write!(f, "'+'"),
152            TokenKind::Minus => write!(f, "'-'"),
153            TokenKind::Star => write!(f, "'*'"),
154            TokenKind::Slash => write!(f, "'/'"),
155            TokenKind::Comma => write!(f, "','"),
156            TokenKind::Percent => write!(f, "'%'"),
157            TokenKind::PercentPercent => write!(f, "'%%'"),
158            TokenKind::Caret => write!(f, "'^'"),
159            TokenKind::Gt => write!(f, "'>'"),
160            TokenKind::Lt => write!(f, "'<'"),
161            TokenKind::Gte => write!(f, "'>='"),
162            TokenKind::Lte => write!(f, "'<='"),
163            TokenKind::Colon => write!(f, "':'"),
164            TokenKind::Arrow => write!(f, "'->'"),
165            TokenKind::Ellipsis => write!(f, "'...'"),
166            TokenKind::Dot => write!(f, "'.'"),
167            TokenKind::At => write!(f, "'@'"),
168            TokenKind::LParen => write!(f, "'('"),
169            TokenKind::RParen => write!(f, "')'"),
170            TokenKind::NumberLit => write!(f, "a number"),
171            TokenKind::StringLit => write!(f, "a string"),
172            TokenKind::Commentary => write!(f, "commentary block"),
173            TokenKind::Identifier => write!(f, "an identifier"),
174            TokenKind::Eof => write!(f, "end of file"),
175        }
176    }
177}
178
179#[derive(Debug, Clone)]
180pub struct Token {
181    pub kind: TokenKind,
182    pub span: Span,
183    pub text: String,
184}
185
186impl Token {
187    pub fn eof(offset: usize, line: usize, col: usize) -> Self {
188        Token {
189            kind: TokenKind::Eof,
190            span: Span {
191                start: offset,
192                end: offset,
193                line,
194                col,
195            },
196            text: String::new(),
197        }
198    }
199}
200
201#[derive(Clone)]
202pub struct LexerCheckpoint {
203    pos: usize,
204    line: usize,
205    col: usize,
206    byte_offset: usize,
207    peeked: Option<Token>,
208    peeked2: Option<Token>,
209}
210
211// todo: find out why derive Clone is necessary
212#[derive(Clone)]
213pub struct Lexer {
214    source: Vec<char>,
215    pos: usize,
216    line: usize,
217    col: usize,
218    byte_offset: usize,
219    source_type: crate::parsing::source::SourceType,
220    source_text: Arc<str>,
221    peeked: Option<Token>,
222    peeked2: Option<Token>,
223}
224
225impl Lexer {
226    pub fn new(input: &str, source_type: &crate::parsing::source::SourceType) -> Self {
227        let source_text: Arc<str> = Arc::from(input);
228        Lexer {
229            source: input.chars().collect(),
230            pos: 0,
231            line: 1,
232            col: 1,
233            byte_offset: 0,
234            source_type: source_type.clone(),
235            source_text,
236            peeked: None,
237            peeked2: None,
238        }
239    }
240
241    pub fn peek(&mut self) -> Result<&Token, Error> {
242        if self.peeked.is_none() {
243            let token = self.lex_token()?;
244            self.peeked = Some(token);
245        }
246        Ok(self.peeked.as_ref().expect("just assigned"))
247    }
248
249    pub fn peek_second(&mut self) -> Result<&Token, Error> {
250        self.peek()?;
251        if self.peeked2.is_none() {
252            let token = self.lex_token()?;
253            self.peeked2 = Some(token);
254        }
255        Ok(self.peeked2.as_ref().expect("just assigned"))
256    }
257
258    pub fn next_token(&mut self) -> Result<Token, Error> {
259        if let Some(token) = self.peeked.take() {
260            self.peeked = self.peeked2.take();
261            return Ok(token);
262        }
263        self.lex_token()
264    }
265
266    /// Saved lexer position for speculative parsing.
267    pub fn checkpoint(&self) -> LexerCheckpoint {
268        LexerCheckpoint {
269            pos: self.pos,
270            line: self.line,
271            col: self.col,
272            byte_offset: self.byte_offset,
273            peeked: self.peeked.clone(),
274            peeked2: self.peeked2.clone(),
275        }
276    }
277
278    pub fn restore(&mut self, checkpoint: LexerCheckpoint) {
279        self.pos = checkpoint.pos;
280        self.line = checkpoint.line;
281        self.col = checkpoint.col;
282        self.byte_offset = checkpoint.byte_offset;
283        self.peeked = checkpoint.peeked;
284        self.peeked2 = checkpoint.peeked2;
285    }
286
287    fn current_char(&self) -> Option<char> {
288        self.source.get(self.pos).copied()
289    }
290
291    fn peek_char(&self) -> Option<char> {
292        self.source.get(self.pos + 1).copied()
293    }
294
295    fn peek_char_at(&self, offset: usize) -> Option<char> {
296        self.source.get(self.pos + offset).copied()
297    }
298
299    fn advance(&mut self) {
300        if let Some(ch) = self.current_char() {
301            self.byte_offset += ch.len_utf8();
302            if ch == '\n' {
303                self.line += 1;
304                self.col = 1;
305            } else {
306                self.col += 1;
307            }
308            self.pos += 1;
309        }
310    }
311
312    fn skip_whitespace(&mut self) {
313        while let Some(ch) = self.current_char() {
314            if ch.is_whitespace() {
315                self.advance();
316            } else {
317                break;
318            }
319        }
320    }
321
322    fn make_span(&self, start_byte: usize, start_line: usize, start_col: usize) -> Span {
323        Span {
324            start: start_byte,
325            end: self.byte_offset,
326            line: start_line,
327            col: start_col,
328        }
329    }
330
331    fn make_error(&self, message: impl Into<String>, span: Span) -> Error {
332        Error::parsing(
333            message,
334            Source::new(self.source_type.clone(), span),
335            None::<String>,
336        )
337    }
338
339    fn lex_token(&mut self) -> Result<Token, Error> {
340        self.skip_whitespace();
341
342        let start_byte = self.byte_offset;
343        let start_line = self.line;
344        let start_col = self.col;
345
346        let Some(ch) = self.current_char() else {
347            return Ok(Token::eof(start_byte, start_line, start_col));
348        };
349
350        // Triple-quote commentary
351        if ch == '"' && self.peek_char() == Some('"') && self.peek_char_at(2) == Some('"') {
352            return self.scan_triple_quote(start_byte, start_line, start_col);
353        }
354
355        // String literal
356        if ch == '"' {
357            return self.scan_string(start_byte, start_line, start_col);
358        }
359
360        // Number literal (sign handled by parser, not lexer)
361        if ch.is_ascii_digit() {
362            return self.scan_number(start_byte, start_line, start_col);
363        }
364
365        // Two-character operators (check before single-char)
366        if let Some(token) = self.try_two_char_operator(start_byte, start_line, start_col) {
367            return Ok(token);
368        }
369
370        // Three-character ellipsis
371        if ch == '.' && self.peek_char() == Some('.') && self.peek_char_at(2) == Some('.') {
372            self.advance();
373            self.advance();
374            self.advance();
375            let span = self.make_span(start_byte, start_line, start_col);
376            return Ok(Token {
377                kind: TokenKind::Ellipsis,
378                span,
379                text: "...".to_string(),
380            });
381        }
382
383        // Single-character operators/punctuation
384        if let Some(kind) = self.single_char_token(ch) {
385            self.advance();
386            let span = self.make_span(start_byte, start_line, start_col);
387            let text = ch.to_string();
388            return Ok(Token { kind, span, text });
389        }
390
391        // Identifier or keyword (starts with letter or @)
392        if ch.is_ascii_alphabetic() || ch == '_' {
393            return Ok(self.scan_identifier(start_byte, start_line, start_col));
394        }
395
396        // @ prefix for registry references
397        if ch == '@' {
398            self.advance();
399            let span = self.make_span(start_byte, start_line, start_col);
400            return Ok(Token {
401                kind: TokenKind::At,
402                span,
403                text: "@".to_string(),
404            });
405        }
406
407        // Unknown character
408        self.advance();
409        let span = self.make_span(start_byte, start_line, start_col);
410        Err(self.make_error(format!("Unexpected character '{}'", ch), span))
411    }
412
413    fn scan_triple_quote(
414        &mut self,
415        start_byte: usize,
416        start_line: usize,
417        start_col: usize,
418    ) -> Result<Token, Error> {
419        self.advance(); // "
420        self.advance(); // "
421        self.advance(); // "
422
423        let content_start = self.byte_offset;
424        loop {
425            match self.current_char() {
426                None => {
427                    let span = self.make_span(start_byte, start_line, start_col);
428                    return Err(self.make_error(
429                        "Unterminated commentary block: expected closing \"\"\"",
430                        span,
431                    ));
432                }
433                Some('"')
434                    if self.source.get(self.pos + 1) == Some(&'"')
435                        && self.source.get(self.pos + 2) == Some(&'"') =>
436                {
437                    let content_end = self.byte_offset;
438                    self.advance(); // "
439                    self.advance(); // "
440                    self.advance(); // "
441                    let raw: String = self.source_text[content_start..content_end].to_string();
442                    let span = self.make_span(start_byte, start_line, start_col);
443                    return Ok(Token {
444                        kind: TokenKind::Commentary,
445                        span,
446                        text: raw,
447                    });
448                }
449                Some(_) => {
450                    self.advance();
451                }
452            }
453        }
454    }
455
456    fn scan_string(
457        &mut self,
458        start_byte: usize,
459        start_line: usize,
460        start_col: usize,
461    ) -> Result<Token, Error> {
462        self.advance(); // consume opening "
463        let mut content = String::new();
464        loop {
465            match self.current_char() {
466                None => {
467                    let span = self.make_span(start_byte, start_line, start_col);
468                    return Err(self.make_error("String starting here was never closed", span));
469                }
470                Some('"') => {
471                    self.advance(); // consume closing "
472                    break;
473                }
474                Some(ch) => {
475                    content.push(ch);
476                    self.advance();
477                }
478            }
479        }
480        let span = self.make_span(start_byte, start_line, start_col);
481        if content.len() > crate::limits::MAX_TEXT_VALUE_LENGTH {
482            return Err(self.make_error(
483                format!(
484                    "Text literal exceeds maximum length of {} characters (found {})",
485                    crate::limits::MAX_TEXT_VALUE_LENGTH,
486                    content.len()
487                ),
488                span,
489            ));
490        }
491        // Token text is the unquoted content. The span still covers the quotes so
492        // diagnostics point at the full literal; the parser must not strip again.
493        Ok(Token {
494            kind: TokenKind::StringLit,
495            span,
496            text: content,
497        })
498    }
499
500    fn scan_number(
501        &mut self,
502        start_byte: usize,
503        start_line: usize,
504        start_col: usize,
505    ) -> Result<Token, Error> {
506        let mut text = String::new();
507
508        // Integer part: digits with optional _ or , separators
509        while let Some(ch) = self.current_char() {
510            if ch.is_ascii_digit() || ch == '_' || ch == ',' {
511                text.push(ch);
512                self.advance();
513            } else {
514                break;
515            }
516        }
517
518        // Decimal part
519        if self.current_char() == Some('.') {
520            // Check if next char after dot is a digit (not a method call or dotted reference)
521            if let Some(next) = self.peek_char() {
522                if next.is_ascii_digit() {
523                    text.push('.');
524                    self.advance(); // consume .
525                    while let Some(ch) = self.current_char() {
526                        if ch.is_ascii_digit() {
527                            text.push(ch);
528                            self.advance();
529                        } else {
530                            break;
531                        }
532                    }
533                }
534            }
535        }
536
537        // Scientific notation: e or E followed by optional +/- and digits
538        if let Some(ch) = self.current_char() {
539            if ch == 'e' || ch == 'E' {
540                let mut sci_text = String::new();
541                sci_text.push(ch);
542                let save_pos = self.pos;
543                let save_byte = self.byte_offset;
544                let save_line = self.line;
545                let save_col = self.col;
546                self.advance(); // consume e/E
547
548                if let Some(sign) = self.current_char() {
549                    if sign == '+' || sign == '-' {
550                        sci_text.push(sign);
551                        self.advance();
552                    }
553                }
554
555                if let Some(d) = self.current_char() {
556                    if d.is_ascii_digit() {
557                        while let Some(ch) = self.current_char() {
558                            if ch.is_ascii_digit() {
559                                sci_text.push(ch);
560                                self.advance();
561                            } else {
562                                break;
563                            }
564                        }
565                        text.push_str(&sci_text);
566                    } else {
567                        // Not actually scientific notation, backtrack
568                        self.pos = save_pos;
569                        self.byte_offset = save_byte;
570                        self.line = save_line;
571                        self.col = save_col;
572                    }
573                } else {
574                    self.pos = save_pos;
575                    self.byte_offset = save_byte;
576                    self.line = save_line;
577                    self.col = save_col;
578                }
579            }
580        }
581
582        let span = self.make_span(start_byte, start_line, start_col);
583        Ok(Token {
584            kind: TokenKind::NumberLit,
585            span,
586            text,
587        })
588    }
589
590    fn try_two_char_operator(
591        &mut self,
592        start_byte: usize,
593        start_line: usize,
594        start_col: usize,
595    ) -> Option<Token> {
596        let ch = self.current_char()?;
597        let next = self.peek_char();
598
599        let kind = match (ch, next) {
600            ('-', Some('>')) => TokenKind::Arrow,
601            ('>', Some('=')) => TokenKind::Gte,
602            ('<', Some('=')) => TokenKind::Lte,
603            ('%', Some('%')) => {
604                // Check that it's not followed by a digit (invalid permille like 10%%5)
605                TokenKind::PercentPercent
606            }
607            _ => return None,
608        };
609
610        self.advance();
611        self.advance();
612        let span = self.make_span(start_byte, start_line, start_col);
613        let text: String = self.source_text[span.start..span.end].to_string();
614        Some(Token { kind, span, text })
615    }
616
617    fn single_char_token(&self, ch: char) -> Option<TokenKind> {
618        match ch {
619            '+' => Some(TokenKind::Plus),
620            '*' => Some(TokenKind::Star),
621            '/' => Some(TokenKind::Slash),
622            ',' => Some(TokenKind::Comma),
623            '^' => Some(TokenKind::Caret),
624            ':' => Some(TokenKind::Colon),
625            '.' => Some(TokenKind::Dot),
626            '(' => Some(TokenKind::LParen),
627            ')' => Some(TokenKind::RParen),
628            '>' => Some(TokenKind::Gt),
629            '<' => Some(TokenKind::Lt),
630            '%' => Some(TokenKind::Percent),
631            '-' => Some(TokenKind::Minus),
632            _ => None,
633        }
634    }
635
636    fn scan_identifier(&mut self, start_byte: usize, start_line: usize, start_col: usize) -> Token {
637        let mut text = String::new();
638        while let Some(ch) = self.current_char() {
639            if ch.is_ascii_alphanumeric() || ch == '_' {
640                text.push(ch);
641                self.advance();
642            } else {
643                break;
644            }
645        }
646
647        let kind = keyword_from_identifier(&text);
648        let span = self.make_span(start_byte, start_line, start_col);
649        Token { kind, span, text }
650    }
651}
652
653fn keyword_from_identifier(text: &str) -> TokenKind {
654    // ASCII fold without allocating a lowercased String on every identifier.
655    match text {
656        s if s.eq_ignore_ascii_case("spec") => TokenKind::Spec,
657        s if s.eq_ignore_ascii_case("repo") => TokenKind::Repo,
658        s if s.eq_ignore_ascii_case("data") => TokenKind::Data,
659        s if s.eq_ignore_ascii_case("rule") => TokenKind::Rule,
660        s if s.eq_ignore_ascii_case("unless") => TokenKind::Unless,
661        s if s.eq_ignore_ascii_case("then") => TokenKind::Then,
662        s if s.eq_ignore_ascii_case("not") => TokenKind::Not,
663        s if s.eq_ignore_ascii_case("and") => TokenKind::And,
664        s if s.eq_ignore_ascii_case("in") => TokenKind::In,
665        s if s.eq_ignore_ascii_case("as") => TokenKind::As,
666        s if s.eq_ignore_ascii_case("uses") => TokenKind::Uses,
667        s if s.eq_ignore_ascii_case("with") => TokenKind::With,
668        s if s.eq_ignore_ascii_case("meta") => TokenKind::Meta,
669        s if s.eq_ignore_ascii_case("veto") => TokenKind::Veto,
670        s if s.eq_ignore_ascii_case("now") => TokenKind::Now,
671        s if s.eq_ignore_ascii_case("past") => TokenKind::Past,
672        s if s.eq_ignore_ascii_case("future") => TokenKind::Future,
673        s if s.eq_ignore_ascii_case("true") => TokenKind::True,
674        s if s.eq_ignore_ascii_case("false") => TokenKind::False,
675        s if s.eq_ignore_ascii_case("yes") => TokenKind::Yes,
676        s if s.eq_ignore_ascii_case("no") => TokenKind::No,
677        s if s.eq_ignore_ascii_case("measure") => TokenKind::MeasureKw,
678        s if s.eq_ignore_ascii_case("number") => TokenKind::NumberKw,
679        s if s.eq_ignore_ascii_case("text") => TokenKind::TextKw,
680        s if s.eq_ignore_ascii_case("date") => TokenKind::DateKw,
681        s if s.eq_ignore_ascii_case("time") => TokenKind::TimeKw,
682        s if s.eq_ignore_ascii_case("boolean") => TokenKind::BooleanKw,
683        s if s.eq_ignore_ascii_case("ratio") => TokenKind::RatioKw,
684        s if s.eq_ignore_ascii_case("sqrt") => TokenKind::Sqrt,
685        s if s.eq_ignore_ascii_case("sin") => TokenKind::Sin,
686        s if s.eq_ignore_ascii_case("cos") => TokenKind::Cos,
687        s if s.eq_ignore_ascii_case("tan") => TokenKind::Tan,
688        s if s.eq_ignore_ascii_case("asin") => TokenKind::Asin,
689        s if s.eq_ignore_ascii_case("acos") => TokenKind::Acos,
690        s if s.eq_ignore_ascii_case("atan") => TokenKind::Atan,
691        s if s.eq_ignore_ascii_case("log") => TokenKind::Log,
692        s if s.eq_ignore_ascii_case("exp") => TokenKind::Exp,
693        s if s.eq_ignore_ascii_case("abs") => TokenKind::Abs,
694        s if s.eq_ignore_ascii_case("floor") => TokenKind::Floor,
695        s if s.eq_ignore_ascii_case("ceil") => TokenKind::Ceil,
696        s if s.eq_ignore_ascii_case("round") => TokenKind::Round,
697        s if s.eq_ignore_ascii_case("lower") => TokenKind::Lower,
698        s if s.eq_ignore_ascii_case("upper") => TokenKind::Upper,
699        s if s.eq_ignore_ascii_case("is") => TokenKind::Is,
700        s if s.eq_ignore_ascii_case("permille") => TokenKind::Permille,
701        _ => TokenKind::Identifier,
702    }
703}
704
705/// Structural keywords can never be used as identifiers (data/rule names).
706/// Type keywords (measure, number, text, date, time, boolean, ratio)
707/// are reserved and cannot be used as names.
708pub fn is_keyword(kind: &TokenKind) -> bool {
709    matches!(
710        kind,
711        TokenKind::Spec
712            | TokenKind::Repo
713            | TokenKind::Data
714            | TokenKind::Rule
715            | TokenKind::Unless
716            | TokenKind::Then
717            | TokenKind::Not
718            | TokenKind::And
719            | TokenKind::In
720            | TokenKind::As
721            | TokenKind::Uses
722            | TokenKind::With
723            | TokenKind::Meta
724            | TokenKind::Veto
725            | TokenKind::Now
726            | TokenKind::Sqrt
727            | TokenKind::Sin
728            | TokenKind::Cos
729            | TokenKind::Tan
730            | TokenKind::Asin
731            | TokenKind::Acos
732            | TokenKind::Atan
733            | TokenKind::Log
734            | TokenKind::Exp
735            | TokenKind::Abs
736            | TokenKind::Floor
737            | TokenKind::Ceil
738            | TokenKind::Round
739            | TokenKind::Lower
740            | TokenKind::Upper
741            | TokenKind::True
742            | TokenKind::False
743            | TokenKind::Yes
744            | TokenKind::No
745            | TokenKind::MeasureKw
746            | TokenKind::NumberKw
747            | TokenKind::TextKw
748            | TokenKind::DateKw
749            | TokenKind::TimeKw
750            | TokenKind::BooleanKw
751            | TokenKind::RatioKw
752    )
753}
754
755/// Map type keyword token to PrimitiveKind. Single source of truth for type keywords.
756#[must_use]
757pub fn token_kind_to_primitive(kind: &TokenKind) -> Option<PrimitiveKind> {
758    match kind {
759        TokenKind::BooleanKw => Some(PrimitiveKind::Boolean),
760        TokenKind::MeasureKw => Some(PrimitiveKind::Measure),
761        TokenKind::NumberKw => Some(PrimitiveKind::Number),
762        TokenKind::RatioKw => Some(PrimitiveKind::Ratio),
763        TokenKind::TextKw => Some(PrimitiveKind::Text),
764        TokenKind::DateKw => Some(PrimitiveKind::Date),
765        TokenKind::TimeKw => Some(PrimitiveKind::Time),
766        _ => None,
767    }
768}
769
770/// Returns true if the token kind represents a boolean literal keyword.
771pub fn is_boolean_keyword(kind: &TokenKind) -> bool {
772    matches!(
773        kind,
774        TokenKind::True | TokenKind::False | TokenKind::Yes | TokenKind::No
775    )
776}
777
778/// Maps a boolean-keyword token kind to BooleanValue. Call only when `is_boolean_keyword(kind)`.
779#[must_use]
780pub fn token_kind_to_boolean_value(kind: &TokenKind) -> BooleanValue {
781    match kind {
782        TokenKind::True => BooleanValue::True,
783        TokenKind::False => BooleanValue::False,
784        TokenKind::Yes => BooleanValue::Yes,
785        TokenKind::No => BooleanValue::No,
786        _ => unreachable!(
787            "BUG: token_kind_to_boolean_value called with non-boolean token {:?}",
788            kind
789        ),
790    }
791}
792
793/// Returns true if the token kind represents a math function keyword.
794pub fn is_math_function(kind: &TokenKind) -> bool {
795    matches!(
796        kind,
797        TokenKind::Sqrt
798            | TokenKind::Sin
799            | TokenKind::Cos
800            | TokenKind::Tan
801            | TokenKind::Asin
802            | TokenKind::Acos
803            | TokenKind::Atan
804            | TokenKind::Log
805            | TokenKind::Exp
806            | TokenKind::Abs
807            | TokenKind::Floor
808            | TokenKind::Ceil
809            | TokenKind::Round
810    )
811}
812
813/// Returns true if the token kind can start the body of a spec
814/// (`data`, `with`, `rule`, `meta`, or `uses`).
815pub fn is_spec_body_keyword(kind: &TokenKind) -> bool {
816    matches!(
817        kind,
818        TokenKind::Data | TokenKind::With | TokenKind::Rule | TokenKind::Meta | TokenKind::Uses
819    )
820}
821
822/// Returns true if the token kind can be used as a label or reference segment
823/// (identifier, or non-reserved contextual keyword such as `past` / `future` /
824/// `permille` / `is`).
825pub fn can_be_label(kind: &TokenKind) -> bool {
826    matches!(
827        kind,
828        TokenKind::Identifier
829            | TokenKind::Past
830            | TokenKind::Future
831            | TokenKind::Permille
832            | TokenKind::Is
833    )
834}
835
836/// `calendar` in `in calendar month` / `past calendar year` — not a type keyword.
837#[must_use]
838pub fn token_is_calendar_period_marker(tok: &Token) -> bool {
839    tok.kind == TokenKind::Identifier && tok.text == "calendar"
840}
841
842/// Slash-/dot-separated registry path segments (`@org/repo/...`). Keywords that are
843/// reserved at the structural level (`spec`, `rule`, etc.) are allowed inside
844/// multi-segment paths (e.g. `@org/repo`) but callers must reject them when they
845/// appear as the entire stand-alone name (e.g. `repo spec`).
846#[must_use]
847pub fn can_be_repository_qualifier_segment(kind: &TokenKind) -> bool {
848    matches!(kind, TokenKind::Identifier)
849        || is_keyword(kind)
850        || can_be_label(kind)
851        || is_boolean_keyword(kind)
852        || is_math_function(kind)
853}
854
855#[cfg(test)]
856mod tests {
857    use super::*;
858
859    fn lex_all(input: &str) -> Result<Vec<Token>, Error> {
860        let mut lexer = Lexer::new(input, &crate::parsing::source::SourceType::Volatile);
861        let mut tokens = Vec::new();
862        loop {
863            let token = lexer.next_token()?;
864            if token.kind == TokenKind::Eof {
865                tokens.push(token);
866                break;
867            }
868            tokens.push(token);
869        }
870        Ok(tokens)
871    }
872
873    fn lex_kinds(input: &str) -> Result<Vec<TokenKind>, Error> {
874        Ok(lex_all(input)?.into_iter().map(|t| t.kind).collect())
875    }
876
877    #[test]
878    fn lex_empty_input() {
879        let tokens = lex_all("").unwrap();
880        assert_eq!(tokens.len(), 1);
881        assert_eq!(tokens[0].kind, TokenKind::Eof);
882    }
883
884    #[test]
885    fn string_literal_at_max_length_is_accepted() {
886        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH);
887        let tokens = lex_all(&format!("\"{content}\"")).unwrap();
888        assert_eq!(tokens[0].kind, TokenKind::StringLit);
889    }
890
891    #[test]
892    fn string_literal_over_max_length_is_parse_error() {
893        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH + 1);
894        let err = lex_all(&format!("\"{content}\"")).unwrap_err();
895        assert!(
896            err.message().contains("maximum length"),
897            "expected length error, got: {err}"
898        );
899        assert!(err.location().is_some(), "parse error must carry a source");
900    }
901
902    #[test]
903    fn number_literal_with_separators_lexes() {
904        let tokens = lex_all("9,999,999,999,999,999,999,999,999,999").unwrap();
905        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
906    }
907
908    #[test]
909    fn lex_spec_declaration() {
910        let kinds = lex_kinds("spec person").unwrap();
911        assert_eq!(
912            kinds,
913            vec![TokenKind::Spec, TokenKind::Identifier, TokenKind::Eof]
914        );
915    }
916
917    #[test]
918    fn lex_data_definition() {
919        let kinds = lex_kinds("data age: 25").unwrap();
920        assert_eq!(
921            kinds,
922            vec![
923                TokenKind::Data,
924                TokenKind::Identifier,
925                TokenKind::Colon,
926                TokenKind::NumberLit,
927                TokenKind::Eof,
928            ]
929        );
930    }
931
932    #[test]
933    fn lex_rule_with_comparison() {
934        let kinds = lex_kinds("rule is_adult: age >= 18").unwrap();
935        assert_eq!(
936            kinds,
937            vec![
938                TokenKind::Rule,
939                TokenKind::Identifier,
940                TokenKind::Colon,
941                TokenKind::Identifier,
942                TokenKind::Gte,
943                TokenKind::NumberLit,
944                TokenKind::Eof,
945            ]
946        );
947    }
948
949    #[test]
950    fn lex_string_literal() {
951        let tokens = lex_all(r#""hello world""#).unwrap();
952        assert_eq!(tokens[0].kind, TokenKind::StringLit);
953        assert_eq!(tokens[0].text, "hello world");
954    }
955
956    #[test]
957    fn lex_unterminated_string() {
958        let result = lex_all(r#""hello"#);
959        assert!(result.is_err());
960    }
961
962    #[test]
963    fn lex_number_with_decimal() {
964        let tokens = lex_all("3.14").unwrap();
965        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
966        assert_eq!(tokens[0].text, "3.14");
967    }
968
969    #[test]
970    fn lex_number_with_underscores() {
971        let tokens = lex_all("1_000_000").unwrap();
972        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
973        assert_eq!(tokens[0].text, "1_000_000");
974    }
975
976    #[test]
977    fn lex_scientific_notation() {
978        let tokens = lex_all("1.5e+10").unwrap();
979        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
980        assert_eq!(tokens[0].text, "1.5e+10");
981    }
982
983    #[test]
984    fn lex_all_operators() {
985        let kinds = lex_kinds("+ - * / % ^ > < >= <= -> %%").unwrap();
986        assert_eq!(
987            &kinds[..12],
988            &[
989                TokenKind::Plus,
990                TokenKind::Minus,
991                TokenKind::Star,
992                TokenKind::Slash,
993                TokenKind::Percent,
994                TokenKind::Caret,
995                TokenKind::Gt,
996                TokenKind::Lt,
997                TokenKind::Gte,
998                TokenKind::Lte,
999                TokenKind::Arrow,
1000                TokenKind::PercentPercent,
1001            ]
1002        );
1003    }
1004
1005    #[test]
1006    fn lex_keywords() {
1007        let kinds =
1008            lex_kinds("spec data rule unless then not and in as uses meta veto now").unwrap();
1009        assert_eq!(
1010            &kinds[..13],
1011            &[
1012                TokenKind::Spec,
1013                TokenKind::Data,
1014                TokenKind::Rule,
1015                TokenKind::Unless,
1016                TokenKind::Then,
1017                TokenKind::Not,
1018                TokenKind::And,
1019                TokenKind::In,
1020                TokenKind::As,
1021                TokenKind::Uses,
1022                TokenKind::Meta,
1023                TokenKind::Veto,
1024                TokenKind::Now,
1025            ]
1026        );
1027    }
1028
1029    #[test]
1030    fn lex_boolean_keywords() {
1031        let kinds = lex_kinds("true false yes no").unwrap();
1032        assert_eq!(
1033            &kinds[..4],
1034            &[
1035                TokenKind::True,
1036                TokenKind::False,
1037                TokenKind::Yes,
1038                TokenKind::No,
1039            ]
1040        );
1041    }
1042
1043    #[test]
1044    fn lex_duration_keywords() {
1045        let kinds = lex_kinds("year month week day hour minute second").unwrap();
1046        assert_eq!(
1047            &kinds[..7],
1048            &[
1049                TokenKind::Identifier,
1050                TokenKind::Identifier,
1051                TokenKind::Identifier,
1052                TokenKind::Identifier,
1053                TokenKind::Identifier,
1054                TokenKind::Identifier,
1055                TokenKind::Identifier,
1056            ]
1057        );
1058    }
1059
1060    #[test]
1061    fn lex_commentary() {
1062        let tokens = lex_all(r#""""hello world""""#).unwrap();
1063        assert_eq!(tokens[0].kind, TokenKind::Commentary);
1064        assert_eq!(tokens[0].text, "hello world");
1065    }
1066
1067    #[test]
1068    fn lex_at_sign() {
1069        let kinds = lex_kinds("@user").unwrap();
1070        assert_eq!(kinds[0], TokenKind::At);
1071        assert_eq!(kinds[1], TokenKind::Identifier);
1072    }
1073
1074    #[test]
1075    fn lex_parentheses() {
1076        let kinds = lex_kinds("(x + 1)").unwrap();
1077        assert_eq!(
1078            &kinds[..5],
1079            &[
1080                TokenKind::LParen,
1081                TokenKind::Identifier,
1082                TokenKind::Plus,
1083                TokenKind::NumberLit,
1084                TokenKind::RParen,
1085            ]
1086        );
1087    }
1088
1089    #[test]
1090    fn lex_dot_for_references() {
1091        let kinds = lex_kinds("employee.salary").unwrap();
1092        assert_eq!(
1093            &kinds[..3],
1094            &[TokenKind::Identifier, TokenKind::Dot, TokenKind::Identifier]
1095        );
1096    }
1097
1098    #[test]
1099    fn lex_spec_name_with_slashes() {
1100        let tokens = lex_all("spec contracts/employment/jack").unwrap();
1101        assert_eq!(tokens[0].kind, TokenKind::Spec);
1102        // The lexer will see "contracts" as identifier, then "/" as Slash
1103        // The parser will handle assembling the spec name.
1104        assert_eq!(tokens[1].kind, TokenKind::Identifier);
1105    }
1106
1107    #[test]
1108    fn lex_number_not_followed_by_e_identifier() {
1109        // "42 eur" should be number then identifier, not scientific notation
1110        let tokens = lex_all("42 eur").unwrap();
1111        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1112        assert_eq!(tokens[0].text, "42");
1113        assert_eq!(tokens[1].kind, TokenKind::Identifier);
1114        assert_eq!(tokens[1].text, "eur");
1115    }
1116
1117    #[test]
1118    fn lex_unknown_character() {
1119        let result = lex_all("§");
1120        assert!(result.is_err());
1121    }
1122
1123    #[test]
1124    fn lex_peek_does_not_consume() {
1125        let mut lexer = Lexer::new("spec test", &crate::parsing::source::SourceType::Volatile);
1126        let peeked_kind = lexer.peek().unwrap().kind.clone();
1127        assert_eq!(peeked_kind, TokenKind::Spec);
1128        let next = lexer.next_token().unwrap();
1129        assert_eq!(next.kind, TokenKind::Spec);
1130    }
1131
1132    #[test]
1133    fn lex_span_byte_offsets() {
1134        let tokens = lex_all("spec test").unwrap();
1135        assert_eq!(tokens[0].span.start, 0);
1136        assert_eq!(tokens[0].span.end, 4);
1137        assert_eq!(tokens[0].span.line, 1);
1138        assert_eq!(tokens[0].span.col, 1);
1139
1140        assert_eq!(tokens[1].span.start, 5);
1141        assert_eq!(tokens[1].span.end, 9);
1142        assert_eq!(tokens[1].span.line, 1);
1143        assert_eq!(tokens[1].span.col, 6);
1144    }
1145
1146    #[test]
1147    fn lex_multiline_span_tracking() {
1148        let tokens = lex_all("spec test\ndata x: 1").unwrap();
1149        // "data" should be on line 2
1150        let data_token = &tokens[2]; // spec, test, data
1151        assert_eq!(data_token.kind, TokenKind::Data);
1152        assert_eq!(data_token.span.line, 2);
1153        assert_eq!(data_token.span.col, 1);
1154    }
1155
1156    #[test]
1157    fn lex_case_insensitive_keywords() {
1158        // Lemma keywords are case-insensitive
1159        let kinds = lex_kinds("SPEC Data RULE").unwrap();
1160        assert_eq!(kinds[0], TokenKind::Spec);
1161        assert_eq!(kinds[1], TokenKind::Data);
1162        assert_eq!(kinds[2], TokenKind::Rule);
1163    }
1164
1165    #[test]
1166    fn lex_math_function_keywords() {
1167        let kinds =
1168            lex_kinds("sqrt sin cos tan asin acos atan log exp abs floor ceil round").unwrap();
1169        assert_eq!(
1170            &kinds[..13],
1171            &[
1172                TokenKind::Sqrt,
1173                TokenKind::Sin,
1174                TokenKind::Cos,
1175                TokenKind::Tan,
1176                TokenKind::Asin,
1177                TokenKind::Acos,
1178                TokenKind::Atan,
1179                TokenKind::Log,
1180                TokenKind::Exp,
1181                TokenKind::Abs,
1182                TokenKind::Floor,
1183                TokenKind::Ceil,
1184                TokenKind::Round,
1185            ]
1186        );
1187    }
1188
1189    #[test]
1190    fn lex_range_bound_keywords() {
1191        let kinds = lex_kinds("lower upper LOWER").unwrap();
1192        assert_eq!(
1193            &kinds[..3],
1194            &[TokenKind::Lower, TokenKind::Upper, TokenKind::Lower,]
1195        );
1196    }
1197
1198    #[test]
1199    fn lex_is_keyword() {
1200        let kinds = lex_kinds("status is \"active\"").unwrap();
1201        assert_eq!(kinds[0], TokenKind::Identifier);
1202        assert_eq!(kinds[1], TokenKind::Is);
1203        assert_eq!(kinds[2], TokenKind::StringLit);
1204    }
1205
1206    #[test]
1207    fn lex_percent_not_followed_by_digit() {
1208        // "50%" should be number then percent
1209        let kinds = lex_kinds("50%").unwrap();
1210        assert_eq!(kinds[0], TokenKind::NumberLit);
1211        assert_eq!(kinds[1], TokenKind::Percent);
1212    }
1213
1214    #[test]
1215    fn lex_number_with_commas() {
1216        let tokens = lex_all("1,000,000").unwrap();
1217        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1218        assert_eq!(tokens[0].text, "1,000,000");
1219    }
1220
1221    #[test]
1222    fn lex_arrow_chain() {
1223        let kinds = lex_kinds("-> unit eur: 1.00 -> decimals 2").unwrap();
1224        assert_eq!(kinds[0], TokenKind::Arrow);
1225        assert_eq!(kinds[1], TokenKind::Identifier);
1226        assert_eq!(kinds[2], TokenKind::Identifier);
1227        assert_eq!(kinds[3], TokenKind::Colon);
1228        assert_eq!(kinds[4], TokenKind::NumberLit);
1229        assert_eq!(kinds[5], TokenKind::Arrow);
1230    }
1231}
lemma/parsing/lexer.rs

lemma/parsing/
lexer.rs
