lemma/parsing/
lexer.rs

1use crate::error::Error;
2use crate::parsing::ast::{BooleanValue, PrimitiveKind, Span};
3use crate::parsing::source::Source;
4use std::sync::Arc;
5
6#[derive(Debug, Clone, PartialEq, Eq)]
7pub enum TokenKind {
8    // Keywords
9    Spec,
10    Repo,
11    Data,
12    Rule,
13    Unless,
14    Then,
15    Not,
16    And,
17    In,
18    As,
19    Uses,
20    With,
21    Meta,
22    Veto,
23    Now,
24    Past,
25    Future,
26
27    // Boolean keywords
28    True,
29    False,
30    Yes,
31    No,
32
33    // Type keywords
34    MeasureKw,
35    NumberKw,
36    TextKw,
37    DateKw,
38    TimeKw,
39    BooleanKw,
40    RatioKw,
41
42    // Math function keywords
43    Sqrt,
44    Sin,
45    Cos,
46    Tan,
47    Asin,
48    Acos,
49    Atan,
50    Log,
51    Exp,
52    Abs,
53    Floor,
54    Ceil,
55    Round,
56
57    Permille,
58
59    // Comparison keyword operators
60    Is,
61
62    // Operators
63    Plus,
64    Minus,
65    Star,
66    Slash,
67    Comma,
68    Percent,
69    PercentPercent,
70    Caret,
71    Gt,
72    Lt,
73    Gte,
74    Lte,
75
76    // Punctuation
77    Colon,
78    Arrow,
79    Ellipsis,
80    Dot,
81    At,
82    LParen,
83    RParen,
84
85    // Literals
86    NumberLit,
87    StringLit,
88
89    // Commentary (raw text between """ delimiters)
90    Commentary,
91
92    // Identifiers
93    Identifier,
94
95    // End of file
96    Eof,
97}
98
99impl std::fmt::Display for TokenKind {
100    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
101        match self {
102            TokenKind::Spec => write!(f, "'spec'"),
103            TokenKind::Repo => write!(f, "'repo'"),
104            TokenKind::Data => write!(f, "'data'"),
105            TokenKind::Rule => write!(f, "'rule'"),
106            TokenKind::Unless => write!(f, "'unless'"),
107            TokenKind::Then => write!(f, "'then'"),
108            TokenKind::Not => write!(f, "'not'"),
109            TokenKind::And => write!(f, "'and'"),
110            TokenKind::In => write!(f, "'in'"),
111            TokenKind::As => write!(f, "'as'"),
112            TokenKind::Uses => write!(f, "'uses'"),
113            TokenKind::With => write!(f, "'with'"),
114            TokenKind::Meta => write!(f, "'meta'"),
115            TokenKind::Veto => write!(f, "'veto'"),
116            TokenKind::Now => write!(f, "'now'"),
117            TokenKind::Past => write!(f, "'past'"),
118            TokenKind::Future => write!(f, "'future'"),
119            TokenKind::True => write!(f, "'true'"),
120            TokenKind::False => write!(f, "'false'"),
121            TokenKind::Yes => write!(f, "'yes'"),
122            TokenKind::No => write!(f, "'no'"),
123            TokenKind::MeasureKw => write!(f, "'measure'"),
124            TokenKind::NumberKw => write!(f, "'number'"),
125            TokenKind::TextKw => write!(f, "'text'"),
126            TokenKind::DateKw => write!(f, "'date'"),
127            TokenKind::TimeKw => write!(f, "'time'"),
128            TokenKind::BooleanKw => write!(f, "'boolean'"),
129            TokenKind::RatioKw => write!(f, "'ratio'"),
130            TokenKind::Sqrt => write!(f, "'sqrt'"),
131            TokenKind::Sin => write!(f, "'sin'"),
132            TokenKind::Cos => write!(f, "'cos'"),
133            TokenKind::Tan => write!(f, "'tan'"),
134            TokenKind::Asin => write!(f, "'asin'"),
135            TokenKind::Acos => write!(f, "'acos'"),
136            TokenKind::Atan => write!(f, "'atan'"),
137            TokenKind::Log => write!(f, "'log'"),
138            TokenKind::Exp => write!(f, "'exp'"),
139            TokenKind::Abs => write!(f, "'abs'"),
140            TokenKind::Floor => write!(f, "'floor'"),
141            TokenKind::Ceil => write!(f, "'ceil'"),
142            TokenKind::Round => write!(f, "'round'"),
143            TokenKind::Permille => write!(f, "'permille'"),
144            TokenKind::Is => write!(f, "'is'"),
145            TokenKind::Plus => write!(f, "'+'"),
146            TokenKind::Minus => write!(f, "'-'"),
147            TokenKind::Star => write!(f, "'*'"),
148            TokenKind::Slash => write!(f, "'/'"),
149            TokenKind::Comma => write!(f, "','"),
150            TokenKind::Percent => write!(f, "'%'"),
151            TokenKind::PercentPercent => write!(f, "'%%'"),
152            TokenKind::Caret => write!(f, "'^'"),
153            TokenKind::Gt => write!(f, "'>'"),
154            TokenKind::Lt => write!(f, "'<'"),
155            TokenKind::Gte => write!(f, "'>='"),
156            TokenKind::Lte => write!(f, "'<='"),
157            TokenKind::Colon => write!(f, "':'"),
158            TokenKind::Arrow => write!(f, "'->'"),
159            TokenKind::Ellipsis => write!(f, "'...'"),
160            TokenKind::Dot => write!(f, "'.'"),
161            TokenKind::At => write!(f, "'@'"),
162            TokenKind::LParen => write!(f, "'('"),
163            TokenKind::RParen => write!(f, "')'"),
164            TokenKind::NumberLit => write!(f, "a number"),
165            TokenKind::StringLit => write!(f, "a string"),
166            TokenKind::Commentary => write!(f, "commentary block"),
167            TokenKind::Identifier => write!(f, "an identifier"),
168            TokenKind::Eof => write!(f, "end of file"),
169        }
170    }
171}
172
173#[derive(Debug, Clone)]
174pub struct Token {
175    pub kind: TokenKind,
176    pub span: Span,
177    pub text: String,
178}
179
180impl Token {
181    pub fn eof(offset: usize, line: usize, col: usize) -> Self {
182        Token {
183            kind: TokenKind::Eof,
184            span: Span {
185                start: offset,
186                end: offset,
187                line,
188                col,
189            },
190            text: String::new(),
191        }
192    }
193}
194
195#[derive(Clone)]
196pub struct LexerCheckpoint {
197    pos: usize,
198    line: usize,
199    col: usize,
200    byte_offset: usize,
201    peeked: Option<Token>,
202    peeked2: Option<Token>,
203}
204
205// todo: find out why derive Clone is necessary
206#[derive(Clone)]
207pub struct Lexer {
208    source: Vec<char>,
209    pos: usize,
210    line: usize,
211    col: usize,
212    byte_offset: usize,
213    source_type: crate::parsing::source::SourceType,
214    source_text: Arc<str>,
215    peeked: Option<Token>,
216    peeked2: Option<Token>,
217}
218
219impl Lexer {
220    pub fn new(input: &str, source_type: &crate::parsing::source::SourceType) -> Self {
221        let source_text: Arc<str> = Arc::from(input);
222        Lexer {
223            source: input.chars().collect(),
224            pos: 0,
225            line: 1,
226            col: 1,
227            byte_offset: 0,
228            source_type: source_type.clone(),
229            source_text,
230            peeked: None,
231            peeked2: None,
232        }
233    }
234
235    pub fn peek(&mut self) -> Result<&Token, Error> {
236        if self.peeked.is_none() {
237            let token = self.lex_token()?;
238            self.peeked = Some(token);
239        }
240        Ok(self.peeked.as_ref().expect("just assigned"))
241    }
242
243    pub fn peek_second(&mut self) -> Result<&Token, Error> {
244        self.peek()?;
245        if self.peeked2.is_none() {
246            let token = self.lex_token()?;
247            self.peeked2 = Some(token);
248        }
249        Ok(self.peeked2.as_ref().expect("just assigned"))
250    }
251
252    pub fn next_token(&mut self) -> Result<Token, Error> {
253        if let Some(token) = self.peeked.take() {
254            self.peeked = self.peeked2.take();
255            return Ok(token);
256        }
257        self.lex_token()
258    }
259
260    /// Saved lexer position for speculative parsing.
261    pub fn checkpoint(&self) -> LexerCheckpoint {
262        LexerCheckpoint {
263            pos: self.pos,
264            line: self.line,
265            col: self.col,
266            byte_offset: self.byte_offset,
267            peeked: self.peeked.clone(),
268            peeked2: self.peeked2.clone(),
269        }
270    }
271
272    pub fn restore(&mut self, checkpoint: LexerCheckpoint) {
273        self.pos = checkpoint.pos;
274        self.line = checkpoint.line;
275        self.col = checkpoint.col;
276        self.byte_offset = checkpoint.byte_offset;
277        self.peeked = checkpoint.peeked;
278        self.peeked2 = checkpoint.peeked2;
279    }
280
281    fn current_char(&self) -> Option<char> {
282        self.source.get(self.pos).copied()
283    }
284
285    fn peek_char(&self) -> Option<char> {
286        self.source.get(self.pos + 1).copied()
287    }
288
289    fn peek_char_at(&self, offset: usize) -> Option<char> {
290        self.source.get(self.pos + offset).copied()
291    }
292
293    fn advance(&mut self) {
294        if let Some(ch) = self.current_char() {
295            self.byte_offset += ch.len_utf8();
296            if ch == '\n' {
297                self.line += 1;
298                self.col = 1;
299            } else {
300                self.col += 1;
301            }
302            self.pos += 1;
303        }
304    }
305
306    fn skip_whitespace(&mut self) {
307        while let Some(ch) = self.current_char() {
308            if ch.is_whitespace() {
309                self.advance();
310            } else {
311                break;
312            }
313        }
314    }
315
316    fn make_span(&self, start_byte: usize, start_line: usize, start_col: usize) -> Span {
317        Span {
318            start: start_byte,
319            end: self.byte_offset,
320            line: start_line,
321            col: start_col,
322        }
323    }
324
325    fn make_error(&self, message: impl Into<String>, span: Span) -> Error {
326        Error::parsing(
327            message,
328            Source::new(self.source_type.clone(), span),
329            None::<String>,
330        )
331    }
332
333    fn lex_token(&mut self) -> Result<Token, Error> {
334        self.skip_whitespace();
335
336        let start_byte = self.byte_offset;
337        let start_line = self.line;
338        let start_col = self.col;
339
340        let Some(ch) = self.current_char() else {
341            return Ok(Token::eof(start_byte, start_line, start_col));
342        };
343
344        // Triple-quote commentary
345        if ch == '"' && self.peek_char() == Some('"') && self.peek_char_at(2) == Some('"') {
346            return self.scan_triple_quote(start_byte, start_line, start_col);
347        }
348
349        // String literal
350        if ch == '"' {
351            return self.scan_string(start_byte, start_line, start_col);
352        }
353
354        // Number literal (sign handled by parser, not lexer)
355        if ch.is_ascii_digit() {
356            return self.scan_number(start_byte, start_line, start_col);
357        }
358
359        // Two-character operators (check before single-char)
360        if let Some(token) = self.try_two_char_operator(start_byte, start_line, start_col) {
361            return Ok(token);
362        }
363
364        // Three-character ellipsis
365        if ch == '.' && self.peek_char() == Some('.') && self.peek_char_at(2) == Some('.') {
366            self.advance();
367            self.advance();
368            self.advance();
369            let span = self.make_span(start_byte, start_line, start_col);
370            return Ok(Token {
371                kind: TokenKind::Ellipsis,
372                span,
373                text: "...".to_string(),
374            });
375        }
376
377        // Single-character operators/punctuation
378        if let Some(kind) = self.single_char_token(ch) {
379            self.advance();
380            let span = self.make_span(start_byte, start_line, start_col);
381            let text = ch.to_string();
382            return Ok(Token { kind, span, text });
383        }
384
385        // Identifier or keyword (starts with letter or @)
386        if ch.is_ascii_alphabetic() || ch == '_' {
387            return Ok(self.scan_identifier(start_byte, start_line, start_col));
388        }
389
390        // @ prefix for registry references
391        if ch == '@' {
392            self.advance();
393            let span = self.make_span(start_byte, start_line, start_col);
394            return Ok(Token {
395                kind: TokenKind::At,
396                span,
397                text: "@".to_string(),
398            });
399        }
400
401        // Unknown character
402        self.advance();
403        let span = self.make_span(start_byte, start_line, start_col);
404        Err(self.make_error(format!("Unexpected character '{}'", ch), span))
405    }
406
407    fn scan_triple_quote(
408        &mut self,
409        start_byte: usize,
410        start_line: usize,
411        start_col: usize,
412    ) -> Result<Token, Error> {
413        self.advance(); // "
414        self.advance(); // "
415        self.advance(); // "
416
417        let content_start = self.byte_offset;
418        loop {
419            match self.current_char() {
420                None => {
421                    let span = self.make_span(start_byte, start_line, start_col);
422                    return Err(self.make_error(
423                        "Unterminated commentary block: expected closing \"\"\"",
424                        span,
425                    ));
426                }
427                Some('"')
428                    if self.source.get(self.pos + 1) == Some(&'"')
429                        && self.source.get(self.pos + 2) == Some(&'"') =>
430                {
431                    let content_end = self.byte_offset;
432                    self.advance(); // "
433                    self.advance(); // "
434                    self.advance(); // "
435                    let raw: String = self.source_text[content_start..content_end].to_string();
436                    let span = self.make_span(start_byte, start_line, start_col);
437                    return Ok(Token {
438                        kind: TokenKind::Commentary,
439                        span,
440                        text: raw,
441                    });
442                }
443                Some(_) => {
444                    self.advance();
445                }
446            }
447        }
448    }
449
450    fn scan_string(
451        &mut self,
452        start_byte: usize,
453        start_line: usize,
454        start_col: usize,
455    ) -> Result<Token, Error> {
456        self.advance(); // consume opening "
457        let mut content = String::new();
458        loop {
459            match self.current_char() {
460                None => {
461                    let span = self.make_span(start_byte, start_line, start_col);
462                    return Err(self.make_error("String starting here was never closed", span));
463                }
464                Some('"') => {
465                    self.advance(); // consume closing "
466                    break;
467                }
468                Some(ch) => {
469                    content.push(ch);
470                    self.advance();
471                }
472            }
473        }
474        let span = self.make_span(start_byte, start_line, start_col);
475        if content.len() > crate::limits::MAX_TEXT_VALUE_LENGTH {
476            return Err(self.make_error(
477                format!(
478                    "Text literal exceeds maximum length of {} characters (found {})",
479                    crate::limits::MAX_TEXT_VALUE_LENGTH,
480                    content.len()
481                ),
482                span,
483            ));
484        }
485        // Store the full text including quotes for span accuracy,
486        // but content without quotes for the parser to use.
487        let full_text = format!("\"{}\"", content);
488        Ok(Token {
489            kind: TokenKind::StringLit,
490            span,
491            text: full_text,
492        })
493    }
494
495    fn scan_number(
496        &mut self,
497        start_byte: usize,
498        start_line: usize,
499        start_col: usize,
500    ) -> Result<Token, Error> {
501        let mut text = String::new();
502
503        // Integer part: digits with optional _ or , separators
504        while let Some(ch) = self.current_char() {
505            if ch.is_ascii_digit() || ch == '_' || ch == ',' {
506                text.push(ch);
507                self.advance();
508            } else {
509                break;
510            }
511        }
512
513        // Decimal part
514        if self.current_char() == Some('.') {
515            // Check if next char after dot is a digit (not a method call or dotted reference)
516            if let Some(next) = self.peek_char() {
517                if next.is_ascii_digit() {
518                    text.push('.');
519                    self.advance(); // consume .
520                    while let Some(ch) = self.current_char() {
521                        if ch.is_ascii_digit() {
522                            text.push(ch);
523                            self.advance();
524                        } else {
525                            break;
526                        }
527                    }
528                }
529            }
530        }
531
532        // Scientific notation: e or E followed by optional +/- and digits
533        if let Some(ch) = self.current_char() {
534            if ch == 'e' || ch == 'E' {
535                let mut sci_text = String::new();
536                sci_text.push(ch);
537                let save_pos = self.pos;
538                let save_byte = self.byte_offset;
539                let save_line = self.line;
540                let save_col = self.col;
541                self.advance(); // consume e/E
542
543                if let Some(sign) = self.current_char() {
544                    if sign == '+' || sign == '-' {
545                        sci_text.push(sign);
546                        self.advance();
547                    }
548                }
549
550                if let Some(d) = self.current_char() {
551                    if d.is_ascii_digit() {
552                        while let Some(ch) = self.current_char() {
553                            if ch.is_ascii_digit() {
554                                sci_text.push(ch);
555                                self.advance();
556                            } else {
557                                break;
558                            }
559                        }
560                        text.push_str(&sci_text);
561                    } else {
562                        // Not actually scientific notation, backtrack
563                        self.pos = save_pos;
564                        self.byte_offset = save_byte;
565                        self.line = save_line;
566                        self.col = save_col;
567                    }
568                } else {
569                    self.pos = save_pos;
570                    self.byte_offset = save_byte;
571                    self.line = save_line;
572                    self.col = save_col;
573                }
574            }
575        }
576
577        let span = self.make_span(start_byte, start_line, start_col);
578        Ok(Token {
579            kind: TokenKind::NumberLit,
580            span,
581            text,
582        })
583    }
584
585    fn try_two_char_operator(
586        &mut self,
587        start_byte: usize,
588        start_line: usize,
589        start_col: usize,
590    ) -> Option<Token> {
591        let ch = self.current_char()?;
592        let next = self.peek_char();
593
594        let kind = match (ch, next) {
595            ('-', Some('>')) => TokenKind::Arrow,
596            ('>', Some('=')) => TokenKind::Gte,
597            ('<', Some('=')) => TokenKind::Lte,
598            ('%', Some('%')) => {
599                // Check that it's not followed by a digit (invalid permille like 10%%5)
600                TokenKind::PercentPercent
601            }
602            _ => return None,
603        };
604
605        self.advance();
606        self.advance();
607        let span = self.make_span(start_byte, start_line, start_col);
608        let text: String = self.source_text[span.start..span.end].to_string();
609        Some(Token { kind, span, text })
610    }
611
612    fn single_char_token(&self, ch: char) -> Option<TokenKind> {
613        match ch {
614            '+' => Some(TokenKind::Plus),
615            '*' => Some(TokenKind::Star),
616            '/' => Some(TokenKind::Slash),
617            ',' => Some(TokenKind::Comma),
618            '^' => Some(TokenKind::Caret),
619            ':' => Some(TokenKind::Colon),
620            '.' => Some(TokenKind::Dot),
621            '(' => Some(TokenKind::LParen),
622            ')' => Some(TokenKind::RParen),
623            '>' => Some(TokenKind::Gt),
624            '<' => Some(TokenKind::Lt),
625            '%' => Some(TokenKind::Percent),
626            '-' => Some(TokenKind::Minus),
627            _ => None,
628        }
629    }
630
631    fn scan_identifier(&mut self, start_byte: usize, start_line: usize, start_col: usize) -> Token {
632        let mut text = String::new();
633        while let Some(ch) = self.current_char() {
634            if ch.is_ascii_alphanumeric() || ch == '_' {
635                text.push(ch);
636                self.advance();
637            } else {
638                break;
639            }
640        }
641
642        let kind = keyword_from_identifier(&text);
643        let span = self.make_span(start_byte, start_line, start_col);
644        Token { kind, span, text }
645    }
646}
647
648fn keyword_from_identifier(text: &str) -> TokenKind {
649    match text.to_lowercase().as_str() {
650        "spec" => TokenKind::Spec,
651        "repo" => TokenKind::Repo,
652        "data" => TokenKind::Data,
653        "rule" => TokenKind::Rule,
654        "unless" => TokenKind::Unless,
655        "then" => TokenKind::Then,
656        "not" => TokenKind::Not,
657        "and" => TokenKind::And,
658        "in" => TokenKind::In,
659        "as" => TokenKind::As,
660        "uses" => TokenKind::Uses,
661        "with" => TokenKind::With,
662        "meta" => TokenKind::Meta,
663        "veto" => TokenKind::Veto,
664        "now" => TokenKind::Now,
665        "past" => TokenKind::Past,
666        "future" => TokenKind::Future,
667        "true" => TokenKind::True,
668        "false" => TokenKind::False,
669        "yes" => TokenKind::Yes,
670        "no" => TokenKind::No,
671        "measure" => TokenKind::MeasureKw,
672        "number" => TokenKind::NumberKw,
673        "text" => TokenKind::TextKw,
674        "date" => TokenKind::DateKw,
675        "time" => TokenKind::TimeKw,
676        "boolean" => TokenKind::BooleanKw,
677        "ratio" => TokenKind::RatioKw,
678        "sqrt" => TokenKind::Sqrt,
679        "sin" => TokenKind::Sin,
680        "cos" => TokenKind::Cos,
681        "tan" => TokenKind::Tan,
682        "asin" => TokenKind::Asin,
683        "acos" => TokenKind::Acos,
684        "atan" => TokenKind::Atan,
685        "log" => TokenKind::Log,
686        "exp" => TokenKind::Exp,
687        "abs" => TokenKind::Abs,
688        "floor" => TokenKind::Floor,
689        "ceil" => TokenKind::Ceil,
690        "round" => TokenKind::Round,
691        "is" => TokenKind::Is,
692        "permille" => TokenKind::Permille,
693        _ => TokenKind::Identifier,
694    }
695}
696
697/// Structural keywords can never be used as identifiers (data/rule names).
698/// Type keywords (measure, number, text, date, time, boolean, ratio)
699/// are reserved and cannot be used as names.
700pub fn is_keyword(kind: &TokenKind) -> bool {
701    matches!(
702        kind,
703        TokenKind::Spec
704            | TokenKind::Repo
705            | TokenKind::Data
706            | TokenKind::Rule
707            | TokenKind::Unless
708            | TokenKind::Then
709            | TokenKind::Not
710            | TokenKind::And
711            | TokenKind::In
712            | TokenKind::As
713            | TokenKind::Uses
714            | TokenKind::With
715            | TokenKind::Meta
716            | TokenKind::Veto
717            | TokenKind::Now
718            | TokenKind::Sqrt
719            | TokenKind::Sin
720            | TokenKind::Cos
721            | TokenKind::Tan
722            | TokenKind::Asin
723            | TokenKind::Acos
724            | TokenKind::Atan
725            | TokenKind::Log
726            | TokenKind::Exp
727            | TokenKind::Abs
728            | TokenKind::Floor
729            | TokenKind::Ceil
730            | TokenKind::Round
731            | TokenKind::True
732            | TokenKind::False
733            | TokenKind::Yes
734            | TokenKind::No
735            | TokenKind::MeasureKw
736            | TokenKind::NumberKw
737            | TokenKind::TextKw
738            | TokenKind::DateKw
739            | TokenKind::TimeKw
740            | TokenKind::BooleanKw
741            | TokenKind::RatioKw
742    )
743}
744
745/// Map type keyword token to PrimitiveKind. Single source of truth for type keywords.
746#[must_use]
747pub fn token_kind_to_primitive(kind: &TokenKind) -> Option<PrimitiveKind> {
748    match kind {
749        TokenKind::BooleanKw => Some(PrimitiveKind::Boolean),
750        TokenKind::MeasureKw => Some(PrimitiveKind::Measure),
751        TokenKind::NumberKw => Some(PrimitiveKind::Number),
752        TokenKind::RatioKw => Some(PrimitiveKind::Ratio),
753        TokenKind::TextKw => Some(PrimitiveKind::Text),
754        TokenKind::DateKw => Some(PrimitiveKind::Date),
755        TokenKind::TimeKw => Some(PrimitiveKind::Time),
756        _ => None,
757    }
758}
759
760/// Returns true if the token kind represents a boolean literal keyword.
761pub fn is_boolean_keyword(kind: &TokenKind) -> bool {
762    matches!(
763        kind,
764        TokenKind::True | TokenKind::False | TokenKind::Yes | TokenKind::No
765    )
766}
767
768/// Maps a boolean-keyword token kind to BooleanValue. Call only when `is_boolean_keyword(kind)`.
769#[must_use]
770pub fn token_kind_to_boolean_value(kind: &TokenKind) -> BooleanValue {
771    match kind {
772        TokenKind::True => BooleanValue::True,
773        TokenKind::False => BooleanValue::False,
774        TokenKind::Yes => BooleanValue::Yes,
775        TokenKind::No => BooleanValue::No,
776        _ => unreachable!(
777            "BUG: token_kind_to_boolean_value called with non-boolean token {:?}",
778            kind
779        ),
780    }
781}
782
783/// Returns true if the token kind represents a math function keyword.
784pub fn is_math_function(kind: &TokenKind) -> bool {
785    matches!(
786        kind,
787        TokenKind::Sqrt
788            | TokenKind::Sin
789            | TokenKind::Cos
790            | TokenKind::Tan
791            | TokenKind::Asin
792            | TokenKind::Acos
793            | TokenKind::Atan
794            | TokenKind::Log
795            | TokenKind::Exp
796            | TokenKind::Abs
797            | TokenKind::Floor
798            | TokenKind::Ceil
799            | TokenKind::Round
800    )
801}
802
803/// Returns true if the token kind can start the body of a spec
804/// (data, rule, or meta definition).
805pub fn is_spec_body_keyword(kind: &TokenKind) -> bool {
806    matches!(
807        kind,
808        TokenKind::Data | TokenKind::With | TokenKind::Rule | TokenKind::Meta
809    )
810}
811
812/// Returns true if the token kind can be used as a label or reference segment
813/// (identifier, or non-reserved contextual keyword such as `past` / `future` /
814/// `permille` / `is`).
815pub fn can_be_label(kind: &TokenKind) -> bool {
816    matches!(
817        kind,
818        TokenKind::Identifier
819            | TokenKind::Past
820            | TokenKind::Future
821            | TokenKind::Permille
822            | TokenKind::Is
823    )
824}
825
826/// `calendar` in `in calendar month` / `past calendar year` — not a type keyword.
827#[must_use]
828pub fn token_is_calendar_period_marker(tok: &Token) -> bool {
829    tok.kind == TokenKind::Identifier && tok.text == "calendar"
830}
831
832/// Slash-/dot-separated registry path segments (`@org/repo/...`). Keywords that are
833/// reserved at the structural level (`spec`, `rule`, etc.) are allowed inside
834/// multi-segment paths (e.g. `@org/repo`) but callers must reject them when they
835/// appear as the entire stand-alone name (e.g. `repo spec`).
836#[must_use]
837pub fn can_be_repository_qualifier_segment(kind: &TokenKind) -> bool {
838    matches!(kind, TokenKind::Identifier)
839        || is_keyword(kind)
840        || can_be_label(kind)
841        || is_boolean_keyword(kind)
842        || is_math_function(kind)
843}
844
845#[cfg(test)]
846mod tests {
847    use super::*;
848
849    fn lex_all(input: &str) -> Result<Vec<Token>, Error> {
850        let mut lexer = Lexer::new(input, &crate::parsing::source::SourceType::Volatile);
851        let mut tokens = Vec::new();
852        loop {
853            let token = lexer.next_token()?;
854            if token.kind == TokenKind::Eof {
855                tokens.push(token);
856                break;
857            }
858            tokens.push(token);
859        }
860        Ok(tokens)
861    }
862
863    fn lex_kinds(input: &str) -> Result<Vec<TokenKind>, Error> {
864        Ok(lex_all(input)?.into_iter().map(|t| t.kind).collect())
865    }
866
867    #[test]
868    fn lex_empty_input() {
869        let tokens = lex_all("").unwrap();
870        assert_eq!(tokens.len(), 1);
871        assert_eq!(tokens[0].kind, TokenKind::Eof);
872    }
873
874    #[test]
875    fn string_literal_at_max_length_is_accepted() {
876        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH);
877        let tokens = lex_all(&format!("\"{content}\"")).unwrap();
878        assert_eq!(tokens[0].kind, TokenKind::StringLit);
879    }
880
881    #[test]
882    fn string_literal_over_max_length_is_parse_error() {
883        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH + 1);
884        let err = lex_all(&format!("\"{content}\"")).unwrap_err();
885        assert!(
886            err.message().contains("maximum length"),
887            "expected length error, got: {err}"
888        );
889        assert!(err.location().is_some(), "parse error must carry a source");
890    }
891
892    #[test]
893    fn number_literal_with_separators_lexes() {
894        let tokens = lex_all("9,999,999,999,999,999,999,999,999,999").unwrap();
895        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
896    }
897
898    #[test]
899    fn lex_spec_declaration() {
900        let kinds = lex_kinds("spec person").unwrap();
901        assert_eq!(
902            kinds,
903            vec![TokenKind::Spec, TokenKind::Identifier, TokenKind::Eof]
904        );
905    }
906
907    #[test]
908    fn lex_data_definition() {
909        let kinds = lex_kinds("data age: 25").unwrap();
910        assert_eq!(
911            kinds,
912            vec![
913                TokenKind::Data,
914                TokenKind::Identifier,
915                TokenKind::Colon,
916                TokenKind::NumberLit,
917                TokenKind::Eof,
918            ]
919        );
920    }
921
922    #[test]
923    fn lex_rule_with_comparison() {
924        let kinds = lex_kinds("rule is_adult: age >= 18").unwrap();
925        assert_eq!(
926            kinds,
927            vec![
928                TokenKind::Rule,
929                TokenKind::Identifier,
930                TokenKind::Colon,
931                TokenKind::Identifier,
932                TokenKind::Gte,
933                TokenKind::NumberLit,
934                TokenKind::Eof,
935            ]
936        );
937    }
938
939    #[test]
940    fn lex_string_literal() {
941        let tokens = lex_all(r#""hello world""#).unwrap();
942        assert_eq!(tokens[0].kind, TokenKind::StringLit);
943        assert_eq!(tokens[0].text, "\"hello world\"");
944    }
945
946    #[test]
947    fn lex_unterminated_string() {
948        let result = lex_all(r#""hello"#);
949        assert!(result.is_err());
950    }
951
952    #[test]
953    fn lex_number_with_decimal() {
954        let tokens = lex_all("3.14").unwrap();
955        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
956        assert_eq!(tokens[0].text, "3.14");
957    }
958
959    #[test]
960    fn lex_number_with_underscores() {
961        let tokens = lex_all("1_000_000").unwrap();
962        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
963        assert_eq!(tokens[0].text, "1_000_000");
964    }
965
966    #[test]
967    fn lex_scientific_notation() {
968        let tokens = lex_all("1.5e+10").unwrap();
969        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
970        assert_eq!(tokens[0].text, "1.5e+10");
971    }
972
973    #[test]
974    fn lex_all_operators() {
975        let kinds = lex_kinds("+ - * / % ^ > < >= <= -> %%").unwrap();
976        assert_eq!(
977            &kinds[..12],
978            &[
979                TokenKind::Plus,
980                TokenKind::Minus,
981                TokenKind::Star,
982                TokenKind::Slash,
983                TokenKind::Percent,
984                TokenKind::Caret,
985                TokenKind::Gt,
986                TokenKind::Lt,
987                TokenKind::Gte,
988                TokenKind::Lte,
989                TokenKind::Arrow,
990                TokenKind::PercentPercent,
991            ]
992        );
993    }
994
995    #[test]
996    fn lex_keywords() {
997        let kinds =
998            lex_kinds("spec data rule unless then not and in as uses meta veto now").unwrap();
999        assert_eq!(
1000            &kinds[..13],
1001            &[
1002                TokenKind::Spec,
1003                TokenKind::Data,
1004                TokenKind::Rule,
1005                TokenKind::Unless,
1006                TokenKind::Then,
1007                TokenKind::Not,
1008                TokenKind::And,
1009                TokenKind::In,
1010                TokenKind::As,
1011                TokenKind::Uses,
1012                TokenKind::Meta,
1013                TokenKind::Veto,
1014                TokenKind::Now,
1015            ]
1016        );
1017    }
1018
1019    #[test]
1020    fn lex_boolean_keywords() {
1021        let kinds = lex_kinds("true false yes no").unwrap();
1022        assert_eq!(
1023            &kinds[..4],
1024            &[
1025                TokenKind::True,
1026                TokenKind::False,
1027                TokenKind::Yes,
1028                TokenKind::No,
1029            ]
1030        );
1031    }
1032
1033    #[test]
1034    fn lex_duration_keywords() {
1035        let kinds = lex_kinds("year month week day hour minute second").unwrap();
1036        assert_eq!(
1037            &kinds[..7],
1038            &[
1039                TokenKind::Identifier,
1040                TokenKind::Identifier,
1041                TokenKind::Identifier,
1042                TokenKind::Identifier,
1043                TokenKind::Identifier,
1044                TokenKind::Identifier,
1045                TokenKind::Identifier,
1046            ]
1047        );
1048    }
1049
1050    #[test]
1051    fn lex_commentary() {
1052        let tokens = lex_all(r#""""hello world""""#).unwrap();
1053        assert_eq!(tokens[0].kind, TokenKind::Commentary);
1054        assert_eq!(tokens[0].text, "hello world");
1055    }
1056
1057    #[test]
1058    fn lex_at_sign() {
1059        let kinds = lex_kinds("@user").unwrap();
1060        assert_eq!(kinds[0], TokenKind::At);
1061        assert_eq!(kinds[1], TokenKind::Identifier);
1062    }
1063
1064    #[test]
1065    fn lex_parentheses() {
1066        let kinds = lex_kinds("(x + 1)").unwrap();
1067        assert_eq!(
1068            &kinds[..5],
1069            &[
1070                TokenKind::LParen,
1071                TokenKind::Identifier,
1072                TokenKind::Plus,
1073                TokenKind::NumberLit,
1074                TokenKind::RParen,
1075            ]
1076        );
1077    }
1078
1079    #[test]
1080    fn lex_dot_for_references() {
1081        let kinds = lex_kinds("employee.salary").unwrap();
1082        assert_eq!(
1083            &kinds[..3],
1084            &[TokenKind::Identifier, TokenKind::Dot, TokenKind::Identifier]
1085        );
1086    }
1087
1088    #[test]
1089    fn lex_spec_name_with_slashes() {
1090        let tokens = lex_all("spec contracts/employment/jack").unwrap();
1091        assert_eq!(tokens[0].kind, TokenKind::Spec);
1092        // The lexer will see "contracts" as identifier, then "/" as Slash
1093        // The parser will handle assembling the spec name.
1094        assert_eq!(tokens[1].kind, TokenKind::Identifier);
1095    }
1096
1097    #[test]
1098    fn lex_number_not_followed_by_e_identifier() {
1099        // "42 eur" should be number then identifier, not scientific notation
1100        let tokens = lex_all("42 eur").unwrap();
1101        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1102        assert_eq!(tokens[0].text, "42");
1103        assert_eq!(tokens[1].kind, TokenKind::Identifier);
1104        assert_eq!(tokens[1].text, "eur");
1105    }
1106
1107    #[test]
1108    fn lex_unknown_character() {
1109        let result = lex_all("§");
1110        assert!(result.is_err());
1111    }
1112
1113    #[test]
1114    fn lex_peek_does_not_consume() {
1115        let mut lexer = Lexer::new("spec test", &crate::parsing::source::SourceType::Volatile);
1116        let peeked_kind = lexer.peek().unwrap().kind.clone();
1117        assert_eq!(peeked_kind, TokenKind::Spec);
1118        let next = lexer.next_token().unwrap();
1119        assert_eq!(next.kind, TokenKind::Spec);
1120    }
1121
1122    #[test]
1123    fn lex_span_byte_offsets() {
1124        let tokens = lex_all("spec test").unwrap();
1125        assert_eq!(tokens[0].span.start, 0);
1126        assert_eq!(tokens[0].span.end, 4);
1127        assert_eq!(tokens[0].span.line, 1);
1128        assert_eq!(tokens[0].span.col, 1);
1129
1130        assert_eq!(tokens[1].span.start, 5);
1131        assert_eq!(tokens[1].span.end, 9);
1132        assert_eq!(tokens[1].span.line, 1);
1133        assert_eq!(tokens[1].span.col, 6);
1134    }
1135
1136    #[test]
1137    fn lex_multiline_span_tracking() {
1138        let tokens = lex_all("spec test\ndata x: 1").unwrap();
1139        // "data" should be on line 2
1140        let data_token = &tokens[2]; // spec, test, data
1141        assert_eq!(data_token.kind, TokenKind::Data);
1142        assert_eq!(data_token.span.line, 2);
1143        assert_eq!(data_token.span.col, 1);
1144    }
1145
1146    #[test]
1147    fn lex_case_insensitive_keywords() {
1148        // Lemma keywords are case-insensitive
1149        let kinds = lex_kinds("SPEC Data RULE").unwrap();
1150        assert_eq!(kinds[0], TokenKind::Spec);
1151        assert_eq!(kinds[1], TokenKind::Data);
1152        assert_eq!(kinds[2], TokenKind::Rule);
1153    }
1154
1155    #[test]
1156    fn lex_math_function_keywords() {
1157        let kinds =
1158            lex_kinds("sqrt sin cos tan asin acos atan log exp abs floor ceil round").unwrap();
1159        assert_eq!(
1160            &kinds[..13],
1161            &[
1162                TokenKind::Sqrt,
1163                TokenKind::Sin,
1164                TokenKind::Cos,
1165                TokenKind::Tan,
1166                TokenKind::Asin,
1167                TokenKind::Acos,
1168                TokenKind::Atan,
1169                TokenKind::Log,
1170                TokenKind::Exp,
1171                TokenKind::Abs,
1172                TokenKind::Floor,
1173                TokenKind::Ceil,
1174                TokenKind::Round,
1175            ]
1176        );
1177    }
1178
1179    #[test]
1180    fn lex_is_keyword() {
1181        let kinds = lex_kinds("status is \"active\"").unwrap();
1182        assert_eq!(kinds[0], TokenKind::Identifier);
1183        assert_eq!(kinds[1], TokenKind::Is);
1184        assert_eq!(kinds[2], TokenKind::StringLit);
1185    }
1186
1187    #[test]
1188    fn lex_percent_not_followed_by_digit() {
1189        // "50%" should be number then percent
1190        let kinds = lex_kinds("50%").unwrap();
1191        assert_eq!(kinds[0], TokenKind::NumberLit);
1192        assert_eq!(kinds[1], TokenKind::Percent);
1193    }
1194
1195    #[test]
1196    fn lex_number_with_commas() {
1197        let tokens = lex_all("1,000,000").unwrap();
1198        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1199        assert_eq!(tokens[0].text, "1,000,000");
1200    }
1201
1202    #[test]
1203    fn lex_arrow_chain() {
1204        let kinds = lex_kinds("-> unit eur 1.00 -> decimals 2").unwrap();
1205        assert_eq!(kinds[0], TokenKind::Arrow);
1206        assert_eq!(kinds[1], TokenKind::Identifier);
1207        assert_eq!(kinds[2], TokenKind::Identifier);
1208        assert_eq!(kinds[3], TokenKind::NumberLit);
1209        assert_eq!(kinds[4], TokenKind::Arrow);
1210    }
1211}
lemma/parsing/lexer.rs

lemma/parsing/
lexer.rs
