lemma/parsing/
lexer.rs

1use crate::error::Error;
2use crate::parsing::ast::{BooleanValue, PrimitiveKind, Span};
3use crate::parsing::source::Source;
4use std::sync::Arc;
5
6#[derive(Debug, Clone, PartialEq, Eq)]
7pub enum TokenKind {
8    // Keywords
9    Spec,
10    Repo,
11    Data,
12    Rule,
13    Unless,
14    Then,
15    Not,
16    And,
17    In,
18    As,
19    Uses,
20    With,
21    Meta,
22    Veto,
23    Now,
24    Past,
25    Future,
26
27    // Boolean keywords
28    True,
29    False,
30    Yes,
31    No,
32    Accept,
33    Reject,
34
35    // Type keywords
36    MeasureKw,
37    NumberKw,
38    TextKw,
39    DateKw,
40    TimeKw,
41    BooleanKw,
42    RatioKw,
43
44    // Math function keywords
45    Sqrt,
46    Sin,
47    Cos,
48    Tan,
49    Asin,
50    Acos,
51    Atan,
52    Log,
53    Exp,
54    Abs,
55    Floor,
56    Ceil,
57    Round,
58
59    Permille,
60
61    // Comparison keyword operators
62    Is,
63
64    // Operators
65    Plus,
66    Minus,
67    Star,
68    Slash,
69    Comma,
70    Percent,
71    PercentPercent,
72    Caret,
73    Gt,
74    Lt,
75    Gte,
76    Lte,
77
78    // Punctuation
79    Colon,
80    Arrow,
81    Ellipsis,
82    Dot,
83    At,
84    LParen,
85    RParen,
86
87    // Literals
88    NumberLit,
89    StringLit,
90
91    // Commentary (raw text between """ delimiters)
92    Commentary,
93
94    // Identifiers
95    Identifier,
96
97    // End of file
98    Eof,
99}
100
101impl std::fmt::Display for TokenKind {
102    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103        match self {
104            TokenKind::Spec => write!(f, "'spec'"),
105            TokenKind::Repo => write!(f, "'repo'"),
106            TokenKind::Data => write!(f, "'data'"),
107            TokenKind::Rule => write!(f, "'rule'"),
108            TokenKind::Unless => write!(f, "'unless'"),
109            TokenKind::Then => write!(f, "'then'"),
110            TokenKind::Not => write!(f, "'not'"),
111            TokenKind::And => write!(f, "'and'"),
112            TokenKind::In => write!(f, "'in'"),
113            TokenKind::As => write!(f, "'as'"),
114            TokenKind::Uses => write!(f, "'uses'"),
115            TokenKind::With => write!(f, "'with'"),
116            TokenKind::Meta => write!(f, "'meta'"),
117            TokenKind::Veto => write!(f, "'veto'"),
118            TokenKind::Now => write!(f, "'now'"),
119            TokenKind::Past => write!(f, "'past'"),
120            TokenKind::Future => write!(f, "'future'"),
121            TokenKind::True => write!(f, "'true'"),
122            TokenKind::False => write!(f, "'false'"),
123            TokenKind::Yes => write!(f, "'yes'"),
124            TokenKind::No => write!(f, "'no'"),
125            TokenKind::Accept => write!(f, "'accept'"),
126            TokenKind::Reject => write!(f, "'reject'"),
127            TokenKind::MeasureKw => write!(f, "'measure'"),
128            TokenKind::NumberKw => write!(f, "'number'"),
129            TokenKind::TextKw => write!(f, "'text'"),
130            TokenKind::DateKw => write!(f, "'date'"),
131            TokenKind::TimeKw => write!(f, "'time'"),
132            TokenKind::BooleanKw => write!(f, "'boolean'"),
133            TokenKind::RatioKw => write!(f, "'ratio'"),
134            TokenKind::Sqrt => write!(f, "'sqrt'"),
135            TokenKind::Sin => write!(f, "'sin'"),
136            TokenKind::Cos => write!(f, "'cos'"),
137            TokenKind::Tan => write!(f, "'tan'"),
138            TokenKind::Asin => write!(f, "'asin'"),
139            TokenKind::Acos => write!(f, "'acos'"),
140            TokenKind::Atan => write!(f, "'atan'"),
141            TokenKind::Log => write!(f, "'log'"),
142            TokenKind::Exp => write!(f, "'exp'"),
143            TokenKind::Abs => write!(f, "'abs'"),
144            TokenKind::Floor => write!(f, "'floor'"),
145            TokenKind::Ceil => write!(f, "'ceil'"),
146            TokenKind::Round => write!(f, "'round'"),
147            TokenKind::Permille => write!(f, "'permille'"),
148            TokenKind::Is => write!(f, "'is'"),
149            TokenKind::Plus => write!(f, "'+'"),
150            TokenKind::Minus => write!(f, "'-'"),
151            TokenKind::Star => write!(f, "'*'"),
152            TokenKind::Slash => write!(f, "'/'"),
153            TokenKind::Comma => write!(f, "','"),
154            TokenKind::Percent => write!(f, "'%'"),
155            TokenKind::PercentPercent => write!(f, "'%%'"),
156            TokenKind::Caret => write!(f, "'^'"),
157            TokenKind::Gt => write!(f, "'>'"),
158            TokenKind::Lt => write!(f, "'<'"),
159            TokenKind::Gte => write!(f, "'>='"),
160            TokenKind::Lte => write!(f, "'<='"),
161            TokenKind::Colon => write!(f, "':'"),
162            TokenKind::Arrow => write!(f, "'->'"),
163            TokenKind::Ellipsis => write!(f, "'...'"),
164            TokenKind::Dot => write!(f, "'.'"),
165            TokenKind::At => write!(f, "'@'"),
166            TokenKind::LParen => write!(f, "'('"),
167            TokenKind::RParen => write!(f, "')'"),
168            TokenKind::NumberLit => write!(f, "a number"),
169            TokenKind::StringLit => write!(f, "a string"),
170            TokenKind::Commentary => write!(f, "commentary block"),
171            TokenKind::Identifier => write!(f, "an identifier"),
172            TokenKind::Eof => write!(f, "end of file"),
173        }
174    }
175}
176
177#[derive(Debug, Clone)]
178pub struct Token {
179    pub kind: TokenKind,
180    pub span: Span,
181    pub text: String,
182}
183
184impl Token {
185    pub fn eof(offset: usize, line: usize, col: usize) -> Self {
186        Token {
187            kind: TokenKind::Eof,
188            span: Span {
189                start: offset,
190                end: offset,
191                line,
192                col,
193            },
194            text: String::new(),
195        }
196    }
197}
198
199#[derive(Clone)]
200pub struct LexerCheckpoint {
201    pos: usize,
202    line: usize,
203    col: usize,
204    byte_offset: usize,
205    peeked: Option<Token>,
206    peeked2: Option<Token>,
207}
208
209// todo: find out why derive Clone is necessary
210#[derive(Clone)]
211pub struct Lexer {
212    source: Vec<char>,
213    pos: usize,
214    line: usize,
215    col: usize,
216    byte_offset: usize,
217    source_type: crate::parsing::source::SourceType,
218    source_text: Arc<str>,
219    peeked: Option<Token>,
220    peeked2: Option<Token>,
221}
222
223impl Lexer {
224    pub fn new(input: &str, source_type: &crate::parsing::source::SourceType) -> Self {
225        let source_text: Arc<str> = Arc::from(input);
226        Lexer {
227            source: input.chars().collect(),
228            pos: 0,
229            line: 1,
230            col: 1,
231            byte_offset: 0,
232            source_type: source_type.clone(),
233            source_text,
234            peeked: None,
235            peeked2: None,
236        }
237    }
238
239    pub fn source_text(&self) -> Arc<str> {
240        self.source_text.clone()
241    }
242
243    pub fn source_type(&self) -> crate::parsing::source::SourceType {
244        self.source_type.clone()
245    }
246
247    pub fn peek(&mut self) -> Result<&Token, Error> {
248        if self.peeked.is_none() {
249            let token = self.lex_token()?;
250            self.peeked = Some(token);
251        }
252        Ok(self.peeked.as_ref().expect("just assigned"))
253    }
254
255    pub fn peek_second(&mut self) -> Result<&Token, Error> {
256        self.peek()?;
257        if self.peeked2.is_none() {
258            let token = self.lex_token()?;
259            self.peeked2 = Some(token);
260        }
261        Ok(self.peeked2.as_ref().expect("just assigned"))
262    }
263
264    /// Current raw position as a Span. Does not trigger tokenization.
265    pub fn current_span(&self) -> Span {
266        Span {
267            start: self.byte_offset,
268            end: self.byte_offset,
269            line: self.line,
270            col: self.col,
271        }
272    }
273
274    pub fn next_token(&mut self) -> Result<Token, Error> {
275        if let Some(token) = self.peeked.take() {
276            self.peeked = self.peeked2.take();
277            return Ok(token);
278        }
279        self.lex_token()
280    }
281
282    /// Saved lexer position for speculative parsing.
283    pub fn checkpoint(&self) -> LexerCheckpoint {
284        LexerCheckpoint {
285            pos: self.pos,
286            line: self.line,
287            col: self.col,
288            byte_offset: self.byte_offset,
289            peeked: self.peeked.clone(),
290            peeked2: self.peeked2.clone(),
291        }
292    }
293
294    pub fn restore(&mut self, checkpoint: LexerCheckpoint) {
295        self.pos = checkpoint.pos;
296        self.line = checkpoint.line;
297        self.col = checkpoint.col;
298        self.byte_offset = checkpoint.byte_offset;
299        self.peeked = checkpoint.peeked;
300        self.peeked2 = checkpoint.peeked2;
301    }
302
303    fn current_char(&self) -> Option<char> {
304        self.source.get(self.pos).copied()
305    }
306
307    fn peek_char(&self) -> Option<char> {
308        self.source.get(self.pos + 1).copied()
309    }
310
311    fn peek_char_at(&self, offset: usize) -> Option<char> {
312        self.source.get(self.pos + offset).copied()
313    }
314
315    fn advance(&mut self) {
316        if let Some(ch) = self.current_char() {
317            self.byte_offset += ch.len_utf8();
318            if ch == '\n' {
319                self.line += 1;
320                self.col = 1;
321            } else {
322                self.col += 1;
323            }
324            self.pos += 1;
325        }
326    }
327
328    fn skip_whitespace(&mut self) {
329        while let Some(ch) = self.current_char() {
330            if ch.is_whitespace() {
331                self.advance();
332            } else {
333                break;
334            }
335        }
336    }
337
338    fn make_span(&self, start_byte: usize, start_line: usize, start_col: usize) -> Span {
339        Span {
340            start: start_byte,
341            end: self.byte_offset,
342            line: start_line,
343            col: start_col,
344        }
345    }
346
347    fn make_error(&self, message: impl Into<String>, span: Span) -> Error {
348        Error::parsing(
349            message,
350            Source::new(self.source_type.clone(), span),
351            None::<String>,
352        )
353    }
354
355    fn lex_token(&mut self) -> Result<Token, Error> {
356        self.skip_whitespace();
357
358        let start_byte = self.byte_offset;
359        let start_line = self.line;
360        let start_col = self.col;
361
362        let Some(ch) = self.current_char() else {
363            return Ok(Token::eof(start_byte, start_line, start_col));
364        };
365
366        // Triple-quote commentary
367        if ch == '"' && self.peek_char() == Some('"') && self.peek_char_at(2) == Some('"') {
368            return self.scan_triple_quote(start_byte, start_line, start_col);
369        }
370
371        // String literal
372        if ch == '"' {
373            return self.scan_string(start_byte, start_line, start_col);
374        }
375
376        // Number literal (sign handled by parser, not lexer)
377        if ch.is_ascii_digit() {
378            return self.scan_number(start_byte, start_line, start_col);
379        }
380
381        // Two-character operators (check before single-char)
382        if let Some(token) = self.try_two_char_operator(start_byte, start_line, start_col) {
383            return Ok(token);
384        }
385
386        // Three-character ellipsis
387        if ch == '.' && self.peek_char() == Some('.') && self.peek_char_at(2) == Some('.') {
388            self.advance();
389            self.advance();
390            self.advance();
391            let span = self.make_span(start_byte, start_line, start_col);
392            return Ok(Token {
393                kind: TokenKind::Ellipsis,
394                span,
395                text: "...".to_string(),
396            });
397        }
398
399        // Single-character operators/punctuation
400        if let Some(kind) = self.single_char_token(ch) {
401            self.advance();
402            let span = self.make_span(start_byte, start_line, start_col);
403            let text = ch.to_string();
404            return Ok(Token { kind, span, text });
405        }
406
407        // Identifier or keyword (starts with letter or @)
408        if ch.is_ascii_alphabetic() || ch == '_' {
409            return Ok(self.scan_identifier(start_byte, start_line, start_col));
410        }
411
412        // @ prefix for registry references
413        if ch == '@' {
414            self.advance();
415            let span = self.make_span(start_byte, start_line, start_col);
416            return Ok(Token {
417                kind: TokenKind::At,
418                span,
419                text: "@".to_string(),
420            });
421        }
422
423        // Unknown character
424        self.advance();
425        let span = self.make_span(start_byte, start_line, start_col);
426        Err(self.make_error(format!("Unexpected character '{}'", ch), span))
427    }
428
429    fn scan_triple_quote(
430        &mut self,
431        start_byte: usize,
432        start_line: usize,
433        start_col: usize,
434    ) -> Result<Token, Error> {
435        self.advance(); // "
436        self.advance(); // "
437        self.advance(); // "
438
439        let content_start = self.byte_offset;
440        loop {
441            match self.current_char() {
442                None => {
443                    let span = self.make_span(start_byte, start_line, start_col);
444                    return Err(self.make_error(
445                        "Unterminated commentary block: expected closing \"\"\"",
446                        span,
447                    ));
448                }
449                Some('"')
450                    if self.source.get(self.pos + 1) == Some(&'"')
451                        && self.source.get(self.pos + 2) == Some(&'"') =>
452                {
453                    let content_end = self.byte_offset;
454                    self.advance(); // "
455                    self.advance(); // "
456                    self.advance(); // "
457                    let raw: String = self.source_text[content_start..content_end].to_string();
458                    let span = self.make_span(start_byte, start_line, start_col);
459                    return Ok(Token {
460                        kind: TokenKind::Commentary,
461                        span,
462                        text: raw,
463                    });
464                }
465                Some(_) => {
466                    self.advance();
467                }
468            }
469        }
470    }
471
472    fn scan_string(
473        &mut self,
474        start_byte: usize,
475        start_line: usize,
476        start_col: usize,
477    ) -> Result<Token, Error> {
478        self.advance(); // consume opening "
479        let mut content = String::new();
480        loop {
481            match self.current_char() {
482                None => {
483                    let span = self.make_span(start_byte, start_line, start_col);
484                    return Err(self.make_error("String starting here was never closed", span));
485                }
486                Some('"') => {
487                    self.advance(); // consume closing "
488                    break;
489                }
490                Some(ch) => {
491                    content.push(ch);
492                    self.advance();
493                }
494            }
495        }
496        let span = self.make_span(start_byte, start_line, start_col);
497        if content.len() > crate::limits::MAX_TEXT_VALUE_LENGTH {
498            return Err(self.make_error(
499                format!(
500                    "Text literal exceeds maximum length of {} characters (found {})",
501                    crate::limits::MAX_TEXT_VALUE_LENGTH,
502                    content.len()
503                ),
504                span,
505            ));
506        }
507        // Store the full text including quotes for span accuracy,
508        // but content without quotes for the parser to use.
509        let full_text = format!("\"{}\"", content);
510        Ok(Token {
511            kind: TokenKind::StringLit,
512            span,
513            text: full_text,
514        })
515    }
516
517    fn scan_number(
518        &mut self,
519        start_byte: usize,
520        start_line: usize,
521        start_col: usize,
522    ) -> Result<Token, Error> {
523        let mut text = String::new();
524
525        // Integer part: digits with optional _ or , separators
526        while let Some(ch) = self.current_char() {
527            if ch.is_ascii_digit() || ch == '_' || ch == ',' {
528                text.push(ch);
529                self.advance();
530            } else {
531                break;
532            }
533        }
534
535        // Decimal part
536        if self.current_char() == Some('.') {
537            // Check if next char after dot is a digit (not a method call or dotted reference)
538            if let Some(next) = self.peek_char() {
539                if next.is_ascii_digit() {
540                    text.push('.');
541                    self.advance(); // consume .
542                    while let Some(ch) = self.current_char() {
543                        if ch.is_ascii_digit() {
544                            text.push(ch);
545                            self.advance();
546                        } else {
547                            break;
548                        }
549                    }
550                }
551            }
552        }
553
554        // Scientific notation: e or E followed by optional +/- and digits
555        if let Some(ch) = self.current_char() {
556            if ch == 'e' || ch == 'E' {
557                let mut sci_text = String::new();
558                sci_text.push(ch);
559                let save_pos = self.pos;
560                let save_byte = self.byte_offset;
561                let save_line = self.line;
562                let save_col = self.col;
563                self.advance(); // consume e/E
564
565                if let Some(sign) = self.current_char() {
566                    if sign == '+' || sign == '-' {
567                        sci_text.push(sign);
568                        self.advance();
569                    }
570                }
571
572                if let Some(d) = self.current_char() {
573                    if d.is_ascii_digit() {
574                        while let Some(ch) = self.current_char() {
575                            if ch.is_ascii_digit() {
576                                sci_text.push(ch);
577                                self.advance();
578                            } else {
579                                break;
580                            }
581                        }
582                        text.push_str(&sci_text);
583                    } else {
584                        // Not actually scientific notation, backtrack
585                        self.pos = save_pos;
586                        self.byte_offset = save_byte;
587                        self.line = save_line;
588                        self.col = save_col;
589                    }
590                } else {
591                    self.pos = save_pos;
592                    self.byte_offset = save_byte;
593                    self.line = save_line;
594                    self.col = save_col;
595                }
596            }
597        }
598
599        let span = self.make_span(start_byte, start_line, start_col);
600        Ok(Token {
601            kind: TokenKind::NumberLit,
602            span,
603            text,
604        })
605    }
606
607    fn try_two_char_operator(
608        &mut self,
609        start_byte: usize,
610        start_line: usize,
611        start_col: usize,
612    ) -> Option<Token> {
613        let ch = self.current_char()?;
614        let next = self.peek_char();
615
616        let kind = match (ch, next) {
617            ('-', Some('>')) => TokenKind::Arrow,
618            ('>', Some('=')) => TokenKind::Gte,
619            ('<', Some('=')) => TokenKind::Lte,
620            ('%', Some('%')) => {
621                // Check that it's not followed by a digit (invalid permille like 10%%5)
622                TokenKind::PercentPercent
623            }
624            _ => return None,
625        };
626
627        self.advance();
628        self.advance();
629        let span = self.make_span(start_byte, start_line, start_col);
630        let text: String = self.source_text[span.start..span.end].to_string();
631        Some(Token { kind, span, text })
632    }
633
634    fn single_char_token(&self, ch: char) -> Option<TokenKind> {
635        match ch {
636            '+' => Some(TokenKind::Plus),
637            '*' => Some(TokenKind::Star),
638            '/' => Some(TokenKind::Slash),
639            ',' => Some(TokenKind::Comma),
640            '^' => Some(TokenKind::Caret),
641            ':' => Some(TokenKind::Colon),
642            '.' => Some(TokenKind::Dot),
643            '(' => Some(TokenKind::LParen),
644            ')' => Some(TokenKind::RParen),
645            '>' => Some(TokenKind::Gt),
646            '<' => Some(TokenKind::Lt),
647            '%' => Some(TokenKind::Percent),
648            '-' => Some(TokenKind::Minus),
649            _ => None,
650        }
651    }
652
653    fn scan_identifier(&mut self, start_byte: usize, start_line: usize, start_col: usize) -> Token {
654        let mut text = String::new();
655        while let Some(ch) = self.current_char() {
656            if ch.is_ascii_alphanumeric() || ch == '_' {
657                text.push(ch);
658                self.advance();
659            } else {
660                break;
661            }
662        }
663
664        let kind = keyword_from_identifier(&text);
665        let span = self.make_span(start_byte, start_line, start_col);
666        Token { kind, span, text }
667    }
668}
669
670fn keyword_from_identifier(text: &str) -> TokenKind {
671    match text.to_lowercase().as_str() {
672        "spec" => TokenKind::Spec,
673        "repo" => TokenKind::Repo,
674        "data" => TokenKind::Data,
675        "rule" => TokenKind::Rule,
676        "unless" => TokenKind::Unless,
677        "then" => TokenKind::Then,
678        "not" => TokenKind::Not,
679        "and" => TokenKind::And,
680        "in" => TokenKind::In,
681        "as" => TokenKind::As,
682        "uses" => TokenKind::Uses,
683        "with" => TokenKind::With,
684        "meta" => TokenKind::Meta,
685        "veto" => TokenKind::Veto,
686        "now" => TokenKind::Now,
687        "past" => TokenKind::Past,
688        "future" => TokenKind::Future,
689        "true" => TokenKind::True,
690        "false" => TokenKind::False,
691        "yes" => TokenKind::Yes,
692        "no" => TokenKind::No,
693        "accept" => TokenKind::Accept,
694        "reject" => TokenKind::Reject,
695        "measure" => TokenKind::MeasureKw,
696        "number" => TokenKind::NumberKw,
697        "text" => TokenKind::TextKw,
698        "date" => TokenKind::DateKw,
699        "time" => TokenKind::TimeKw,
700        "boolean" => TokenKind::BooleanKw,
701        "ratio" => TokenKind::RatioKw,
702        "sqrt" => TokenKind::Sqrt,
703        "sin" => TokenKind::Sin,
704        "cos" => TokenKind::Cos,
705        "tan" => TokenKind::Tan,
706        "asin" => TokenKind::Asin,
707        "acos" => TokenKind::Acos,
708        "atan" => TokenKind::Atan,
709        "log" => TokenKind::Log,
710        "exp" => TokenKind::Exp,
711        "abs" => TokenKind::Abs,
712        "floor" => TokenKind::Floor,
713        "ceil" => TokenKind::Ceil,
714        "round" => TokenKind::Round,
715        "is" => TokenKind::Is,
716        "permille" => TokenKind::Permille,
717        _ => TokenKind::Identifier,
718    }
719}
720
721/// Structural keywords can never be used as identifiers (data/rule names).
722/// Type keywords (measure, number, text, date, time, boolean, ratio)
723/// are reserved and cannot be used as names.
724pub fn is_keyword(kind: &TokenKind) -> bool {
725    matches!(
726        kind,
727        TokenKind::Spec
728            | TokenKind::Repo
729            | TokenKind::Data
730            | TokenKind::Rule
731            | TokenKind::Unless
732            | TokenKind::Then
733            | TokenKind::Not
734            | TokenKind::And
735            | TokenKind::In
736            | TokenKind::As
737            | TokenKind::Uses
738            | TokenKind::With
739            | TokenKind::Meta
740            | TokenKind::Veto
741            | TokenKind::Now
742            | TokenKind::Sqrt
743            | TokenKind::Sin
744            | TokenKind::Cos
745            | TokenKind::Tan
746            | TokenKind::Asin
747            | TokenKind::Acos
748            | TokenKind::Atan
749            | TokenKind::Log
750            | TokenKind::Exp
751            | TokenKind::Abs
752            | TokenKind::Floor
753            | TokenKind::Ceil
754            | TokenKind::Round
755            | TokenKind::True
756            | TokenKind::False
757            | TokenKind::Yes
758            | TokenKind::No
759            | TokenKind::Accept
760            | TokenKind::Reject
761            | TokenKind::MeasureKw
762            | TokenKind::NumberKw
763            | TokenKind::TextKw
764            | TokenKind::DateKw
765            | TokenKind::TimeKw
766            | TokenKind::BooleanKw
767            | TokenKind::RatioKw
768    )
769}
770
771/// Map type keyword token to PrimitiveKind. Single source of truth for type keywords.
772#[must_use]
773pub fn token_kind_to_primitive(kind: &TokenKind) -> Option<PrimitiveKind> {
774    match kind {
775        TokenKind::BooleanKw => Some(PrimitiveKind::Boolean),
776        TokenKind::MeasureKw => Some(PrimitiveKind::Measure),
777        TokenKind::NumberKw => Some(PrimitiveKind::Number),
778        TokenKind::RatioKw => Some(PrimitiveKind::Ratio),
779        TokenKind::TextKw => Some(PrimitiveKind::Text),
780        TokenKind::DateKw => Some(PrimitiveKind::Date),
781        TokenKind::TimeKw => Some(PrimitiveKind::Time),
782        _ => None,
783    }
784}
785
786/// Returns true if the token kind represents a boolean literal keyword.
787pub fn is_boolean_keyword(kind: &TokenKind) -> bool {
788    matches!(
789        kind,
790        TokenKind::True
791            | TokenKind::False
792            | TokenKind::Yes
793            | TokenKind::No
794            | TokenKind::Accept
795            | TokenKind::Reject
796    )
797}
798
799/// Maps a boolean-keyword token kind to BooleanValue. Call only when `is_boolean_keyword(kind)`.
800#[must_use]
801pub fn token_kind_to_boolean_value(kind: &TokenKind) -> BooleanValue {
802    match kind {
803        TokenKind::True => BooleanValue::True,
804        TokenKind::False => BooleanValue::False,
805        TokenKind::Yes => BooleanValue::Yes,
806        TokenKind::No => BooleanValue::No,
807        TokenKind::Accept => BooleanValue::Accept,
808        TokenKind::Reject => BooleanValue::Reject,
809        _ => unreachable!(
810            "BUG: token_kind_to_boolean_value called with non-boolean token {:?}",
811            kind
812        ),
813    }
814}
815
816/// Returns true if the token kind represents a math function keyword.
817pub fn is_math_function(kind: &TokenKind) -> bool {
818    matches!(
819        kind,
820        TokenKind::Sqrt
821            | TokenKind::Sin
822            | TokenKind::Cos
823            | TokenKind::Tan
824            | TokenKind::Asin
825            | TokenKind::Acos
826            | TokenKind::Atan
827            | TokenKind::Log
828            | TokenKind::Exp
829            | TokenKind::Abs
830            | TokenKind::Floor
831            | TokenKind::Ceil
832            | TokenKind::Round
833    )
834}
835
836/// Returns true if the token kind can start the body of a spec
837/// (data, rule, or meta definition).
838pub fn is_spec_body_keyword(kind: &TokenKind) -> bool {
839    matches!(
840        kind,
841        TokenKind::Data | TokenKind::With | TokenKind::Rule | TokenKind::Meta
842    )
843}
844
845/// Returns true if the token can be used as a label/identifier
846/// (i.e. it is a non-reserved keyword or an identifier).
847pub fn can_be_label(kind: &TokenKind) -> bool {
848    matches!(
849        kind,
850        TokenKind::Identifier
851            | TokenKind::Past
852            | TokenKind::Future
853            | TokenKind::Permille
854            | TokenKind::Is
855    )
856}
857
858/// `calendar` in `in calendar month` / `past calendar year` — not a type keyword.
859#[must_use]
860pub fn token_is_calendar_period_marker(tok: &Token) -> bool {
861    tok.kind == TokenKind::Identifier && tok.text == "calendar"
862}
863
864/// Returns true if the token kind can be used as a reference segment
865/// (identifier, type keyword, or non-reserved contextual keyword).
866pub fn can_be_reference_segment(kind: &TokenKind) -> bool {
867    can_be_label(kind)
868}
869
870/// Slash-/dot-separated registry path segments (`@org/repo/...`). Keywords that are
871/// reserved at the structural level (`spec`, `rule`, etc.) are allowed inside
872/// multi-segment paths (e.g. `@org/repo`) but callers must reject them when they
873/// appear as the entire stand-alone name (e.g. `repo spec`).
874#[must_use]
875pub fn can_be_repository_qualifier_segment(kind: &TokenKind) -> bool {
876    matches!(kind, TokenKind::Identifier)
877        || is_keyword(kind)
878        || can_be_label(kind)
879        || is_boolean_keyword(kind)
880        || is_math_function(kind)
881}
882
883#[cfg(test)]
884mod tests {
885    use super::*;
886
887    fn lex_all(input: &str) -> Result<Vec<Token>, Error> {
888        let mut lexer = Lexer::new(input, &crate::parsing::source::SourceType::Volatile);
889        let mut tokens = Vec::new();
890        loop {
891            let token = lexer.next_token()?;
892            if token.kind == TokenKind::Eof {
893                tokens.push(token);
894                break;
895            }
896            tokens.push(token);
897        }
898        Ok(tokens)
899    }
900
901    fn lex_kinds(input: &str) -> Result<Vec<TokenKind>, Error> {
902        Ok(lex_all(input)?.into_iter().map(|t| t.kind).collect())
903    }
904
905    #[test]
906    fn lex_empty_input() {
907        let tokens = lex_all("").unwrap();
908        assert_eq!(tokens.len(), 1);
909        assert_eq!(tokens[0].kind, TokenKind::Eof);
910    }
911
912    #[test]
913    fn string_literal_at_max_length_is_accepted() {
914        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH);
915        let tokens = lex_all(&format!("\"{content}\"")).unwrap();
916        assert_eq!(tokens[0].kind, TokenKind::StringLit);
917    }
918
919    #[test]
920    fn string_literal_over_max_length_is_parse_error() {
921        let content = "a".repeat(crate::limits::MAX_TEXT_VALUE_LENGTH + 1);
922        let err = lex_all(&format!("\"{content}\"")).unwrap_err();
923        assert!(
924            err.message().contains("maximum length"),
925            "expected length error, got: {err}"
926        );
927        assert!(err.location().is_some(), "parse error must carry a source");
928    }
929
930    #[test]
931    fn number_literal_with_separators_lexes() {
932        let tokens = lex_all("9,999,999,999,999,999,999,999,999,999").unwrap();
933        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
934    }
935
936    #[test]
937    fn lex_spec_declaration() {
938        let kinds = lex_kinds("spec person").unwrap();
939        assert_eq!(
940            kinds,
941            vec![TokenKind::Spec, TokenKind::Identifier, TokenKind::Eof]
942        );
943    }
944
945    #[test]
946    fn lex_data_definition() {
947        let kinds = lex_kinds("data age: 25").unwrap();
948        assert_eq!(
949            kinds,
950            vec![
951                TokenKind::Data,
952                TokenKind::Identifier,
953                TokenKind::Colon,
954                TokenKind::NumberLit,
955                TokenKind::Eof,
956            ]
957        );
958    }
959
960    #[test]
961    fn lex_rule_with_comparison() {
962        let kinds = lex_kinds("rule is_adult: age >= 18").unwrap();
963        assert_eq!(
964            kinds,
965            vec![
966                TokenKind::Rule,
967                TokenKind::Identifier,
968                TokenKind::Colon,
969                TokenKind::Identifier,
970                TokenKind::Gte,
971                TokenKind::NumberLit,
972                TokenKind::Eof,
973            ]
974        );
975    }
976
977    #[test]
978    fn lex_string_literal() {
979        let tokens = lex_all(r#""hello world""#).unwrap();
980        assert_eq!(tokens[0].kind, TokenKind::StringLit);
981        assert_eq!(tokens[0].text, "\"hello world\"");
982    }
983
984    #[test]
985    fn lex_unterminated_string() {
986        let result = lex_all(r#""hello"#);
987        assert!(result.is_err());
988    }
989
990    #[test]
991    fn lex_number_with_decimal() {
992        let tokens = lex_all("3.14").unwrap();
993        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
994        assert_eq!(tokens[0].text, "3.14");
995    }
996
997    #[test]
998    fn lex_number_with_underscores() {
999        let tokens = lex_all("1_000_000").unwrap();
1000        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1001        assert_eq!(tokens[0].text, "1_000_000");
1002    }
1003
1004    #[test]
1005    fn lex_scientific_notation() {
1006        let tokens = lex_all("1.5e+10").unwrap();
1007        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1008        assert_eq!(tokens[0].text, "1.5e+10");
1009    }
1010
1011    #[test]
1012    fn lex_all_operators() {
1013        let kinds = lex_kinds("+ - * / % ^ > < >= <= -> %%").unwrap();
1014        assert_eq!(
1015            &kinds[..12],
1016            &[
1017                TokenKind::Plus,
1018                TokenKind::Minus,
1019                TokenKind::Star,
1020                TokenKind::Slash,
1021                TokenKind::Percent,
1022                TokenKind::Caret,
1023                TokenKind::Gt,
1024                TokenKind::Lt,
1025                TokenKind::Gte,
1026                TokenKind::Lte,
1027                TokenKind::Arrow,
1028                TokenKind::PercentPercent,
1029            ]
1030        );
1031    }
1032
1033    #[test]
1034    fn lex_keywords() {
1035        let kinds =
1036            lex_kinds("spec data rule unless then not and in as uses meta veto now").unwrap();
1037        assert_eq!(
1038            &kinds[..13],
1039            &[
1040                TokenKind::Spec,
1041                TokenKind::Data,
1042                TokenKind::Rule,
1043                TokenKind::Unless,
1044                TokenKind::Then,
1045                TokenKind::Not,
1046                TokenKind::And,
1047                TokenKind::In,
1048                TokenKind::As,
1049                TokenKind::Uses,
1050                TokenKind::Meta,
1051                TokenKind::Veto,
1052                TokenKind::Now,
1053            ]
1054        );
1055    }
1056
1057    #[test]
1058    fn lex_boolean_keywords() {
1059        let kinds = lex_kinds("true false yes no accept reject").unwrap();
1060        assert_eq!(
1061            &kinds[..6],
1062            &[
1063                TokenKind::True,
1064                TokenKind::False,
1065                TokenKind::Yes,
1066                TokenKind::No,
1067                TokenKind::Accept,
1068                TokenKind::Reject,
1069            ]
1070        );
1071    }
1072
1073    #[test]
1074    fn lex_duration_keywords() {
1075        let kinds = lex_kinds("years months weeks days hours minutes seconds").unwrap();
1076        assert_eq!(
1077            &kinds[..7],
1078            &[
1079                TokenKind::Identifier,
1080                TokenKind::Identifier,
1081                TokenKind::Identifier,
1082                TokenKind::Identifier,
1083                TokenKind::Identifier,
1084                TokenKind::Identifier,
1085                TokenKind::Identifier,
1086            ]
1087        );
1088    }
1089
1090    #[test]
1091    fn lex_commentary() {
1092        let tokens = lex_all(r#""""hello world""""#).unwrap();
1093        assert_eq!(tokens[0].kind, TokenKind::Commentary);
1094        assert_eq!(tokens[0].text, "hello world");
1095    }
1096
1097    #[test]
1098    fn lex_at_sign() {
1099        let kinds = lex_kinds("@user").unwrap();
1100        assert_eq!(kinds[0], TokenKind::At);
1101        assert_eq!(kinds[1], TokenKind::Identifier);
1102    }
1103
1104    #[test]
1105    fn lex_parentheses() {
1106        let kinds = lex_kinds("(x + 1)").unwrap();
1107        assert_eq!(
1108            &kinds[..5],
1109            &[
1110                TokenKind::LParen,
1111                TokenKind::Identifier,
1112                TokenKind::Plus,
1113                TokenKind::NumberLit,
1114                TokenKind::RParen,
1115            ]
1116        );
1117    }
1118
1119    #[test]
1120    fn lex_dot_for_references() {
1121        let kinds = lex_kinds("employee.salary").unwrap();
1122        assert_eq!(
1123            &kinds[..3],
1124            &[TokenKind::Identifier, TokenKind::Dot, TokenKind::Identifier]
1125        );
1126    }
1127
1128    #[test]
1129    fn lex_spec_name_with_slashes() {
1130        let tokens = lex_all("spec contracts/employment/jack").unwrap();
1131        assert_eq!(tokens[0].kind, TokenKind::Spec);
1132        // The lexer will see "contracts" as identifier, then "/" as Slash
1133        // The parser will handle assembling the spec name.
1134        assert_eq!(tokens[1].kind, TokenKind::Identifier);
1135    }
1136
1137    #[test]
1138    fn lex_number_not_followed_by_e_identifier() {
1139        // "42 eur" should be number then identifier, not scientific notation
1140        let tokens = lex_all("42 eur").unwrap();
1141        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1142        assert_eq!(tokens[0].text, "42");
1143        assert_eq!(tokens[1].kind, TokenKind::Identifier);
1144        assert_eq!(tokens[1].text, "eur");
1145    }
1146
1147    #[test]
1148    fn lex_unknown_character() {
1149        let result = lex_all("§");
1150        assert!(result.is_err());
1151    }
1152
1153    #[test]
1154    fn lex_peek_does_not_consume() {
1155        let mut lexer = Lexer::new("spec test", &crate::parsing::source::SourceType::Volatile);
1156        let peeked_kind = lexer.peek().unwrap().kind.clone();
1157        assert_eq!(peeked_kind, TokenKind::Spec);
1158        let next = lexer.next_token().unwrap();
1159        assert_eq!(next.kind, TokenKind::Spec);
1160    }
1161
1162    #[test]
1163    fn lex_span_byte_offsets() {
1164        let tokens = lex_all("spec test").unwrap();
1165        assert_eq!(tokens[0].span.start, 0);
1166        assert_eq!(tokens[0].span.end, 4);
1167        assert_eq!(tokens[0].span.line, 1);
1168        assert_eq!(tokens[0].span.col, 1);
1169
1170        assert_eq!(tokens[1].span.start, 5);
1171        assert_eq!(tokens[1].span.end, 9);
1172        assert_eq!(tokens[1].span.line, 1);
1173        assert_eq!(tokens[1].span.col, 6);
1174    }
1175
1176    #[test]
1177    fn lex_multiline_span_tracking() {
1178        let tokens = lex_all("spec test\ndata x: 1").unwrap();
1179        // "data" should be on line 2
1180        let data_token = &tokens[2]; // spec, test, data
1181        assert_eq!(data_token.kind, TokenKind::Data);
1182        assert_eq!(data_token.span.line, 2);
1183        assert_eq!(data_token.span.col, 1);
1184    }
1185
1186    #[test]
1187    fn lex_case_insensitive_keywords() {
1188        // Lemma keywords are case-insensitive
1189        let kinds = lex_kinds("SPEC Data RULE").unwrap();
1190        assert_eq!(kinds[0], TokenKind::Spec);
1191        assert_eq!(kinds[1], TokenKind::Data);
1192        assert_eq!(kinds[2], TokenKind::Rule);
1193    }
1194
1195    #[test]
1196    fn lex_math_function_keywords() {
1197        let kinds =
1198            lex_kinds("sqrt sin cos tan asin acos atan log exp abs floor ceil round").unwrap();
1199        assert_eq!(
1200            &kinds[..13],
1201            &[
1202                TokenKind::Sqrt,
1203                TokenKind::Sin,
1204                TokenKind::Cos,
1205                TokenKind::Tan,
1206                TokenKind::Asin,
1207                TokenKind::Acos,
1208                TokenKind::Atan,
1209                TokenKind::Log,
1210                TokenKind::Exp,
1211                TokenKind::Abs,
1212                TokenKind::Floor,
1213                TokenKind::Ceil,
1214                TokenKind::Round,
1215            ]
1216        );
1217    }
1218
1219    #[test]
1220    fn lex_is_keyword() {
1221        let kinds = lex_kinds("status is \"active\"").unwrap();
1222        assert_eq!(kinds[0], TokenKind::Identifier);
1223        assert_eq!(kinds[1], TokenKind::Is);
1224        assert_eq!(kinds[2], TokenKind::StringLit);
1225    }
1226
1227    #[test]
1228    fn lex_percent_not_followed_by_digit() {
1229        // "50%" should be number then percent
1230        let kinds = lex_kinds("50%").unwrap();
1231        assert_eq!(kinds[0], TokenKind::NumberLit);
1232        assert_eq!(kinds[1], TokenKind::Percent);
1233    }
1234
1235    #[test]
1236    fn lex_number_with_commas() {
1237        let tokens = lex_all("1,000,000").unwrap();
1238        assert_eq!(tokens[0].kind, TokenKind::NumberLit);
1239        assert_eq!(tokens[0].text, "1,000,000");
1240    }
1241
1242    #[test]
1243    fn lex_arrow_chain() {
1244        let kinds = lex_kinds("-> unit eur 1.00 -> decimals 2").unwrap();
1245        assert_eq!(kinds[0], TokenKind::Arrow);
1246        assert_eq!(kinds[1], TokenKind::Identifier);
1247        assert_eq!(kinds[2], TokenKind::Identifier);
1248        assert_eq!(kinds[3], TokenKind::NumberLit);
1249        assert_eq!(kinds[4], TokenKind::Arrow);
1250    }
1251}
lemma/parsing/lexer.rs

lemma/parsing/
lexer.rs
