probl_syntax/
token.rs

1//! Tokens produced by the lexer.
2
3use crate::span::Span;
4
5#[derive(Clone, Debug, PartialEq)]
6pub enum Tok {
7    Int(probl_number::Integer),
8    Float(f64),
9    /// A percentage, already divided by 100: `30%` is `Percent(0.3)`.
10    Percent(f64),
11    /// `2d6` is `Dice { count: 2, sides: 6 }`; `d6` has a count of 1.
12    Dice {
13        count: u32,
14        sides: u32,
15    },
16    Str(Vec<StrPart>),
17    /// Identifiers, including the contextual keywords `as`, `by`, `from` and `to`.
18    Ident(String),
19
20    // Keywords
21    And,
22    Break,
23    Chance,
24    Continue,
25    Div,
26    Else,
27    Enum,
28    False,
29    Fn,
30    For,
31    If,
32    Import,
33    In,
34    Let,
35    Loop,
36    Match,
37    Mod,
38    Not,
39    Observe,
40    Score,
41    Or,
42    Repeat,
43    Report,
44    Return,
45    Simulate,
46    True,
47    Type,
48    Typeof,
49    Var,
50    While,
51    With,
52
53    // Punctuation
54    LParen,
55    RParen,
56    LBracket,
57    RBracket,
58    LBrace,
59    RBrace,
60    Comma,
61    Dot,
62    Colon,
63    Semi,
64    At,
65    Pipe,
66    Underscore,
67    /// `->`
68    Arrow,
69    /// `=>`
70    FatArrow,
71    Tilde,
72    Assign,
73    PlusAssign,
74    MinusAssign,
75    StarAssign,
76    SlashAssign,
77    EqEq,
78    NotEq,
79    Lt,
80    Le,
81    Gt,
82    Ge,
83    Plus,
84    Minus,
85    Star,
86    Slash,
87    Caret,
88    DotDot,
89    /// `..<`
90    DotDotLt,
91
92    Newline,
93    Eof,
94}
95
96/// A piece of a string literal: plain text, or the source of an interpolated `{expression}`.
97#[derive(Clone, Debug, PartialEq)]
98pub enum StrPart {
99    Lit(String),
100    Expr { src: String, offset: u32 },
101}
102
103#[derive(Clone, Debug, PartialEq)]
104pub struct Token {
105    pub tok: Tok,
106    pub span: Span,
107}
108
109/// Whether `text` can be a name in a program: ASCII letters, digits and
110/// `_`, not starting with a digit, and neither a keyword nor a die like `d6`.
111pub fn is_name(text: &str) -> bool {
112    let is_die = text
113        .strip_prefix('d')
114        .is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()));
115    text.bytes()
116        .next()
117        .is_some_and(|b| b.is_ascii_alphabetic() || b == b'_')
118        && text.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_')
119        && text != "_"
120        && !is_die
121        && keyword(text).is_none()
122}
123
124pub fn keyword(word: &str) -> Option<Tok> {
125    Some(match word {
126        "and" => Tok::And,
127        "break" => Tok::Break,
128        "chance" => Tok::Chance,
129        "continue" => Tok::Continue,
130        "div" => Tok::Div,
131        "else" => Tok::Else,
132        "enum" => Tok::Enum,
133        "false" => Tok::False,
134        "fn" => Tok::Fn,
135        "for" => Tok::For,
136        "if" => Tok::If,
137        "import" => Tok::Import,
138        "in" => Tok::In,
139        "let" => Tok::Let,
140        "loop" => Tok::Loop,
141        "match" => Tok::Match,
142        "mod" => Tok::Mod,
143        "not" => Tok::Not,
144        "observe" => Tok::Observe,
145        "score" => Tok::Score,
146        "or" => Tok::Or,
147        "repeat" => Tok::Repeat,
148        "report" => Tok::Report,
149        "return" => Tok::Return,
150        "simulate" => Tok::Simulate,
151        "true" => Tok::True,
152        "type" => Tok::Type,
153        "typeof" => Tok::Typeof,
154        "var" => Tok::Var,
155        "while" => Tok::While,
156        "with" => Tok::With,
157        _ => return None,
158    })
159}
160
161impl Tok {
162    /// How the token is described in error messages.
163    pub fn describe(&self) -> String {
164        match self {
165            Tok::Int(v) => format!("number `{v}`"),
166            Tok::Float(v) => format!("number `{v}`"),
167            Tok::Percent(v) => format!("percentage `{}%`", v * 100.0),
168            Tok::Dice { count, sides } => format!("dice `{count}d{sides}`"),
169            Tok::Str(_) => "a string".to_string(),
170            Tok::Ident(name) => format!("`{name}`"),
171            Tok::Newline => "the end of the line".to_string(),
172            Tok::Eof => "the end of the file".to_string(),
173            other => format!("`{}`", other.text()),
174        }
175    }
176
177    /// Source text of keywords and punctuation.
178    pub fn text(&self) -> &'static str {
179        match self {
180            Tok::And => "and",
181            Tok::Break => "break",
182            Tok::Chance => "chance",
183            Tok::Continue => "continue",
184            Tok::Div => "div",
185            Tok::Else => "else",
186            Tok::Enum => "enum",
187            Tok::False => "false",
188            Tok::Fn => "fn",
189            Tok::For => "for",
190            Tok::If => "if",
191            Tok::Import => "import",
192            Tok::In => "in",
193            Tok::Let => "let",
194            Tok::Loop => "loop",
195            Tok::Match => "match",
196            Tok::Mod => "mod",
197            Tok::Not => "not",
198            Tok::Observe => "observe",
199            Tok::Score => "score",
200            Tok::Or => "or",
201            Tok::Repeat => "repeat",
202            Tok::Report => "report",
203            Tok::Return => "return",
204            Tok::Simulate => "simulate",
205            Tok::True => "true",
206            Tok::Type => "type",
207            Tok::Typeof => "typeof",
208            Tok::Var => "var",
209            Tok::While => "while",
210            Tok::With => "with",
211            Tok::LParen => "(",
212            Tok::RParen => ")",
213            Tok::LBracket => "[",
214            Tok::RBracket => "]",
215            Tok::LBrace => "{",
216            Tok::RBrace => "}",
217            Tok::Comma => ",",
218            Tok::Dot => ".",
219            Tok::Colon => ":",
220            Tok::Semi => ";",
221            Tok::At => "@",
222            Tok::Pipe => "|",
223            Tok::Underscore => "_",
224            Tok::Arrow => "->",
225            Tok::FatArrow => "=>",
226            Tok::Tilde => "~",
227            Tok::Assign => "=",
228            Tok::PlusAssign => "+=",
229            Tok::MinusAssign => "-=",
230            Tok::StarAssign => "*=",
231            Tok::SlashAssign => "/=",
232            Tok::EqEq => "==",
233            Tok::NotEq => "!=",
234            Tok::Lt => "<",
235            Tok::Le => "<=",
236            Tok::Gt => ">",
237            Tok::Ge => ">=",
238            Tok::Plus => "+",
239            Tok::Minus => "-",
240            Tok::Star => "*",
241            Tok::Slash => "/",
242            Tok::Caret => "^",
243            Tok::DotDot => "..",
244            Tok::DotDotLt => "..<",
245            Tok::Int(_)
246            | Tok::Float(_)
247            | Tok::Percent(_)
248            | Tok::Dice { .. }
249            | Tok::Str(_)
250            | Tok::Ident(_)
251            | Tok::Newline
252            | Tok::Eof => "",
253        }
254    }
255
256    pub fn is_ident(&self, word: &str) -> bool {
257        matches!(self, Tok::Ident(name) if name == word)
258    }
259}
probl_syntax/token.rs

probl_syntax/
token.rs
