probl_syntax/
token.rs

1//! Tokens produced by the lexer.
2
3use crate::span::Span;
4
5#[derive(Clone, Debug, PartialEq)]
6pub enum Tok {
7    Int(probl_number::Integer),
8    Float(f64),
9    /// A percentage, already divided by 100: `30%` is `Percent(0.3)`.
10    Percent(f64),
11    /// `2d6` is `Dice { count: 2, sides: 6 }`; `d6` has a count of 1.
12    Dice {
13        count: u32,
14        sides: u32,
15    },
16    Str(Vec<StrPart>),
17    /// Identifiers, including the contextual keywords `as`, `by`, `from` and `to`.
18    Ident(String),
19
20    // Keywords
21    And,
22    Break,
23    Catch,
24    Chance,
25    Continue,
26    Div,
27    Else,
28    Enum,
29    False,
30    Fn,
31    For,
32    If,
33    Import,
34    In,
35    Let,
36    Loop,
37    Match,
38    Mod,
39    Not,
40    Observe,
41    Score,
42    Or,
43    Repeat,
44    Report,
45    Return,
46    Simulate,
47    True,
48    Try,
49    Type,
50    Typeof,
51    Var,
52    While,
53    With,
54
55    // Punctuation
56    LParen,
57    RParen,
58    LBracket,
59    RBracket,
60    LBrace,
61    RBrace,
62    Comma,
63    Dot,
64    Colon,
65    Semi,
66    At,
67    Pipe,
68    Underscore,
69    /// `->`
70    Arrow,
71    /// `=>`
72    FatArrow,
73    Tilde,
74    Assign,
75    PlusAssign,
76    MinusAssign,
77    StarAssign,
78    SlashAssign,
79    EqEq,
80    NotEq,
81    Lt,
82    Le,
83    Gt,
84    Ge,
85    Plus,
86    Minus,
87    Star,
88    Slash,
89    Caret,
90    DotDot,
91    /// `..<`
92    DotDotLt,
93
94    Newline,
95    Eof,
96}
97
98/// A piece of a string literal: plain text, or the source of an interpolated `{expression}`.
99#[derive(Clone, Debug, PartialEq)]
100pub enum StrPart {
101    Lit(String),
102    Expr { src: String, offset: u32 },
103}
104
105#[derive(Clone, Debug, PartialEq)]
106pub struct Token {
107    pub tok: Tok,
108    pub span: Span,
109}
110
111/// Whether `text` can be a name in a program: ASCII letters, digits and
112/// `_`, not starting with a digit, and neither a keyword nor a die like `d6`.
113pub fn is_name(text: &str) -> bool {
114    let is_die = text
115        .strip_prefix('d')
116        .is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_digit()));
117    text.bytes()
118        .next()
119        .is_some_and(|b| b.is_ascii_alphabetic() || b == b'_')
120        && text.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_')
121        && text != "_"
122        && !is_die
123        && keyword(text).is_none()
124}
125
126pub fn keyword(word: &str) -> Option<Tok> {
127    Some(match word {
128        "and" => Tok::And,
129        "break" => Tok::Break,
130        "catch" => Tok::Catch,
131        "chance" => Tok::Chance,
132        "continue" => Tok::Continue,
133        "div" => Tok::Div,
134        "else" => Tok::Else,
135        "enum" => Tok::Enum,
136        "false" => Tok::False,
137        "fn" => Tok::Fn,
138        "for" => Tok::For,
139        "if" => Tok::If,
140        "import" => Tok::Import,
141        "in" => Tok::In,
142        "let" => Tok::Let,
143        "loop" => Tok::Loop,
144        "match" => Tok::Match,
145        "mod" => Tok::Mod,
146        "not" => Tok::Not,
147        "observe" => Tok::Observe,
148        "score" => Tok::Score,
149        "or" => Tok::Or,
150        "repeat" => Tok::Repeat,
151        "report" => Tok::Report,
152        "return" => Tok::Return,
153        "simulate" => Tok::Simulate,
154        "true" => Tok::True,
155        "try" => Tok::Try,
156        "type" => Tok::Type,
157        "typeof" => Tok::Typeof,
158        "var" => Tok::Var,
159        "while" => Tok::While,
160        "with" => Tok::With,
161        _ => return None,
162    })
163}
164
165impl Tok {
166    /// How the token is described in error messages.
167    pub fn describe(&self) -> String {
168        match self {
169            Tok::Int(v) => format!("number `{v}`"),
170            Tok::Float(v) => format!("number `{v}`"),
171            Tok::Percent(v) => format!("percentage `{}%`", v * 100.0),
172            Tok::Dice { count, sides } => format!("dice `{count}d{sides}`"),
173            Tok::Str(_) => "a string".to_string(),
174            Tok::Ident(name) => format!("`{name}`"),
175            Tok::Newline => "the end of the line".to_string(),
176            Tok::Eof => "the end of the file".to_string(),
177            other => format!("`{}`", other.text()),
178        }
179    }
180
181    /// Source text of keywords and punctuation.
182    pub fn text(&self) -> &'static str {
183        match self {
184            Tok::And => "and",
185            Tok::Break => "break",
186            Tok::Catch => "catch",
187            Tok::Chance => "chance",
188            Tok::Continue => "continue",
189            Tok::Div => "div",
190            Tok::Else => "else",
191            Tok::Enum => "enum",
192            Tok::False => "false",
193            Tok::Fn => "fn",
194            Tok::For => "for",
195            Tok::If => "if",
196            Tok::Import => "import",
197            Tok::In => "in",
198            Tok::Let => "let",
199            Tok::Loop => "loop",
200            Tok::Match => "match",
201            Tok::Mod => "mod",
202            Tok::Not => "not",
203            Tok::Observe => "observe",
204            Tok::Score => "score",
205            Tok::Or => "or",
206            Tok::Repeat => "repeat",
207            Tok::Report => "report",
208            Tok::Return => "return",
209            Tok::Simulate => "simulate",
210            Tok::True => "true",
211            Tok::Try => "try",
212            Tok::Type => "type",
213            Tok::Typeof => "typeof",
214            Tok::Var => "var",
215            Tok::While => "while",
216            Tok::With => "with",
217            Tok::LParen => "(",
218            Tok::RParen => ")",
219            Tok::LBracket => "[",
220            Tok::RBracket => "]",
221            Tok::LBrace => "{",
222            Tok::RBrace => "}",
223            Tok::Comma => ",",
224            Tok::Dot => ".",
225            Tok::Colon => ":",
226            Tok::Semi => ";",
227            Tok::At => "@",
228            Tok::Pipe => "|",
229            Tok::Underscore => "_",
230            Tok::Arrow => "->",
231            Tok::FatArrow => "=>",
232            Tok::Tilde => "~",
233            Tok::Assign => "=",
234            Tok::PlusAssign => "+=",
235            Tok::MinusAssign => "-=",
236            Tok::StarAssign => "*=",
237            Tok::SlashAssign => "/=",
238            Tok::EqEq => "==",
239            Tok::NotEq => "!=",
240            Tok::Lt => "<",
241            Tok::Le => "<=",
242            Tok::Gt => ">",
243            Tok::Ge => ">=",
244            Tok::Plus => "+",
245            Tok::Minus => "-",
246            Tok::Star => "*",
247            Tok::Slash => "/",
248            Tok::Caret => "^",
249            Tok::DotDot => "..",
250            Tok::DotDotLt => "..<",
251            Tok::Int(_)
252            | Tok::Float(_)
253            | Tok::Percent(_)
254            | Tok::Dice { .. }
255            | Tok::Str(_)
256            | Tok::Ident(_)
257            | Tok::Newline
258            | Tok::Eof => "",
259        }
260    }
261
262    pub fn is_ident(&self, word: &str) -> bool {
263        matches!(self, Tok::Ident(name) if name == word)
264    }
265}
probl_syntax/token.rs

probl_syntax/
token.rs
