workshop_rs/
lexer.rs

1//! Tokenizer for localized vanilla Workshop text.
2//!
3//! Tokens carry 1-based line/column spans so parser diagnostics and the
4//! produced WIR preserve source locations. Workshop identifiers are
5//! multi-word phrases; the tokenizer emits single [`Word`] tokens and the
6//! parser groups them, which keeps locale spellings data-driven.
7
8use crate::source::Position;
9
10/// A lexical token kind.
11#[derive(Debug, Clone, PartialEq)]
12pub enum TokenKind {
13    /// A run of identifier characters (`[A-Za-z_][A-Za-z0-9_]*`).
14    Word(String),
15    /// A numeric literal.
16    /// A numeric literal with its source spelling.
17    Number {
18        value: f64,
19        text: String,
20    },
21    /// A string literal (content, unescaped).
22    String(String),
23    /// An operator token: `= == != < <= > >= && || ! + - * / %`.
24    Op(String),
25    LParen,
26    RParen,
27    Comma,
28    Semi,
29    LBrace,
30    RBrace,
31    Colon,
32    Dot,
33    LBracket,
34    RBracket,
35    Eof,
36}
37
38/// A token with its source span.
39#[derive(Debug, Clone, PartialEq)]
40pub struct Token {
41    pub kind: TokenKind,
42    pub start: Position,
43    pub end: Position,
44}
45
46/// A lexing error with a source position.
47#[derive(Debug, Clone, PartialEq)]
48pub struct LexError {
49    pub message: String,
50    pub position: Position,
51}
52
53/// Tokenize Workshop text.
54pub fn tokenize(input: &str) -> Result<Vec<Token>, LexError> {
55    let chars: Vec<char> = input.chars().collect();
56    let mut tokens = Vec::new();
57    let mut index = 0;
58    let mut line = 1u32;
59    let mut col = 1u32;
60
61    macro_rules! pos {
62        () => {
63            Position::new(line, col)
64        };
65    }
66
67    let advance = |index: &mut usize, line: &mut u32, col: &mut u32, chars: &Vec<char>| {
68        let ch = chars[*index];
69        *index += 1;
70        if ch == '\n' {
71            *line += 1;
72            *col = 1;
73        } else {
74            *col += 1;
75        }
76    };
77
78    while index < chars.len() {
79        let ch = chars[index];
80        match ch {
81            ' ' | '\t' | '\r' | '\n' => advance(&mut index, &mut line, &mut col, &chars),
82            '(' => {
83                let start = pos!();
84                advance(&mut index, &mut line, &mut col, &chars);
85                tokens.push(Token {
86                    kind: TokenKind::LParen,
87                    start,
88                    end: pos!(),
89                });
90            }
91            ')' => {
92                let start = pos!();
93                advance(&mut index, &mut line, &mut col, &chars);
94                tokens.push(Token {
95                    kind: TokenKind::RParen,
96                    start,
97                    end: pos!(),
98                });
99            }
100            ',' => {
101                let start = pos!();
102                advance(&mut index, &mut line, &mut col, &chars);
103                tokens.push(Token {
104                    kind: TokenKind::Comma,
105                    start,
106                    end: pos!(),
107                });
108            }
109            ';' => {
110                let start = pos!();
111                advance(&mut index, &mut line, &mut col, &chars);
112                tokens.push(Token {
113                    kind: TokenKind::Semi,
114                    start,
115                    end: pos!(),
116                });
117            }
118            '{' => {
119                let start = pos!();
120                advance(&mut index, &mut line, &mut col, &chars);
121                tokens.push(Token {
122                    kind: TokenKind::LBrace,
123                    start,
124                    end: pos!(),
125                });
126            }
127            '}' => {
128                let start = pos!();
129                advance(&mut index, &mut line, &mut col, &chars);
130                tokens.push(Token {
131                    kind: TokenKind::RBrace,
132                    start,
133                    end: pos!(),
134                });
135            }
136            ':' => {
137                let start = pos!();
138                advance(&mut index, &mut line, &mut col, &chars);
139                tokens.push(Token {
140                    kind: TokenKind::Colon,
141                    start,
142                    end: pos!(),
143                });
144            }
145            '.' => {
146                let start = pos!();
147                advance(&mut index, &mut line, &mut col, &chars);
148                tokens.push(Token {
149                    kind: TokenKind::Dot,
150                    start,
151                    end: pos!(),
152                });
153            }
154            '[' => {
155                let start = pos!();
156                advance(&mut index, &mut line, &mut col, &chars);
157                tokens.push(Token {
158                    kind: TokenKind::LBracket,
159                    start,
160                    end: pos!(),
161                });
162            }
163            ']' => {
164                let start = pos!();
165                advance(&mut index, &mut line, &mut col, &chars);
166                tokens.push(Token {
167                    kind: TokenKind::RBracket,
168                    start,
169                    end: pos!(),
170                });
171            }
172            '"' => {
173                let start = pos!();
174                advance(&mut index, &mut line, &mut col, &chars);
175                let mut content = String::new();
176                let mut closed = false;
177                while index < chars.len() {
178                    let c = chars[index];
179                    // Decode the escape spellings the emitter produces so a
180                    // settings/value string round-trips byte-identically
181                    // (`\"`, `\\`, `\n`, `\r`, `\t`; #87).
182                    if c == '\\' && index + 1 < chars.len() {
183                        let escaped = chars[index + 1];
184                        let decoded = match escaped {
185                            '"' => Some('"'),
186                            '\\' => Some('\\'),
187                            'n' => Some('\n'),
188                            'r' => Some('\r'),
189                            't' => Some('\t'),
190                            _ => None,
191                        };
192                        if let Some(decoded) = decoded {
193                            advance(&mut index, &mut line, &mut col, &chars);
194                            advance(&mut index, &mut line, &mut col, &chars);
195                            content.push(decoded);
196                            continue;
197                        }
198                    }
199                    if c == '"' {
200                        advance(&mut index, &mut line, &mut col, &chars);
201                        closed = true;
202                        break;
203                    }
204                    advance(&mut index, &mut line, &mut col, &chars);
205                    content.push(c);
206                }
207                if !closed {
208                    return Err(LexError {
209                        message: "unterminated string literal".to_string(),
210                        position: start,
211                    });
212                }
213                tokens.push(Token {
214                    kind: TokenKind::String(content),
215                    start,
216                    end: pos!(),
217                });
218            }
219            '0'..='9' => {
220                let start = pos!();
221                let mut text = String::new();
222                while index < chars.len() && (chars[index].is_ascii_digit() || chars[index] == '.')
223                {
224                    text.push(chars[index]);
225                    advance(&mut index, &mut line, &mut col, &chars);
226                }
227                let value: f64 = text.parse().map_err(|_| LexError {
228                    message: format!("invalid number '{text}'"),
229                    position: start,
230                })?;
231                tokens.push(Token {
232                    kind: TokenKind::Number { value, text },
233                    start,
234                    end: pos!(),
235                });
236            }
237            '=' => {
238                let start = pos!();
239                advance(&mut index, &mut line, &mut col, &chars);
240                if index < chars.len() && chars[index] == '=' {
241                    advance(&mut index, &mut line, &mut col, &chars);
242                    tokens.push(Token {
243                        kind: TokenKind::Op("==".to_string()),
244                        start,
245                        end: pos!(),
246                    });
247                } else {
248                    tokens.push(Token {
249                        kind: TokenKind::Op("=".to_string()),
250                        start,
251                        end: pos!(),
252                    });
253                }
254            }
255            '!' => {
256                let start = pos!();
257                advance(&mut index, &mut line, &mut col, &chars);
258                if index < chars.len() && chars[index] == '=' {
259                    advance(&mut index, &mut line, &mut col, &chars);
260                    tokens.push(Token {
261                        kind: TokenKind::Op("!=".to_string()),
262                        start,
263                        end: pos!(),
264                    });
265                } else {
266                    tokens.push(Token {
267                        kind: TokenKind::Op("not".to_string()),
268                        start,
269                        end: pos!(),
270                    });
271                }
272            }
273            '&' | '|' | '?' => {
274                if ch == '?' {
275                    let start = pos!();
276                    advance(&mut index, &mut line, &mut col, &chars);
277                    tokens.push(Token {
278                        kind: TokenKind::Op("?".to_string()),
279                        start,
280                        end: pos!(),
281                    });
282                    continue;
283                }
284                let start = pos!();
285                let operator = ch;
286                advance(&mut index, &mut line, &mut col, &chars);
287                if index < chars.len() && chars[index] == operator {
288                    advance(&mut index, &mut line, &mut col, &chars);
289                    tokens.push(Token {
290                        kind: TokenKind::Op(if operator == '&' {
291                            "and".to_string()
292                        } else {
293                            "or".to_string()
294                        }),
295                        start,
296                        end: pos!(),
297                    });
298                } else {
299                    return Err(LexError {
300                        message: format!(
301                            "unexpected '{operator}'; expected '{operator}{operator}'"
302                        ),
303                        position: start,
304                    });
305                }
306            }
307            '<' | '>' => {
308                let start = pos!();
309                let op = ch.to_string();
310                advance(&mut index, &mut line, &mut col, &chars);
311                if index < chars.len() && chars[index] == '=' {
312                    advance(&mut index, &mut line, &mut col, &chars);
313                    tokens.push(Token {
314                        kind: TokenKind::Op(format!("{op}=")),
315                        start,
316                        end: pos!(),
317                    });
318                } else {
319                    tokens.push(Token {
320                        kind: TokenKind::Op(op),
321                        start,
322                        end: pos!(),
323                    });
324                }
325            }
326            '+' | '*' | '/' | '%' => {
327                if ch == '/' && index + 1 < chars.len() && chars[index + 1] == '/' {
328                    // `//` line comments: the reference's Workshop export
329                    // format carries element-count comments between rules
330                    // (#119 differential evidence); skip to end of line.
331                    while index < chars.len() && chars[index] != '\n' {
332                        advance(&mut index, &mut line, &mut col, &chars);
333                    }
334                    continue;
335                }
336                let start = pos!();
337                let op = ch.to_string();
338                advance(&mut index, &mut line, &mut col, &chars);
339                tokens.push(Token {
340                    kind: TokenKind::Op(op),
341                    start,
342                    end: pos!(),
343                });
344            }
345            '-' => {
346                let start = pos!();
347                advance(&mut index, &mut line, &mut col, &chars);
348                tokens.push(Token {
349                    kind: TokenKind::Op("-".to_string()),
350                    start,
351                    end: pos!(),
352                });
353            }
354            c if is_word_start(c) => {
355                let start = pos!();
356                let mut word = String::new();
357                while index < chars.len() {
358                    let c = chars[index];
359                    let interior_dash = c == '-'
360                        && index + 1 < chars.len()
361                        && (chars[index + 1].is_alphanumeric() || chars[index + 1] == '_');
362                    if is_word_character(c) || interior_dash {
363                        word.push(c);
364                        advance(&mut index, &mut line, &mut col, &chars);
365                    } else {
366                        break;
367                    }
368                }
369                tokens.push(Token {
370                    kind: TokenKind::Word(word),
371                    start,
372                    end: pos!(),
373                });
374            }
375            other => {
376                return Err(LexError {
377                    message: format!("unexpected character '{other}'"),
378                    position: pos!(),
379                });
380            }
381        }
382    }
383
384    let end = pos!();
385    tokens.push(Token {
386        kind: TokenKind::Eof,
387        start: end,
388        end,
389    });
390    Ok(tokens)
391}
392
393fn is_word_start(ch: char) -> bool {
394    ch.is_alphabetic() || ch == '_' || is_raw_label_punctuation(ch)
395}
396
397fn is_word_character(ch: char) -> bool {
398    ch.is_alphanumeric() || ch == '_' || ch == '\'' || is_raw_label_punctuation(ch)
399}
400
401fn is_raw_label_punctuation(ch: char) -> bool {
402    matches!(
403        ch,
404        '“' | '”' | '（' | '）' | '，' | '、' | '！' | '？' | '：' | '【' | '】' | '［' | '］'
405    )
406}
workshop_rs/lexer.rs

workshop_rs/
lexer.rs
