workshop_rs/frontend/
parser.rs

1//! Native localized Workshop parser.
2//!
3//! Parses vanilla Workshop text directly into validated, locale-independent
4//! the canonical public [`Program`]. Localized actions, values, events, enums, and structural
5//! keywords resolve through the canonical catalog; malformed input,
6//! unknown spellings, and recognized-but-unsupported constructs are reported
7//! as distinct structured diagnostics with source spans.
8
9pub(crate) use std::collections::HashMap;
10use std::sync::OnceLock;
11
12pub(crate) use crate::core::signatures::ExpectedDomain;
13pub(crate) use crate::core::source::{Position, SourceFile, Span};
14use crate::program::Program;
15pub(crate) use crate::settings::table::KeyKind;
16pub(crate) use crate::settings::{PathPart, table};
17pub(crate) use crate::settings::{Settings, SettingsListElement, SettingsNode};
18pub(crate) use crate::wir::{
19    self, Action, Event, EventTarget, EventTeam, ModifyOp, PlayerEventKind, Value, ValueNode,
20};
21
22pub(crate) use super::lexer::{Token, TokenKind, tokenize};
23pub(crate) use crate::catalog::{Catalog, Kind, Locale};
24pub(crate) use crate::core::error::{Result, WorkshopError};
25
26/// Where action parsing stopped.
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub(crate) enum Stop {
29    /// The enclosing `}` was consumed.
30    SectionClosed,
31    /// The `End` keyword is next (not consumed).
32    End,
33    /// The `Else If` keyword is next (not consumed).
34    ElseIf,
35    /// The `Else` keyword is next (not consumed).
36    Else,
37}
38
39pub(crate) enum AssignmentOperator {
40    Set,
41    Modify(ModifyOp),
42}
43
44/// Parse localized Workshop text into Workshop IR using the catalog's
45/// canonical call-signature context. Ambiguous bare enum members resolve when
46/// their enclosing call pins one matching domain; unpinned ambiguity remains a
47/// structured canonical value. See [`parse_with_context`] when a
48/// consumer needs to provide additional signature context.
49pub fn parse(input: &str, catalog: &Catalog, locale: &Locale) -> Result<Program> {
50    parse_with_context(input, catalog, locale, catalog)
51}
52
53#[cfg(test)]
54pub(crate) fn parse_wir(input: &str, catalog: &Catalog, locale: &Locale) -> Result<wir::Program> {
55    parse_wir_with_context(input, catalog, locale, catalog)
56}
57
58/// Compatibility entry point for crate-internal storage tests and migration
59/// checks. Ordinary consumers should use [`parse`], which returns [`Program`].
60/// Parse localized Workshop text into Workshop IR, resolving ambiguous bare
61/// enum members from the enclosing call's canonical signature context (#111).
62///
63/// When a bare member spelling matches several enum domains, the parser asks
64/// [`ExpectedDomain::expected_domain`] for the domain the enclosing call's
65/// signature expects at that argument position; the member resolves only when
66/// that expected domain is one of the matching domains (i.e. the signature
67/// pins exactly one). Without a pin all matching candidates are preserved.
68pub fn parse_with_context(
69    input: &str,
70    catalog: &Catalog,
71    locale: &Locale,
72    context: &dyn ExpectedDomain,
73) -> Result<Program> {
74    parse_wir_with_context(input, catalog, locale, context).and_then(Program::from_wir)
75}
76
77pub(crate) fn parse_wir_with_context(
78    input: &str,
79    catalog: &Catalog,
80    locale: &Locale,
81    context: &dyn ExpectedDomain,
82) -> Result<wir::Program> {
83    let tokens = tokenize(input).map_err(|error| WorkshopError::Malformed {
84        message: error.message,
85        span: Some(synthetic_span(error.position)),
86    })?;
87    ParseContext {
88        tokens,
89        pos: 0,
90        source: input,
91        catalog,
92        locale: locale.clone(),
93        context,
94        expected_domain: None,
95        call_stack: Vec::new(),
96        target: wir::Program::default(),
97        globals: HashMap::new(),
98        players: HashMap::new(),
99        subroutines: HashMap::new(),
100        next_global_index: 0,
101        next_player_index: 0,
102    }
103    .program()
104}
105
106/// A synthetic single-position span (used before a file registry exists).
107pub(crate) fn synthetic_span(position: Position) -> Span {
108    Span::new(crate::core::ids::Id::from_index(0), position, position)
109}
110
111pub(crate) struct ParseContext<'a> {
112    pub(crate) tokens: Vec<Token>,
113    pub(crate) pos: usize,
114    pub(crate) source: &'a str,
115    pub(crate) catalog: &'a Catalog,
116    pub(crate) locale: Locale,
117    /// Canonical signature context (#111): supplies the expected enum domain
118    /// for the call argument currently being parsed.
119    pub(crate) context: &'a dyn ExpectedDomain,
120    /// The expected enum domain for the value currently being parsed, set by
121    /// [`ParseContext::value_args`] from the enclosing call's signature.
122    pub(crate) expected_domain: Option<&'a str>,
123    pub(crate) call_stack: Vec<String>,
124    pub(crate) target: wir::Program,
125    pub(crate) globals: HashMap<String, wir::GlobalVarId>,
126    pub(crate) players: HashMap<String, wir::PlayerVarId>,
127    pub(crate) subroutines: HashMap<String, wir::SubroutineId>,
128    pub(crate) next_global_index: u32,
129    pub(crate) next_player_index: u32,
130}
131
132impl<'a> ParseContext<'a> {
133    /// Run `f` with the signature-pinned expected domain temporarily replaced,
134    /// restoring the enclosing domain afterwards (#111).
135    pub(crate) fn with_domain<T>(
136        &mut self,
137        domain: Option<&'a str>,
138        f: impl FnOnce(&mut Self) -> Result<T>,
139    ) -> Result<T> {
140        let saved = self.expected_domain;
141        self.expected_domain = domain;
142        let result = f(self);
143        self.expected_domain = saved;
144        result
145    }
146
147    /// Parse a value while `domain` is in effect (or `None` for value
148    /// positions a signature must not pin).
149    pub(crate) fn value_in_domain(&mut self, domain: Option<&'a str>) -> Result<wir::ValueId> {
150        self.with_domain(domain, |p| p.value())
151    }
152
153    pub(crate) fn resolve_entry(
154        &self,
155        kind: Kind,
156        spelling: &str,
157    ) -> Option<&'a crate::catalog::CatalogEntry> {
158        self.catalog
159            .resolve(kind, &self.locale, spelling)
160            .or_else(|| {
161                if self.locale != *self.catalog.primary_locale() {
162                    self.catalog
163                        .resolve(kind, self.catalog.primary_locale(), spelling)
164                } else {
165                    None
166                }
167            })
168    }
169
170    pub(crate) fn canonical_keyword(&self, spelling: &str) -> String {
171        self.catalog
172            .resolve(Kind::Structural, &self.locale, spelling)
173            .or_else(|| {
174                (self.locale != *self.catalog.primary_locale()).then(|| {
175                    self.catalog
176                        .resolve(Kind::Structural, self.catalog.primary_locale(), spelling)
177                })?
178            })
179            .map(|entry| entry.id.clone())
180            .unwrap_or_else(|| canonical_keyword(spelling).to_string())
181    }
182
183    pub(crate) fn line_has_assignment(&self) -> bool {
184        let tokens: Vec<_> = self.tokens[self.pos..]
185            .iter()
186            .take_while(|token| !matches!(token.kind, TokenKind::Semi | TokenKind::RBrace))
187            .collect();
188        tokens.iter().any(|token| {
189            matches!(&token.kind, TokenKind::Op(op) if matches!(op.as_str(), "=" | "+=" | "-=" | "*=" | "/=" | "%="))
190        }) || tokens.windows(2).any(|window| {
191            matches!(&window[0].kind, TokenKind::Word(_))
192                && matches!(&window[1].kind, TokenKind::Op(op) if op == "=")
193        })
194    }
195
196    /// Read the maximal phrase of consecutive words (space-joined). Phrases
197    /// may span lines because long Workshop action arguments wrap mid-phrase.
198    pub(crate) fn phrase(&mut self) -> Result<(String, Position, Position)> {
199        let mut words = Vec::new();
200        let (start, mut end) = match self.peek() {
201            Some(Token {
202                kind: TokenKind::Word(word),
203                start,
204                end,
205            }) => {
206                words.push(word.clone());
207                (start, end)
208            }
209            Some(Token {
210                kind: TokenKind::Number { text, .. },
211                start,
212                end,
213            }) => {
214                words.push(text.clone());
215                (start, end)
216            }
217            Some(token) => return Err(self.malformed("expected an identifier", &token)),
218            None => return Err(self.malformed("expected an identifier", self.eof())),
219        };
220        self.pos += 1;
221        while let Some(token) = self.peek() {
222            match token {
223                Token {
224                    kind: TokenKind::Word(word),
225                    end: word_end,
226                    ..
227                } => {
228                    if matches!(
229                        self.peek_at(1).map(|token| token.kind),
230                        Some(TokenKind::Op(equal)) if equal == "="
231                    ) {
232                        break;
233                    }
234                    words.push(word.clone());
235                    end = word_end;
236                    self.pos += 1;
237                }
238                // Enum members embed numbers (`Team 2`, `Ability 2`); the
239                // lexer splits them from the word, so phrases join Number
240                // tokens too (#119 reference evidence).
241                Token {
242                    kind: TokenKind::Number { text, .. },
243                    end: number_end,
244                    ..
245                } => {
246                    words.push(text.clone());
247                    end = number_end;
248                    self.pos += 1;
249                }
250                _ => break,
251            }
252        }
253        Ok((words.join(" "), start, end))
254    }
255
256    /// Read a single-line phrase (stops at a line boundary). Used for names
257    /// that are structurally one per line, such as variable declarations.
258    pub(crate) fn phrase_on_line(&mut self) -> Result<(String, Position, Position)> {
259        self.phrase_on_line_impl(false)
260    }
261
262    /// Read a settings list element name: adjacent tokens stay joined and
263    /// `:` is part of the name, preserving localized spellings such as
264    /// `66号公路` and `Watchpoint: Gibraltar`.
265    pub(crate) fn settings_list_name(&mut self) -> Result<(String, Position, Position)> {
266        self.phrase_on_line_impl(true)
267    }
268
269    fn phrase_on_line_impl(&mut self, list_name: bool) -> Result<(String, Position, Position)> {
270        let mut words = Vec::new();
271        let (start, mut end, line) = match self.peek() {
272            Some(Token {
273                kind: TokenKind::Word(word),
274                start,
275                end,
276            }) => {
277                words.push(word.clone());
278                (start, end, start.line)
279            }
280            Some(Token {
281                kind: TokenKind::Number { text, .. },
282                start,
283                end,
284            }) => {
285                words.push(text.clone());
286                (start, end, start.line)
287            }
288            Some(token) => return Err(self.malformed("expected an identifier", &token)),
289            None => return Err(self.malformed("expected an identifier", self.eof())),
290        };
291        self.pos += 1;
292        while let Some(token) = self.peek() {
293            let (word, word_start, word_end) = match token {
294                Token {
295                    kind: TokenKind::Word(word),
296                    start,
297                    end,
298                } => (word, start, end),
299                Token {
300                    kind: TokenKind::Number { text, .. },
301                    start,
302                    end,
303                } => (text, start, end),
304                Token {
305                    kind: TokenKind::Dot,
306                    start,
307                    end,
308                } => (".".to_string(), start, end),
309                Token {
310                    kind: TokenKind::Op(op),
311                    start,
312                    end,
313                } if matches!(op.as_str(), "-" | "%") => (op.clone(), start, end),
314                Token {
315                    kind: TokenKind::Colon,
316                    start,
317                    end,
318                } if list_name => (":".to_string(), start, end),
319                _ => break,
320            };
321            if word_start.line != line {
322                break;
323            }
324            if list_name && word_start == end {
325                words
326                    .last_mut()
327                    .expect("phrase has a first token")
328                    .push_str(&word);
329            } else {
330                words.push(word);
331            }
332            end = word_end;
333            self.pos += 1;
334        }
335        let joined = words
336            .join(" ")
337            .replace(" .", ".")
338            .replace(". ", ".")
339            .replace(" %", "%");
340        // Canonical spellings keep the spacing around ':' (e.g. fr-FR
341        // "Observatoire : Gibraltar"); list names must not collapse it.
342        let joined = if list_name {
343            joined
344        } else {
345            joined.replace(" : ", ":")
346        };
347        Ok((joined, start, end))
348    }
349
350    pub(crate) fn phrase_on_line_with_colon(&mut self) -> Result<(String, Position, Position)> {
351        let (mut phrase, start, mut end) = self.phrase_on_line()?;
352        if matches!(
353            self.peek(),
354            Some(Token {
355                kind: TokenKind::Colon,
356                ..
357            })
358        ) {
359            let colon_pos = self.pos;
360            self.next();
361            let (rest, _, rest_end) = self.phrase_on_line()?;
362            if matches!(
363                self.peek(),
364                Some(Token {
365                    kind: TokenKind::LBrace,
366                    ..
367                })
368            ) {
369                phrase.push(':');
370                phrase.push(' ');
371                phrase.push_str(&rest);
372                end = rest_end;
373            } else {
374                self.pos = colon_pos;
375            }
376        }
377        Ok((phrase, start, end))
378    }
379
380    pub(crate) fn enum_member_phrase(&mut self) -> Result<(String, Position, Position)> {
381        let first = self
382            .peek()
383            .ok_or_else(|| self.malformed("expected an enum member", self.eof()))?;
384        let start = first.start;
385        let line = first.start.line;
386        let mut end = first.end;
387        let mut parts = Vec::new();
388        while let Some(token) = self.peek() {
389            if token.start.line != line
390                || matches!(token.kind, TokenKind::RParen | TokenKind::Comma)
391            {
392                break;
393            }
394            self.pos += 1;
395            end = token.end;
396            parts.push(raw_token_text(&token.kind));
397        }
398        if parts.is_empty() {
399            return Err(self.malformed("expected an enum member", &first));
400        }
401        Ok((
402            parts
403                .join(" ")
404                .replace(" : ", ":")
405                .replace(" .", ".")
406                .replace(". ", "."),
407            start,
408            end,
409        ))
410    }
411
412    /// Read a text line (tokens until `;`), joining words and dashes into
413    /// the literal text, and consume the terminating `;`. Returns the text
414    /// and the span covering the line's content tokens.
415    pub(crate) fn line_text(&mut self) -> Result<(String, Option<Span>)> {
416        let mut parts = Vec::new();
417        let mut start = None;
418        let mut end = None;
419        loop {
420            match self.peek() {
421                Some(Token {
422                    kind: TokenKind::Semi,
423                    ..
424                }) => {
425                    self.pos += 1;
426                    break;
427                }
428                Some(token) => {
429                    start.get_or_insert(token.start);
430                    end = Some(token.end);
431                    match &token.kind {
432                        TokenKind::Word(word) => parts.push(word.clone()),
433                        TokenKind::Op(op) if op == "-" => parts.push("-".to_string()),
434                        TokenKind::Number { value, .. } => parts.push(value.to_string()),
435                        TokenKind::Dot => parts.push(".".to_string()),
436                        TokenKind::Colon => parts.push(":".to_string()),
437                        _ => return Err(self.malformed("expected a text line", &token)),
438                    }
439                    self.pos += 1;
440                }
441                None => {
442                    return Err(self.malformed("unexpected end of input in line", self.eof()));
443                }
444            }
445        }
446        Ok((
447            parts
448                .join(" ")
449                .replace(" .", ".")
450                .replace(". ", ".")
451                .replace(" : ", ":"),
452            start
453                .zip(end)
454                .map(|(start, end)| Span::new(self.file(), start, end)),
455        ))
456    }
457
458    /// Consume a known keyword phrase, verifying its spelling.
459    pub(crate) fn consume_phrase(&mut self, expected: &str) -> Result<()> {
460        let (phrase, _, _) = self.phrase()?;
461        if phrase != expected {
462            return Err(self.malformed(&format!("expected '{expected}'"), self.previous()));
463        }
464        Ok(())
465    }
466
467    pub(crate) fn expect_keyword(&mut self, expected: &str) -> Result<Position> {
468        match self.next() {
469            Some(Token {
470                kind: TokenKind::Word(word),
471                start,
472                ..
473            }) if self.canonical_keyword(&word) == expected => Ok(start),
474            Some(token) => Err(self.malformed(&format!("expected '{expected}'"), &token)),
475            None => Err(self.malformed(&format!("expected '{expected}'"), self.eof())),
476        }
477    }
478
479    pub(crate) fn expect(&mut self, kind: TokenKind, message: &str) -> Result<()> {
480        match self.next() {
481            Some(token) if token.kind == kind => Ok(()),
482            Some(token) => Err(self.malformed(message, &token)),
483            None => Err(self.malformed(message, self.eof())),
484        }
485    }
486
487    pub(crate) fn expect_string(&mut self, message: &str) -> Result<String> {
488        match self.next() {
489            Some(Token {
490                kind: TokenKind::String(content),
491                ..
492            }) => Ok(content),
493            Some(token) => Err(self.malformed(message, &token)),
494            None => Err(self.malformed(message, self.eof())),
495        }
496    }
497
498    pub(crate) fn malformed(&self, message: &str, token: &Token) -> WorkshopError {
499        WorkshopError::Malformed {
500            message: message.to_string(),
501            span: Some(Span::new(self.file(), token.start, token.end)),
502        }
503    }
504
505    pub(crate) fn unknown(&self, kind: &'static str, spelling: &str) -> WorkshopError {
506        WorkshopError::Unknown {
507            kind,
508            spelling: spelling.to_string(),
509            locale: self.locale.clone(),
510            span: None,
511        }
512    }
513
514    pub(crate) fn peek(&self) -> Option<Token> {
515        self.tokens.get(self.pos).cloned()
516    }
517
518    pub(crate) fn peek_at(&self, offset: usize) -> Option<Token> {
519        self.tokens.get(self.pos + offset).cloned()
520    }
521
522    pub(crate) fn next(&mut self) -> Option<Token> {
523        let token = self.tokens.get(self.pos).cloned();
524        if token.is_some() {
525            self.pos += 1;
526        }
527        token
528    }
529
530    pub(crate) fn previous(&self) -> &Token {
531        self.tokens
532            .get(self.pos.saturating_sub(1))
533            .unwrap_or_else(|| self.tokens.last().unwrap())
534    }
535
536    pub(crate) fn previous_span(&self) -> (Position, Position) {
537        let token = self.previous();
538        (token.start, token.end)
539    }
540
541    pub(crate) fn span_here(&self) -> (Position, Position) {
542        let token = self
543            .peek()
544            .unwrap_or_else(|| self.tokens.last().unwrap().clone());
545        (token.start, token.end)
546    }
547
548    pub(crate) fn eof(&self) -> &Token {
549        self.tokens.last().unwrap()
550    }
551
552    pub(crate) fn file(&self) -> crate::core::ids::Id<SourceFile> {
553        crate::core::ids::Id::from_index(0)
554    }
555}
556
557pub(crate) fn canonical_keyword(keyword: &str) -> &str {
558    static KEYWORDS: OnceLock<HashMap<String, String>> = OnceLock::new();
559    KEYWORDS
560        .get_or_init(|| {
561            serde_json::from_str(include_str!("structural_keywords.json"))
562                .expect("structural keyword data is valid JSON")
563        })
564        .get(keyword)
565        .map(String::as_str)
566        .unwrap_or(keyword)
567}
568
569pub(crate) fn raw_token_text(kind: &TokenKind) -> String {
570    match kind {
571        TokenKind::Word(value) => value.clone(),
572        TokenKind::Number { text, .. } => text.clone(),
573        TokenKind::String(value) => format!("\"{}\"", value.replace('"', "\\\"")),
574        TokenKind::Op(value) => value.clone(),
575        TokenKind::LParen => "(".to_string(),
576        TokenKind::RParen => ")".to_string(),
577        TokenKind::Comma => ",".to_string(),
578        TokenKind::Semi => ";".to_string(),
579        TokenKind::LBrace => "{".to_string(),
580        TokenKind::RBrace => "}".to_string(),
581        TokenKind::Colon => ":".to_string(),
582        TokenKind::Dot => ".".to_string(),
583        TokenKind::LBracket => "[".to_string(),
584        TokenKind::RBracket => "]".to_string(),
585        TokenKind::Eof => String::new(),
586    }
587}
workshop_rs/frontend/parser.rs

workshop_rs/frontend/
parser.rs
